def createTrainingVectors(tokenized_texts_dict):
    """
        Given the filenames and their contents, this methods creates the training 
        vectors by creating a unique list of all words together in the training
        set
    """
    print("Creating vectors for training data")

    unique_words = []
    for filename, text in tokenized_texts_dict.iteritems():
        # print("Reading {0} and adding to unique word list".format(filename))
        unique_words.extend(word_tokenize(text))

    unique_words = set(unique_words)

    # Creating the initial vector with counts 0 for all training sets
    zero_vector = OrderedDict(zip(unique_words, [0] * len(unique_words)))
    print("Creating the zero vector")

    # For each training file, create an OrderedDict containing its word counts (together with zero counts),
    # and store it in a dict, indexed by its corresponding filename
    vectors = {}
    for filename, token_list in tokenized_texts_dict.iteritems():
        current_vector = zero_vector.copy()
        current_vector.update(Counter(word_tokenize(token_list)))
        vectors[filename] = current_vector

    return vectors, zero_vector
Beispiel #2
0
class ValidatorRegister(object):
    """Validators register."""

    def __init__(self):
        """Initialize the register. Take no parameters."""
        # Dictionary of (code, description) pairs
        self._errors = OrderedDict()
        # Dictionary of (code, callback) pairs
        self._validators = OrderedDict()

    @property
    def errors(self):
        """Return dictionary of (error_code, error_description) pairs."""
        return self._errors.copy()

    @property
    def validators(self):
        """Return dictionary of (error_code, callback) pairs."""
        return self._validators.copy()

    def register(self, callback, error_code, error_description):
        """Register validator.

        @param callback: Function which performs validation.
        @type callback: function
        @param error_code: Error code which will be reported if validation fails.
        @type error_code: text
        @param error_description: Error description which will be reported if validation fails.
        @type error_description: text
        @raises ValueError: If `error_code` is already registered.
        """
        if error_code in self._validators:
            raise ValueError('Validator for %s is already registered.' % error_code)
        self._errors[error_code] = error_description
        self._validators[error_code] = callback
Beispiel #3
0
def parse_input_volfile(orig_volfile):
    volfile = OrderedDict() 
    with open(orig_volfile, "r") as f:
        for line in f:
            try:
                isinstance(opt_dict, dict)
            except UnboundLocalError:
                opt_dict = OrderedDict()

            try:
                isinstance(sub_dict, dict)
            except UnboundLocalError:
                sub_dict = OrderedDict()

            temp = line.split()
            if not temp:
                continue
            if line.startswith("volume "):
                xl_name = temp[1]
            elif temp[0] == "type":
                sub_dict[temp[0]] = temp[1]
                sub_dict["options"] = OrderedDict()
            elif temp[0] == "option":
                opt_dict[temp[1]] = temp[2]
            elif temp[0] == "subvolumes":
                sub_dict[temp[0]] = temp[1:]
            elif line.startswith("end-volume"):
                sub_dict["options"] = opt_dict.copy()
                volfile[xl_name] = sub_dict.copy()
                del(opt_dict)
                del(sub_dict)
            else:
                print "Oops!!! Bug, Parser needs an update..."
                sys.exit(1)
    return volfile
Beispiel #4
0
class DisorderManager(object):
    implements(IHandler, IDisorderObserver)

    def __init__(self, handlers):
        for handler in handlers:
            if IRetrieveCapable.providedBy(handler):
                self.retriever = handler
                break
        else:
            self.retriever = None

        self.seekers = {}
        self.disorders = OrderedDict()

        self.observers = []

    def add(self, key, seeker):
        seeker.add_observer(self, key)
        self.seekers[key] = seeker

        self.disorders[key] = None
        if self.retriever and IReplayable.providedBy(seeker):
            seeker.replay(self.retriever)

    def there_is_disorder(self, key, disorder):
        self.set_disorder_value(key, MaybeDisorder(True, disorder))

    def there_is_no_disorder(self, key, disorder):
        self.set_disorder_value(key, MaybeDisorder(False, disorder))

    def seeker_is_not_functional(self, key):
        self.set_disorder_value(key, None)

    def set_disorder_value(self, key, value):
        if value != self.disorders[key]:
            self.disorders[key] = value
            self.notify_observers()

    def add_observer(self, observer):
        self.observers.append(observer)
        observer.update_disorders(self.disorders.copy())

    def notify_observers(self):
        for observer in self.observers:
            observer.update_disorders(self.disorders.copy())

    def initialize(self):
        pass

    def process(self, record):
        for seeker in self.seekers.values():
            seeker.receive_record(record)
Beispiel #5
0
class YkmanContextObject(MutableMapping):
    def __init__(self):
        self._objects = OrderedDict()
        self._resolved = False

    def add_resolver(self, key, f):
        if self._resolved:
            f = f()
        self._objects[key] = f

    def resolve(self):
        if not self._resolved:
            self._resolved = True
            for k, f in self._objects.copy().items():
                self._objects[k] = f()

    def __getitem__(self, key):
        self.resolve()
        return self._objects[key]

    def __setitem__(self, key, value):
        if not self._resolved:
            raise ValueError('BUG: Attempted to set item when unresolved.')
        self._objects[key] = value

    def __delitem__(self, key):
        del self._objects[key]

    def __len__(self):
        return len(self._objects)

    def __iter__(self):
        return iter(self._objects)
Beispiel #6
0
def _post(uri, data, ext_encode_data=None, add_security_key=True):
    """
    :type uri: str
    :type data: OrderedDict
    :rtype: Response
    """
    request_data = OrderedDict(MerchantId=MERCHANT_ID)
    request_data.update(data)
    if add_security_key:
        encode_data = request_data.copy()
        encode_data.update(ext_encode_data or {})
        request_data['SecurityKey'] = _gen_security_key(encode_data)

    try:
        r = requests.post(base_url + uri, data=request_data)
        if r.text:
            print r.text
            _create_log(uri, request_data, r.text)
        else:
            print r.headers
        return Response(body=r.text)
    except:
        print 'Error Payonline'
        print traceback.format_exc()
        return ErrorResponse()
def naive_model(game, df):
    state_rep = ['time', 'home_score', 'away_score']

    new_game = game.copy()
    new_game['win'] = pd.Series()
    new_game['tie'] = pd.Series()
    new_game['win_or_tie'] = pd.Series()

    for row in new_game.index:

        params = OrderedDict([(k,game.iloc[row][k]) for k in state_rep])
        params_away = params.copy()
        params_away['home_score'], params_away['away_score'] = params['away_score'], params['home_score']

        home = probability_of_consolid_state(params, state_rep, df)
        away = probability_of_consolid_state(params_away, state_rep, df)

        if home is None or away is None:
            continue

        win, tie = home_win(home, away), home_tie(home, away)
        new_game['win'].ix[row] = win
        new_game['tie'].ix[row] = tie
        new_game['win_or_tie'].ix[row] = win+tie    

    return new_game
def goal_diff_model(game, df):
    new_game = game.copy()
    new_game['win'] = pd.Series()
    new_game['tie'] = pd.Series()
    new_game['win_or_tie'] = pd.Series()
    state_match = ['time', 'diff']

    for row in new_game.index:
                
        paramHome = OrderedDict([(k,game.iloc[row][k]) for k in state_match])
        paramAway = paramHome.copy()

        paramAway['diff'] = -1 * paramHome['diff']

        home = probability_of_consolid_state(paramHome, state_match, df)
        away = probability_of_consolid_state(paramAway, state_match, df)

        if home is None or away is None:
            continue

        win, tie = home_win(home, away), home_tie(home, away)

        new_game['win'].ix[row] = win
        new_game['tie'].ix[row] = tie
        new_game['win_or_tie'].ix[row] = win+tie

    return new_game
class FrozenOrderedDict(Mapping):
    """
    Frozen OrderedDict.
    """

    def __init__(self, *args, **kwargs):
        self.__dict = OrderedDict(*args, **kwargs)
        self.__hash = None

    def __getitem__(self, item):
        return self.__dict[item]

    def __iter__(self):
        return iter(self.__dict)

    def __len__(self):
        return len(self.__dict)

    def __hash__(self):
        if self.__hash is None:
            self.__hash = reduce(operator.xor, map(hash, self.iteritems()), 0)

        return self.__hash

    def __repr__(self):
        return '{}({!r})'.format(self.__class__.__name__, self.items())

    def copy(self, *args, **kwargs):
        new_dict = self.__dict.copy()

        if args or kwargs:
            new_dict.update(OrderedDict(*args, **kwargs))

        return self.__class__(new_dict)
Beispiel #10
0
 def test_copying(self):
     # Check that ordered dicts are copyable, deepcopyable, picklable,
     # and have a repr/eval round-trip
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     od = OrderedDict(pairs)
     update_test = OrderedDict()
     update_test.update(od)
     for i, dup in enumerate([
                 od.copy(),
                 copy.copy(od),
                 copy.deepcopy(od),
                 pickle.loads(pickle.dumps(od, 0)),
                 pickle.loads(pickle.dumps(od, 1)),
                 pickle.loads(pickle.dumps(od, 2)),
                 pickle.loads(pickle.dumps(od, 3)),
                 pickle.loads(pickle.dumps(od, -1)),
                 eval(repr(od)),
                 update_test,
                 OrderedDict(od),
                 ]):
         self.assert_(dup is not od)
         self.assertEquals(dup, od)
         self.assertEquals(list(dup.items()), list(od.items()))
         self.assertEquals(len(dup), len(od))
         self.assertEquals(type(dup), type(od))
Beispiel #11
0
 def _fill_related_objects_cache(self):
     cache = OrderedDict()
     parent_list = self.get_parent_list()
     for parent in self.parents:
         for obj, model in parent._meta.get_all_related_objects_with_model(include_hidden=True):
             if (obj.field.creation_counter < 0 or obj.field.rel.parent_link) and obj.model not in parent_list:
                 continue
             if not model:
                 cache[obj] = parent
             else:
                 cache[obj] = model
     # Collect also objects which are in relation to some proxy child/parent of self.
     proxy_cache = cache.copy()
     for klass in self.apps.get_models(include_auto_created=True):
         if not klass._meta.swapped:
             for f in klass._meta.local_fields + klass._meta.virtual_fields:
                 if (hasattr(f, 'rel') and f.rel and not isinstance(f.rel.to, six.string_types)
                         and f.generate_reverse_relation):
                     if self == f.rel.to._meta:
                         cache[f.related] = None
                         proxy_cache[f.related] = None
                     elif self.concrete_model == f.rel.to._meta.concrete_model:
                         proxy_cache[f.related] = None
     self._related_objects_cache = cache
     self._related_objects_proxy_cache = proxy_cache
Beispiel #12
0
class Buckets(object):
    """Proxy for OrderedDict"""
    def __init__(self, *args, **kwargs):
        self._od = OrderedDict(*args, **kwargs)

    def __getattr__(self, a):
        return getattr(self._od, a)

    def __setitem__(self, *args, **kwargs):
        return self._od.__setitem__(*args, **kwargs)

    def __getitem__(self, *args, **kwargs):
        return self._od.__getitem__(*args, **kwargs)

    def __delitem__(self, *args, **kwargs):
        return self._od.__delitem__(*args, **kwargs)

    def __eq__(self, other):
        if isinstance(other, Buckets):
            return self._od.__eq__(other._od)
        else:
            return self._od.__eq__(other)

    def copy(self, *args, **kwargs):
        new = Buckets()
        new._od = self._od.copy()
        return new
Beispiel #13
0
    def post_process(self, paths, dry_run=False, **options):
        if dry_run:
            return

        packager = Packager(storage=self)
        for _abs_path, rel_path in paths:
            files_to_process = OrderedDict()
            files_to_process[rel_path] = (self, rel_path)
            for package_name in packager.packages['css']:
                package = packager.package_for('css', package_name)
                output_file = package.output_filename

                if rel_path in package.paths:
                    if self.packing:
                        packager.pack_stylesheets(package)
                    files_to_process[output_file] = (self, output_file)
                    yield output_file, output_file, True
            for package_name in packager.packages['js']:
                package = packager.package_for('js', package_name)
                output_file = package.output_filename
                if rel_path in package.paths:
                    if self.packing:
                        packager.pack_javascripts(package)
                    files_to_process[output_file] = (self, output_file)
                    yield output_file, output_file, True

            super_class = super(PipelineMixin, self)

            if hasattr(super_class, 'post_process'):
                for name, hashed_name, processed in super_class.post_process(
                        files_to_process.copy(), dry_run, **options):
                    yield name, hashed_name, processed
Beispiel #14
0
def merge_data_for_people(people, models):
    """
        Collect data for a certain set of people from a list of model objects.
        Merge results from models that have the same name.
    """
    
    # All headers from the models
    all_headers = list(chain.from_iterable([m.analytics_headers() for m in models]))

    # Initialize a dict containing all people each with all headers,
    # and default values of '' for each header
    data = OrderedDict()
    headers = []
    for h in all_headers:
        if h['title'] not in data.keys():
            data[h['title']] = ''
            headers.append(h)
    persondata = dict((p, data.copy()) for p in people)
    persondata['_headers'] = headers

    for m in models:
        mdata = m.analytics_data_by_person()
        for p in people:
            if p in mdata:
                persondata[p].update(mdata[p])
    return persondata
class TestSerialization(unittest.TestCase):

    def setUp(self):
        self.serialized = OrderedDict()
        self.serialized["typeid"] = "malcolm:core/MethodMeta:1.0"
        self.takes = MapMeta()
        self.takes.set_elements(ElementMap({"in_attr": StringMeta("desc")}))
        self.serialized["takes"] = self.takes.to_dict()
        self.serialized["defaults"] = OrderedDict({"in_attr": "default"})
        self.serialized["description"] = "test_description"
        self.serialized["tags"] = []
        self.serialized["writeable"] = True
        self.serialized["label"] = ""
        self.serialized["returns"] = MapMeta().to_dict()

    def test_to_dict(self):
        m = MethodMeta("test_description")
        m.set_takes(self.takes)
        m.set_defaults(self.serialized["defaults"])
        self.assertEqual(m.to_dict(), self.serialized)

    def test_from_dict(self):
        m = MethodMeta.from_dict(self.serialized.copy())
        self.assertEqual(m.takes.to_dict(), self.takes.to_dict())
        self.assertEqual(m.defaults, self.serialized["defaults"])
        self.assertEqual(m.tags, [])
        self.assertEqual(m.writeable, True)
        self.assertEqual(m.label, "")
        self.assertEqual(m.returns.to_dict(), MapMeta().to_dict())
 def test_include_exclude_versions(self):
     source_versions = OrderedDict([('egg', '0.1'), ('Egg', '0.2')])
     self.assertEquals(self.checker.include_exclude_versions(
         source_versions), source_versions)
     results = source_versions.copy()
     results['Django'] = '0.0.0'
     self.assertEquals(
         self.checker.include_exclude_versions(
             source_versions, includes=['Django', 'egg']),
         results)
     source_versions['Django'] = '1.5.1'
     source_versions['pytz'] = '2013b'
     results = OrderedDict([('pytz', '2013b')])
     self.assertEquals(
         self.checker.include_exclude_versions(
             source_versions, excludes=['Django', 'egg']),
         results)
     self.assertEquals(
         self.checker.include_exclude_versions(
             source_versions,
             includes=['Django', 'egg'],
             excludes=['Django', 'egg']),
         results)
     results['zc.buildout'] = '0.0.0'
     self.assertEquals(
         self.checker.include_exclude_versions(
             source_versions,
             includes=['zc.buildout'],
             excludes=['Django', 'egg']),
         results)
    def _classes_in_config_sample(self):
        """
        Yields only classes with own traits, and their subclasses.

        Thus, produced sample config-file will contain all classes
        on which a trait-value may be overridden:

        - either on the class owning the trait,
        - or on its subclasses, even if those subclasses do not define
          any traits themselves.
        """
        cls_to_config = OrderedDict( (cls, bool(cls.class_own_traits(config=True)))
                              for cls
                              in self._classes_inc_parents())

        def is_any_parent_included(cls):
            return any(b in cls_to_config and cls_to_config[b] for b in cls.__bases__)

        ## Mark "empty" classes for inclusion if their parents own-traits,
        #  and loop until no more classes gets marked.
        #
        while True:
            to_incl_orig = cls_to_config.copy()
            cls_to_config = OrderedDict( (cls, inc_yes or is_any_parent_included(cls))
                                  for cls, inc_yes
                                  in cls_to_config.items())
            if cls_to_config == to_incl_orig:
                break
        for cl, inc_yes in cls_to_config.items():
            if inc_yes:
                yield cl
Beispiel #18
0
class APIError(exceptions.APIException):
    """
    Base class for JSON-API errors.

    Reference:
    http://jsonapi.org/format/#error-objects
    """

    def __init__(self, *args, **kwargs):

        self._data = OrderedDict([
            ('id',      kwargs.pop('id', None)),
            ('links',   kwargs.pop('links', None)),
            ('status',  kwargs.pop('status', self.status_code)),
            ('code',    kwargs.pop('code', None)),
            ('title',   kwargs.pop('title', None)),
            ('detail',  None),
            ('source',  kwargs.pop('source', None)),
            ('meta',    kwargs.pop('meta', None)),
        ])

        super(APIError, self).__init__(*args, **kwargs)

    @property
    def data(self):
        error = self._data.copy()
        for key, value in error.items():
            error[key] = getattr(self, key, None) if value is None else value

        # filter out empty values
        return OrderedDict((k, v) for k, v in error.items() if v)
Beispiel #19
0
def get_top_scorers():
    current_year = season()
    goals = Goal.objects.filter(match__competition_year__year=current_year).select_related('scorer').prefetch_related(
        'match__competition_year__competition')
    competition_years = CompetitionYear.objects.filter(year=current_year).select_related()
    competition_dct = OrderedDict()
    competitions = OrderedDict()
    for competition_year in competition_years:
        # if competition_year.year == current_year:
        competition_dct[competition_year.competition_id] = 0
        competitions[competition_year.competition_id] = competition_year.competition.name
    players = OrderedDict()
    for goal in goals:
        if goal.own_goal:
            continue
        competition_id = goal.match.competition_year.competition_id
        if goal.scorer not in players:
            players[goal.scorer] = {}
            players[goal.scorer]['competitions'] = competition_dct.copy()
            players[goal.scorer]['total'] = 0
        players[goal.scorer]['competitions'][competition_id] += 1
        players[goal.scorer]['total'] += 1
    players = OrderedDict(sorted(players.items(), key=lambda item: item[1]['total'], reverse=True))
    context = {
        'players': players,
        'competitions': competitions
    }
    return context
Beispiel #20
0
    def _compute_root_packages(self, pool, install_map, update_map):
        """ Look at the root packages in the given maps.

        Root packages are packages which are not dependencies of other
        packages.
        """
        packages = OrderedDict(install_map)
        packages.update(update_map)

        roots = packages.copy()

        for package_id, operation in packages.items():
            package = operation.package

            if package_id not in roots:
                continue

            for dependency in package.dependencies:
                candidates = pool.what_provides(
                    Requirement.from_legacy_requirement_string(dependency)
                )
                for candidate in candidates:
                    candidate_id = pool.package_id(candidate)
                    roots.pop(candidate_id, None)

        return roots
def test_mqtt_subscribes_topics_on_connect(hass):
    """Test subscription to topic on connect."""
    mqtt_client = yield from mock_mqtt_client(hass)

    subscribed_topics = OrderedDict()
    subscribed_topics['topic/test'] = 1
    subscribed_topics['home/sensor'] = 2

    wanted_topics = subscribed_topics.copy()
    wanted_topics['still/pending'] = 0

    hass.data['mqtt'].wanted_topics = wanted_topics
    hass.data['mqtt'].subscribed_topics = subscribed_topics
    hass.data['mqtt'].progress = {1: 'still/pending'}

    # Return values for subscribe calls (rc, mid)
    mqtt_client.subscribe.side_effect = ((0, 2), (0, 3))

    hass.add_job = mock.MagicMock()
    hass.data['mqtt']._mqtt_on_connect(None, None, 0, 0)

    yield from hass.async_block_till_done()

    assert not mqtt_client.disconnect.called

    expected = [(topic, qos) for topic, qos in wanted_topics.items()]

    assert [call[1][1:] for call in hass.add_job.mock_calls] == expected
    assert hass.data['mqtt'].progress == {}
Beispiel #22
0
    def gaylord(self, out, session):
        fields = [
            'First Name', 'Last Name', 'Guest Email Address for confirmation purposes', 'Special Requests',
            'Arrival', 'Departure', 'City', 'State', 'Zip', 'GUEST COUNTRY', 'Telephone',
            'Payment Type', 'Card #', 'Exp.',
            'BILLING ADDRESS', 'BILLING CITY', 'BILLING STATE', 'BILLING ZIP CODE', 'BILLING COUNTRY',
            'Additional Guest First Name-2', 'Additional Guest Last Name-2',
            'Additional Guest First Name-3', 'Additional Guest Last Name3',  # No, this is not a typo
            'Additional Guest First Name-4', 'Additional Guest Last Name-4',
            'Notes', 'Emails',
        ]

        blank = OrderedDict([(field, '') for field in fields])
        out.writerow(fields)
        for room in session.query(Room).order_by(Room.created).all():
            if room.assignments:
                row = blank.copy()
                row.update({
                    'Notes': room.notes,
                    'Arrival': room.check_in_date.strftime('%m/%d/%Y'),
                    'Departure': room.check_out_date.strftime('%m/%d/%Y'),
                    'Emails': ','.join(room.email),
                })
                for i, attendee in enumerate([ra.attendee for ra in room.assignments[0:4]]):
                    if i == 0:
                        prefix, suffix = '', ''
                        row.update({'Guest Email Address for confirmation purposes': attendee.email})
                    else:
                        prefix = 'Additional Guest'
                        suffix = '-{}'.format(i+1) if i != 2 else str(i+1)
                    row.update({
                        prefix + 'First Name' + suffix: attendee.legal_first_name,
                        prefix + 'Last Name' + suffix: attendee.legal_last_name
                    })
                out.writerow(list(row.values()))
Beispiel #23
0
    def hotel_email_info(self, out, session):
        fields = [
            'CheckIn Date', 'CheckOut Date', 'Number of Guests', 'Room Notes',
            'Guest1 First Name', 'Guest1 Last Name', 'Guest1 Legal Name',
            'Guest2 First Name', 'Guest2 Last Name', 'Guest2 Legal Name',
            'Guest3 First Name', 'Guest3 Last Name', 'Guest3 Legal Name',
            'Guest4 First Name', 'Guest4 Last Name', 'Guest4 Legal Name',
            'Guest5 First Name', 'Guest5 Last Name', 'Guest5 Legal Name',
            'Emails',
        ]

        blank = OrderedDict([(field, '') for field in fields])
        out.writerow(fields)
        for room in session.query(Room).order_by(Room.created).all():
            if room.assignments:
                row = blank.copy()
                row.update({
                    'Room Notes': room.notes,
                    'Number of Guests': min(4, len(room.assignments)),
                    'CheckIn Date': room.check_in_date.strftime('%m/%d/%Y'),
                    'CheckOut Date': room.check_out_date.strftime('%m/%d/%Y'),
                    'Emails': ','.join(room.email),
                })
                for i, attendee in enumerate([ra.attendee for ra in room.assignments[:4]]):
                    prefix = 'Guest{}'.format(i + 1)
                    row.update({
                        prefix + ' First Name': attendee.first_name,
                        prefix + ' Last Name': attendee.last_name,
                        prefix + ' Legal Name': attendee.legal_name,
                    })
                out.writerow(list(row.values()))
Beispiel #24
0
def findDensestsSubgraphParallell(edgeWeights, nodeDegrees, epsilon):
    # Create a sorted dictionary with ascending node degrees. NB! Does not sort itself later!
    nodeDegrees = OrderedDict(sorted(nodeDegrees.iteritems(), key=lambda (k,v): (v,k)))

    avgGraphDensity = computeAverageDensity(edgeWeights, nodeDegrees)
    avgGraphDensityBest = avgGraphDensity

    count = 0
    while count < 50:
        print('Iteration #{}'.format(count))
        nodeDegreesUnchanged = nodeDegrees.copy()

        for node in nodeDegrees.keys():
            degree = nodeDegreesUnchanged[node]

            if degree <= 2*(1+epsilon)*avgGraphDensity:
                del nodeDegrees[node]
                
                # Delete all edges connected to this node
                for edge in edgeWeights.keys():
                    node1, node2 = edge.split('-')
                    """If edge is a self edge, delete it and don't reduce the degree
                    of "the other" node, since it is yourself"""
                    if node1 == node2:
                        del edgeWeights[edge]
                    else:
                        if node == node1:
                            nodeDegrees[node2] -= edgeWeights[edge]
                            del edgeWeights[edge]
                        elif node == node2:
                            nodeDegrees[node1] -= edgeWeights[edge]
                            del edgeWeights[edge]

        # If no more edges left, we are finished
        if len(edgeWeights) == 0:
            break

        avgGraphDensity = computeAverageDensity(edgeWeights, nodeDegrees)
        
        if avgGraphDensity > avgGraphDensityBest:
            newEdgeWeights = edgeWeights.copy()
            newNodeDegrees = nodeDegrees.copy()
            avgGraphDensityBest = avgGraphDensity
        
        count += 1

    return newEdgeWeights, newNodeDegrees
Beispiel #25
0
class Basic(object):
    """ Basic object deals with Basic HTTP Authorization configuration file.
    It is passed the path to userdb file. """

    def __init__(self, userdb):
        self.userdb = userdb
        self.initial_users = OrderedDict()
        self.new_users = OrderedDict()

    def __enter__(self):
        with open(self.userdb, "r") as users:
            for i in users:
                user, password = i.split(":", 1)
                self.initial_users[user] = password
        self.new_users = self.initial_users.copy()
        return self

    def __exit__(self, type, value, traceback):
        if self.new_users == self.initial_users:
            return
        with open(self.userdb, "w") as userdb:
            for user in self.new_users:
                userdb.write("%s:%s" % (user, self.new_users[user]))

    def __contains__(self, user):
        return user in self.users

    @property
    def users(self):
        """ Returns users in a tuple """
        return self.new_users.keys()

    def add(self, user, password):
        """ Adds a user with password """
        if self.__contains__(user):
            raise UserExists
        self.new_users[user] = self._crypt_password(password) + "\n"

    def pop(self, user):
        """ Deletes a user """
        if not self.__contains__(user):
            raise UserNotExists
        self.new_users.pop(user)

    def change_password(self, user, password):
        """ Changes user password """
        if not self.__contains__(user):
            raise UserNotExists
        self.new_users[user] = self._crypt_password(password) + "\n"

    def _crypt_password(self, password):
        """ Crypts password """

        def salt():
            """ Generates some salt """
            symbols = ascii_letters + digits
            return choice(symbols) + choice(symbols)

        return crypt(password, salt())
Beispiel #26
0
def get_sched():
    SCHED = OrderedDict()
    SCHED['R'] = ['Semester Courses']
    SCHED['A'] = ['Adult Undergraduate Studies 7-Week Courses']
    SCHED['G'] = ['Graduate Education']
    #SCHED['T'] = ['Accelerated Certification for Teachers (ACT)']
    SCHED['P'] = ['Paralegal Program']
    return SCHED.copy()
Beispiel #27
0
def prepare_dht2_xlator (volfile, volname, mds_count, ds_count, brick_list, server, n):

    opt_dict = OrderedDict()
    sub_dict = OrderedDict()
    mds_bricks = ""
    ds_bricks = ""
    i = 0
    while i < ds_count:
        if i == ds_count -1:
            ds_bricks += volname + "-client-" + str(i)
        else:
            ds_bricks += volname + "-client-" + str(i) + ":"
        i += 1

    i = ds_count 
    while i < mds_count + ds_count:
        if i == mds_count + ds_count - 1:
            mds_bricks += volname + "-client-" + str(i)
        else:
            mds_bricks += volname + "-client-" + str(i) + ":"
        i += 1

    subvols = [volname + "-client-" + str(n)]
    i = 0
    while i < mds_count + ds_count:
        if i == n:
            i += 1
            continue
        subvols.append(volname + "-client-" + str(i))
        i += 1

    if server:
        sub_dict["type"] = "experimental/dht2s"
    else:
        sub_dict["type"] = "experimental/dht2c"
    opt_dict["lock-migration"] = "off"
    opt_dict["dht2-data-subvolumes"] =  ds_bricks
    opt_dict["dht2-metadata-subvolumes"] =  mds_bricks
    if server:
        opt_dict["dht2-server-local-subvol"] = volname + "-client-" + str(n)
    sub_dict["option"] = opt_dict.copy()
    sub_dict["subvolumes"] = subvols
    volfile[volname + "-dht"] = sub_dict.copy()

    del(sub_dict)
    del(opt_dict)
Beispiel #28
0
class TestConfigurationVisitor(ConfigurationVisitor):
    """
    A means to creates configurations for single test
    """
    def __init__(self, test, design_unit, enable_configuration, default_config):
        ConfigurationVisitor.__init__(self)
        self._test = test
        assert test.is_explicit
        self.design_unit = design_unit
        self._enable_configuration = enable_configuration
        self._configs = OrderedDict({default_config.name: default_config})

    @property
    def name(self):
        return self._test.name

    @property
    def test(self):
        return self._test

    def get_default_config(self):
        """
        Get the default configuration of this test case
        """
        self._check_enabled()
        return self._configs[DEFAULT_NAME]

    def _check_enabled(self):
        if not self._enable_configuration:
            raise RuntimeError("Individual test configuration is not possible with run_all_in_same_sim")

    def get_configuration_dicts(self):
        """
        Get all configurations of this test
        """
        return [self._configs]

    def _get_configurations_to_run(self):
        """
        Get all simulation runs for this test bench
        """
        configs = self._configs.copy()
        if len(configs) > 1:
            # Remove default configurations when there are more than one
            del configs[DEFAULT_NAME]
        return configs.values()

    def create_tests(self, simulator_if, elaborate_only, test_list=None):
        """
        Create all tests from this test case which may be several depending on the number of configurations
        """
        for config in self._get_configurations_to_run():
            test_list.add_test(
                IndependentSimTestCase(
                    test=self._test,
                    config=config,
                    simulator_if=simulator_if,
                    elaborate_only=elaborate_only))
Beispiel #29
0
    def _derive(cls, name, info, otherbases, recurse=False):
        # collect the 3 set of infos
        # info = OrderedDict(info)
        baseinfo = cls._getpairs().copy()
        obasesinfo = OrderedDict()
        for obase in otherbases:
            if isinstance(obase, (tuple, dict)):
                obasesinfo.update(obase)
            else:
                obasesinfo.update(obase._getpairs())

        # update the info of this class (base) with that from the other bases
        baseinfo.update(obasesinfo)

        # The info of the new class is a copy of the full base info
        # plus and update from parameter
        clsinfo = baseinfo.copy()
        clsinfo.update(info)

        # The new items to update/set are those from the otherbase plus the new
        info2add = obasesinfo.copy()
        info2add.update(info)

        clsmodule = sys.modules[cls.__module__]
        newclsname = str(cls.__name__ + '_' + name)  # str - Python 2/3 compat

        # This loop makes sure that if the name has already been defined, a new
        # unique name is found. A collision example is in the plotlines names
        # definitions of bt.indicators.MACD and bt.talib.MACD. Both end up
        # definining a MACD_pl_macd and this makes it impossible for the pickle
        # module to send results over a multiprocessing channel
        namecounter = 1
        while hasattr(clsmodule, newclsname):
            newclsname += str(namecounter)
            namecounter += 1

        newcls = type(newclsname, (cls,), {})
        setattr(clsmodule, newclsname, newcls)

        setattr(newcls, '_getpairsbase',
                classmethod(lambda cls: baseinfo.copy()))
        setattr(newcls, '_getpairs', classmethod(lambda cls: clsinfo.copy()))
        setattr(newcls, '_getrecurse', classmethod(lambda cls: recurse))

        for infoname, infoval in info2add.items():
            if recurse:
                recursecls = getattr(newcls, infoname, AutoInfoClass)
                infoval = recursecls._derive(name + '_' + infoname,
                                             infoval,
                                             [])

            setattr(newcls, infoname, infoval)

        return newcls
Beispiel #30
0
def fetch_metadata(metadata_model):
    """
    Fetch the metadata table, and store it in metadata_dict.
    """
    metadata = OrderedDict()
    metadata_query = metadata_model.objects.all()
    for table in metadata_query:
        for field in METADATA_FIELDS:
            metadata[field] = table.__dict__[field]
            # Add the metadata to metadata_dict
        metadata_dict[table.table_name] = metadata.copy()
Beispiel #31
0
class ProteinComplex(object):
    def __init__(self, T=300):
        self.T = T
        self.normalized_constant_energy = 0.0
        self.residue_variables = OrderedDict()
        self.interaction_energies = OrderedDict()
        self.normalized_interaction_energies = None

    def add_residue(self, residue_type, chain, location):
        """Adds a residue to the protein if needed.
           Automatically creates all instances needed before simplification."""
        residue_tuple = (residue_type, chain, location)
        if residue_tuple not in self.residue_variables:
            residue_data = titratable_residue_data.get(residue_type)
            base_name = '_'.join(residue_tuple) + '_'
            res_var = ResidueVariable('_'.join(residue_tuple))

            tautomers = residue_data["tautomers"]
            model_pka = residue_data["model_pka"]

            for tautomer in tautomers["deprotonated"]:
                instance = ResidueInstance(False, base_name + tautomer)
                instance.energy = 0
                res_var.instances[tautomer] = instance

            for tautomer in tautomers["protonated"]:
                instance = ResidueInstance(True, base_name + tautomer)
                instance.energy = -1.0 * math.log(10) * model_pka
                res_var.instances[tautomer] = instance

            if residue_type != "HIS":
                #Placeholder instances for later consolidation
                #HIS is split later so we don't add these as they are not needed.

                instance = ResidueInstance(True, base_name + "PROTONATED")
                res_var.instances["PROTONATED"] = instance

                instance = ResidueInstance(False, base_name + "DEPROTONATED")
                res_var.instances["DEPROTONATED"] = instance

            self.residue_variables[residue_tuple] = res_var

    def get_residue(self, name, chain, location):
        "Returns the residue for this specific name, chain, and location."
        return self.residue_variables.get((name, chain, location))

    def get_instance(self, residue_type, chain, location, state):
        "Returns the instance for this specific name, chain, and location and state name."
        res_var = self.residue_variables.get((residue_type, chain, location))
        if res_var is None:
            return None
        return res_var.get_instance(state)

    def drop_interaction_pairs(self, pair_list):
        "Drop each pair of interaction energies in the pair_list"
        for instance1_name, instance2_name in pair_list:
            del self.interaction_energies[instance1_name, instance2_name]
            del self.interaction_energies[instance2_name, instance1_name]

    def get_interaction_combinations(self, pair1, pair2):
        "Get a list of all pairwise combinations from lists pair1 and pair2."
        product_list = [(x, y) for x, y in product(pair1, pair2)]
        return product_list

    def add_interaction_energy_pair(self,
                                    instance1,
                                    instance2,
                                    energy,
                                    normalized=False):
        "Insert interaction energy pair."
        ie = self.normalized_interaction_energies if normalized else self.interaction_energies
        ie[instance1, instance2] = energy
        ie[instance2, instance1] = energy

    def simplify(self):
        """Simplify the instances into protonated and deprotonated.
           Also divides HIS into HID and HIE."""
        self.update_special_instance_energies()
        self.consolidate()
        self.divide_his()

    def update_special_instance_energies(self):
        """After we have loaded the backgroind and desolvation files we need to
        update the consolidated instances to use the minimum energy of the
        instances they are meant to replace."""
        for key, residue in self.residue_variables.items():
            name = key[0]
            if name == 'HIS':
                continue
            prot_consolidated = residue.instances["PROTONATED"]
            deprot_consolidated = residue.instances["DEPROTONATED"]
            protonated, deprotonated = residue.get_prot_and_deprot_instances()
            #Update consolidated instance energy
            prot_consolidated.energy = min(x.energy for x in protonated)
            deprot_consolidated.energy = min(x.energy for x in deprotonated)

    def consolidate(self):
        """Each residue has multiple protonated and deprotonated states. Here
           we consolidate those into two states for each residue, PROT and DEPROT.
           We take minimums of energies between states in each class. For example,
           assume we have two amino acids, A and B, where A has protonated states 1, 2, 3
           and deprotonated state 4, and B has protonated states 1, 2, and deprotonated
           state 3. Then
           E(A_PROT, B_PROT) = min{E(A1,B1), E(A1,B2), E(A2,B1), E(A2,B2), E(A3,B1), E(A3,B2)},
           E(A_PROT, B_DEPROT) = min{E(A1,B3), E(A2,B3), E(A3,B3)},
           E(A_DEPROT, B_PROT) = min{E(A4,B1), E(A4,B2)}, and
           E(A_DEPROT, B_DEPROT) = E(A4,B3).
           We do not deal with HIS here, it is kept in its 3 states for now.

           After this is finished all unused interaction energies will be removed from
           self.interaction_energies."""

        handled_interaction_pairs = set()

        #See https://docs.python.org/2/library/itertools.html#itertools.combinations
        for v, w in combinations(iter(self.residue_variables.items()), 2):
            v_key, v_residue = v
            w_key, w_residue = w
            v_name = v_key[0]
            w_name = w_key[0]

            #Skip HIS for now
            if v_name == 'HIS' or w_name == 'HIS':
                continue

            v_protinated, v_unprotonated = v_residue.get_prot_and_deprot_instances(
            )
            v_prot_consolidated = v_residue.instances["PROTONATED"]
            v_deprot_consolidated = v_residue.instances["DEPROTONATED"]

            w_protinated, w_unprotonated = w_residue.get_prot_and_deprot_instances(
            )
            w_prot_consolidated = w_residue.instances["PROTONATED"]
            w_deprot_consolidated = w_residue.instances["DEPROTONATED"]

            #For every pairing of v and w (protonated and unprotonated) find the
            # minimum interaction energy and update the interaction map accordingly.
            v_stuff = ((v_protinated, v_prot_consolidated),
                       (v_unprotonated, v_deprot_consolidated))
            w_stuff = ((w_protinated, w_prot_consolidated),
                       (w_unprotonated, w_deprot_consolidated))

            for v_product, w_product in product(v_stuff, w_stuff):
                v_instances, v_consolidated = v_product
                w_instances, w_consolidated = w_product

                energies = []
                #Find all of the pairs
                for v_instance, w_instance in product(v_instances,
                                                      w_instances):
                    energy = self.interaction_energies[v_instance, w_instance]
                    energies.append(energy)
                    #Mark for deletion.
                    handled_interaction_pairs.add((v_instance, w_instance))

                min_energy = min(energies)

                self.add_interaction_energy_pair(v_consolidated,
                                                 w_consolidated, min_energy)

        #Now handle HIS.
        #See https://docs.python.org/2/library/itertools.html#itertools.permutations
        for v, w in permutations(iter(self.residue_variables.items()), 2):
            his_key, his_residue = v
            other_key, other_residue = w
            his_name = his_key[0]
            other_name = other_key[0]

            #We only care about his this pass
            if his_name != 'HIS':
                continue
            #HIS - HIS is already correct for what this pass does.
            if other_name == 'HIS':
                continue

            other_protinated, other_unprotonated = other_residue.get_prot_and_deprot_instances(
            )
            other_prot_consolidated = other_residue.instances["PROTONATED"]
            other_deprot_consolidated = other_residue.instances["DEPROTONATED"]

            his_protinated, his_unprotonated = his_residue.get_prot_and_deprot_instances(
            )
            his_stuff = his_protinated + his_unprotonated
            other_stuff = ((other_protinated, other_prot_consolidated),
                           (other_unprotonated, other_deprot_consolidated))

            #For every pairing of a HIS instance and another non HIS residue find the
            # minimum interaction energy and update the interaction map accordingly.
            #See https://docs.python.org/2/library/itertools.html#itertools.product
            for his_instance, other_product in product(his_stuff, other_stuff):
                other_instances, other_consolidated = other_product

                energies = []

                for other_instance in other_instances:
                    energy = self.interaction_energies[his_instance,
                                                       other_instance]
                    energies.append(energy)
                    handled_interaction_pairs.add(
                        (his_instance, other_instance))

                min_energy = min(energies)

                self.add_interaction_energy_pair(his_instance,
                                                 other_consolidated,
                                                 min_energy)

        #Clean out unused interaction energies
        self.drop_interaction_pairs(handled_interaction_pairs)

        for key, residue in self.residue_variables.items():
            name = key[0]
            if name == 'HIS':
                continue

            residue.instances = OrderedDict(
                (k, v) for k, v in residue.instances.items()
                if "PROTONATED" in k)

    def divide_his(self):
        """Here we divide HIS into two residues - HID, HIE - each with half the pKa value. We
           have to set interaction energies between HIS and other residues and for HIS-HIS
           interactions. Do this based on the values given in the paper"""

        to_drop_his = set()
        new_to_old_his = {}
        handled_interaction_pairs = set()

        items = list(self.residue_variables.items())

        #Find all HIS and split them.
        for key, residue in items:
            name = key[0]
            if name != 'HIS':
                continue

            old_instance_1 = residue.get_instance('1')
            old_instance_2 = residue.get_instance('2')
            old_instance_12 = residue.get_instance('1+2')

            base_name = '_' + '_'.join(key[-2:]) + '_'
            to_drop_his.add(key)

            hid = ResidueVariable("HId" + base_name)
            name = "HId" + base_name + "PROTONATED"
            hid_prot = ResidueInstance(True,
                                       name,
                                       energy=old_instance_1.energy)
            name = "HId" + base_name + "DEPROTONATED"
            hid_deprot = ResidueInstance(False, name)

            hid.instances["PROTONATED"] = hid_prot
            hid.instances["DEPROTONATED"] = hid_deprot

            res_tuple = ("HId", ) + key[-2:]
            self.residue_variables[res_tuple] = hid

            new_to_old_his[hid] = residue

            hie = ResidueVariable("HIe" + base_name)
            name = "HIe" + base_name + "PROTONATED"
            hie_prot = ResidueInstance(True,
                                       name,
                                       energy=old_instance_2.energy)
            name = "HIe" + base_name + "DEPROTONATED"
            hie_deprot = ResidueInstance(False, name)

            hie.instances["PROTONATED"] = hie_prot
            hie.instances["DEPROTONATED"] = hie_deprot

            res_tuple = ("HIe", ) + key[-2:]
            self.residue_variables[res_tuple] = hie

            #Keep a mapping to the original residue object so we can look up
            # interaction energies in the HIS/HIS interactions loop below.
            new_to_old_his[hie] = residue

            #Create interaction energies between newly created residues.
            energy = old_instance_12.energy - old_instance_1.energy - old_instance_2.energy
            self.add_interaction_energy_pair(hid_prot, hie_prot, energy)
            self.add_interaction_energy_pair(hid_prot, hie_deprot, 0.0)
            self.add_interaction_energy_pair(hid_deprot, hie_prot, 0.0)
            self.add_interaction_energy_pair(hid_deprot, hie_deprot,
                                             sys.float_info.max)

        #Delete residue variables from main map.
        for key in to_drop_his:
            del self.residue_variables[key]

        #This loop is order dependent for HIS/HIS interactions. That is why some of the code paths that
        #appear as though they should be duplicate or symmetrical are not
        #and why we compare chain locations.
        # (In case you were wondering why HIe <-> HId is not the same as
        #  HId <-> HIe.)

        #See https://docs.python.org/2/library/itertools.html#itertools.product
        for v, w in product(iter(self.residue_variables.items()), repeat=2):
            his_key, his_residue = v
            other_key, other_residue = w
            his_name, his_chain, his_location = his_key
            other_name, other_chain, other_location = other_key

            if his_name not in ('HId', 'HIe'):
                continue

            his_location = int(his_location)
            other_location = int(other_location)

            his_prot = his_residue.instances["PROTONATED"]
            his_deprot = his_residue.instances["DEPROTONATED"]

            old_his = new_to_old_his[his_residue]
            old_his_instance_1 = old_his.get_instance('1')
            old_his_instance_2 = old_his.get_instance('2')
            old_his_instance_12 = old_his.get_instance('1+2')

            other_prot = other_residue.instances["PROTONATED"]
            other_deprot = other_residue.instances["DEPROTONATED"]

            #Handle create interaction with HId
            if other_name == 'HId':
                # HIS/HIS is order dependent.
                if his_chain > other_chain or his_location >= other_location:
                    continue

                old_other = new_to_old_his[other_residue]
                old_other_instance_1 = old_other.get_instance('1')
                old_other_instance_2 = old_other.get_instance('2')
                old_other_instance_12 = old_other.get_instance('1+2')

                if his_name == 'HIe':
                    energy = self.interaction_energies[old_his_instance_12,
                                                       old_other_instance_12]

                    self.add_interaction_energy_pair(his_prot, other_prot,
                                                     energy)

                    self.add_interaction_energy_pair(his_prot, other_deprot,
                                                     0.0)

                    self.add_interaction_energy_pair(his_deprot, other_prot,
                                                     0.0)

                    energy = (
                        self.interaction_energies[old_his_instance_1,
                                                  old_other_instance_2] -
                        self.interaction_energies[old_his_instance_1,
                                                  old_other_instance_12] -
                        self.interaction_energies[old_his_instance_12,
                                                  old_other_instance_2])

                    self.add_interaction_energy_pair(his_deprot, other_deprot,
                                                     energy)

                elif his_name == 'HId':
                    self.add_interaction_energy_pair(his_prot, other_prot, 0.0)

                    energy = self.interaction_energies[old_his_instance_12,
                                                       old_other_instance_2]
                    self.add_interaction_energy_pair(his_prot, other_deprot,
                                                     energy)

                    energy = (
                        self.interaction_energies[old_his_instance_2,
                                                  old_other_instance_12] -
                        self.interaction_energies[old_his_instance_12,
                                                  old_other_instance_12])
                    self.add_interaction_energy_pair(his_deprot, other_prot,
                                                     energy)

                    energy = self.interaction_energies[old_his_instance_2,
                                                       old_other_instance_2]
                    self.add_interaction_energy_pair(his_deprot, other_deprot,
                                                     energy)

                combinations = self.get_interaction_combinations(
                    (old_his_instance_1, old_his_instance_2,
                     old_his_instance_12),
                    (old_other_instance_1, old_other_instance_2,
                     old_other_instance_12))

                handled_interaction_pairs.update(combinations)

            #Handle create interaction with HIe
            elif other_name == 'HIe':
                # HIS/HIS is order dependent.
                if his_chain > other_chain or his_location >= other_location:
                    continue

                old_other = new_to_old_his[other_residue]
                old_other_instance_1 = old_other.get_instance('1')
                old_other_instance_2 = old_other.get_instance('2')
                old_other_instance_12 = old_other.get_instance('1+2')

                if his_name == 'HIe':
                    self.add_interaction_energy_pair(his_prot, other_prot, 0.0)

                    energy = (self.interaction_energies[old_his_instance_12,
                                                        old_other_instance_1] -
                              self.interaction_energies[old_his_instance_12,
                                                        old_other_instance_12])

                    self.add_interaction_energy_pair(his_prot, other_deprot,
                                                     energy)

                    energy = self.interaction_energies[old_his_instance_1,
                                                       old_other_instance_12]
                    self.add_interaction_energy_pair(his_deprot, other_prot,
                                                     energy)

                    energy = self.interaction_energies[old_his_instance_1,
                                                       old_other_instance_1]
                    self.add_interaction_energy_pair(his_deprot, other_deprot,
                                                     energy)

                elif his_name == 'HId':
                    self.add_interaction_energy_pair(his_prot, other_prot, 0.0)
                    self.add_interaction_energy_pair(his_prot, other_deprot,
                                                     0.0)
                    self.add_interaction_energy_pair(his_deprot, other_prot,
                                                     0.0)

                    energy = (
                        self.interaction_energies[old_his_instance_2,
                                                  old_other_instance_1] +
                        self.interaction_energies[old_his_instance_12,
                                                  old_other_instance_12] -
                        self.interaction_energies[old_his_instance_2,
                                                  old_other_instance_12] -
                        self.interaction_energies[old_his_instance_12,
                                                  old_other_instance_1])
                    self.add_interaction_energy_pair(his_deprot, other_deprot,
                                                     energy)

                combinations = self.get_interaction_combinations(
                    (old_his_instance_1, old_his_instance_2,
                     old_his_instance_12),
                    (old_other_instance_1, old_other_instance_2,
                     old_other_instance_12))

                handled_interaction_pairs.update(combinations)

            #Handle create interaction with non-HIS
            else:
                if his_name == 'HIe':
                    energy = (self.interaction_energies[old_his_instance_12,
                                                        other_prot] -
                              self.interaction_energies[old_his_instance_1,
                                                        other_prot])
                    self.add_interaction_energy_pair(his_prot, other_prot,
                                                     energy)

                    energy = self.interaction_energies[old_his_instance_2,
                                                       other_deprot]
                    self.add_interaction_energy_pair(his_prot, other_deprot,
                                                     energy)

                    self.add_interaction_energy_pair(his_deprot, other_prot,
                                                     0.0)

                    energy = (self.interaction_energies[old_his_instance_1,
                                                        other_deprot] +
                              self.interaction_energies[old_his_instance_2,
                                                        other_deprot] -
                              self.interaction_energies[old_his_instance_12,
                                                        other_deprot])
                    self.add_interaction_energy_pair(his_deprot, other_deprot,
                                                     energy)

                elif his_name == 'HId':
                    energy = self.interaction_energies[old_his_instance_1,
                                                       other_prot]
                    self.add_interaction_energy_pair(his_prot, other_prot,
                                                     energy)

                    energy = (self.interaction_energies[old_his_instance_12,
                                                        other_deprot] -
                              self.interaction_energies[old_his_instance_2,
                                                        other_deprot])
                    self.add_interaction_energy_pair(his_prot, other_deprot,
                                                     energy)

                    energy = (self.interaction_energies[old_his_instance_1,
                                                        other_prot] +
                              self.interaction_energies[old_his_instance_2,
                                                        other_prot] -
                              self.interaction_energies[old_his_instance_12,
                                                        other_prot])
                    self.add_interaction_energy_pair(his_deprot, other_prot,
                                                     energy)

                    self.add_interaction_energy_pair(his_deprot, other_deprot,
                                                     0.0)

                residue_combinations = self.get_interaction_combinations(
                    (old_his_instance_1, old_his_instance_2,
                     old_his_instance_12), (other_prot, other_deprot))
                name_combinations = [(tup[0], tup[1])
                                     for tup in residue_combinations]
                handled_interaction_pairs.update(name_combinations)

        #Clean out unused interaction energies
        self.drop_interaction_pairs(handled_interaction_pairs)

    def evaluate_energy(self, labeling, normal_form=True, pH=0.0):
        """Get the total energy for the residue in the state specified by labeling.

           normal_form and pH are used for testing."""
        energy = 0.0
        if not normal_form:
            ph_multiplier = 0.0
            for instance in list(labeling.values()):
                if instance.protonated:
                    ph_multiplier += 1.0

            #See https://docs.python.org/2/library/itertools.html#itertools.combinations
            for v, w in combinations(iter(self.residue_variables.items()), 2):
                v_key, v_residue = v
                w_key, w_residue = w
                is_same_his = (v_key[1:] == w_key[1:]
                               and v_key[0] in ("HId", "HIe")
                               and w_key[0] in ("HId", "HIe"))

                if is_same_his:
                    if labeling[v_residue].protonated and labeling[
                            w_residue].protonated:
                        ph_multiplier -= 2.0

        for v in self.residue_variables.values():
            v_instance = labeling[v]
            energy += v_instance.energyNF if normal_form else v_instance.energy

        ie = self.normalized_interaction_energies if normal_form else self.interaction_energies
        for v, w in combinations(iter(self.residue_variables.values()), 2):
            v_instance = labeling[v]
            w_instance = labeling[w]
            energy += ie[v_instance, w_instance]

        if normal_form:
            energy += self.normalized_constant_energy
        else:
            energy += pH * ph_multiplier

        return energy

    def instance_interaction_energy(self, instance, labeling,
                                    interaction_energy):
        return sum(
            interaction_energy.get((instance, labeling[x]), 0)
            for x in self.residue_variables.values())

    def evaluate_energy_diff(self, residue, labeling, normal_form=False):
        """Returns the total energy difference between the deprontated and protonated
           states for the supplied residue. (protonated total energy - deprotonated total energy)"""

        ie = self.normalized_interaction_energies if normal_form else self.interaction_energies

        prot_instance = residue.instances["PROTONATED"]
        prot_energy = 0
        for x in self.residue_variables.values():
            prot_energy += ie.get((prot_instance, labeling[x]), 0.0)
        prot_energy += prot_instance.energyNF if normal_form else prot_instance.energy

        deprot_instance = residue.instances["DEPROTONATED"]
        deprot_energy = sum(
            ie.get((deprot_instance, labeling[x]), 0.0)
            for x in self.residue_variables.values())
        deprot_energy += deprot_instance.energyNF if normal_form else deprot_instance.energy

        return prot_energy - deprot_energy

    def evaluate_energy_diff_his(self,
                                 hie_residue,
                                 hid_residue,
                                 labeling,
                                 normal_form=False):
        """Returns the total energy differences between
             1. HSE total energy and HSP total energy (HSP total energy - HSE total energy)
             2. HSD total energy and HSP total energy (HSP total energy - HSD total energy)
            in a tuple."""

        # With both HIE and HID in protonated states:
        # HSP energy adds:
        #    unary energy of all not-this-HIS residues in current state
        #    unary energy of HSE_protonated
        #    unary energy of HSD_protonated
        #    binary energy of all not-this-HIS residues in current state
        #    binary energy of all not-this-HIS (current state) with HSE_protonated
        #    binary energy of all not-this-HIS (current state) with HSD_protonated
        #
        # HSE energy adds:
        #    unary energy of all not-this-HIS residues in current state
        #    unary energy of HSE_protonated
        #    unary energy of HSD_deprotonated
        #    binary energy of all not-this-HIS residues in current state
        #    binary energy of all not-this-HIS (current state) with HSE_protonated
        #    binary energy of all not-this-HIS (current state) with HSD_deprotonated
        #
        # HSD energy adds:
        #    unary energy of all not-this-HIS residues in current state
        #    unary energy of HSE_deprotonated
        #    unary energy of HSD_protonated
        #    binary energy of all not-this-HIS residues in current state
        #    binary energy of all not-this-HIS (current state) with HSE_deprotonated
        #    binary energy of all not-this-HIS (current state) with HSD_protonated
        #
        # HSP-HSE has
        #    unary energy of HSD_protonated (EHd1) minus
        #        unary energy of HSD_deprotonated (EHd0)
        #    binary energy of all not-this-HIS (current state) with HSD_protonated (sum_ErHd_a1) minus
        #        binary energy of all not-this-HIS (current state) with HSD_deprotonated (sum_ErHd_a0)
        #
        # HSP-HSD has
        #    unary energy of HSE_protonated (EHe1) minus
        #        unary energy of HSE_deprotonated (EHe0)
        #    binary energy of all not-this-HIS (current state) with HSE_protonated (sum_ErHe_a1) minus
        #        binary energy of all not-this-HIS (current state) with HSE_deprotonated (sum_ErHe_a0)

        labeling_copy = labeling.copy()
        ie = self.normalized_interaction_energies if normal_form else self.interaction_energies

        hie_prot_instance = hie_residue.instances["PROTONATED"]
        hie_deprot_instance = hie_residue.instances["DEPROTONATED"]
        hid_prot_instance = hid_residue.instances["PROTONATED"]
        hid_deprot_instance = hid_residue.instances["DEPROTONATED"]

        labeling_copy[hid_residue] = hid_prot_instance
        labeling_copy[hie_residue] = hie_prot_instance

        if normal_form:
            EHd1 = hid_prot_instance.energyNF
            EHd0 = hid_deprot_instance.energyNF
        else:
            EHd1 = hid_prot_instance.energy
            EHd0 = hid_deprot_instance.energy

        sum_ErHd_a1 = sum(
            ie.get((labeling_copy[x], hid_prot_instance), 0)
            for x in self.residue_variables.values())
        sum_ErHd_a0 = sum(
            ie.get((labeling_copy[x], hid_deprot_instance), 0)
            for x in self.residue_variables.values())

        if normal_form:
            EHe1 = hie_prot_instance.energyNF
            EHe0 = hie_deprot_instance.energyNF
        else:
            EHe1 = hie_prot_instance.energy
            EHe0 = hie_deprot_instance.energy

        sum_ErHe_a1 = sum(
            ie.get((labeling_copy[x], hie_prot_instance), 0)
            for x in self.residue_variables.values())
        sum_ErHe_a0 = sum(
            ie.get((labeling_copy[x], hie_deprot_instance), 0)
            for x in self.residue_variables.values())

        hsp_hse = EHd1 - EHd0 + sum_ErHd_a1 - sum_ErHd_a0
        hsp_hsd = EHe1 - EHe0 + sum_ErHe_a1 - sum_ErHe_a0

        return hsp_hse, hsp_hsd

    def normalize(self, pH):
        """Finds and stores the normal form of all instance and interaction energies at the supplied pH value.
           Instance energies are saved in instance.energyNF and interaction energies
           are stored in self.interaction_energies"""
        self.normalized_interaction_energies = self.interaction_energies.copy()
        self.normalized_constant_energy = 0.0

        #Add the pH to the instances.
        for residue in self.residue_variables.values():
            residue.instances["DEPROTONATED"].energyNF = residue.instances[
                "DEPROTONATED"].energy
            # TODO - I'm not sure why this is +pH....
            residue.instances["PROTONATED"].energyNF = residue.instances[
                "PROTONATED"].energy + math.log(10) * pH

        rv = self.residue_variables
        keys = list(rv.keys())

        #Add the pH values to the HIS self interactions.
        for v_key, w_key in combinations(keys, 2):
            v_residue = rv[v_key]
            w_residue = rv[w_key]

            is_same_his = (v_key[1:] == w_key[1:]
                           and v_key[0] in ("HId", "HIe")
                           and w_key[0] in ("HId", "HIe"))

            if is_same_his:
                v_prot = v_residue.instances["PROTONATED"]
                w_prot = w_residue.instances["PROTONATED"]
                self.normalized_interaction_energies[v_prot,
                                                     w_prot] -= (2.0 *
                                                                 math.log(10) *
                                                                 pH)
                self.normalized_interaction_energies[w_prot,
                                                     v_prot] -= (2.0 *
                                                                 math.log(10) *
                                                                 pH)

        #Normalize the interaction energies.
        for v_key, w_key in permutations(keys, 2):
            v_residue = rv[v_key]
            w_residue = rv[w_key]

            for v_instance in v_residue.instances.values():
                w_instances = list(w_residue.instances.values())
                min_energy = min(
                    self.normalized_interaction_energies[v_instance,
                                                         w_instance]
                    for w_instance in w_instances)
                v_instance.energyNF += min_energy

                if min_energy != sys.float_info.max:
                    for w_instance in w_instances:
                        self.normalized_interaction_energies[
                            v_instance, w_instance] -= min_energy
                        self.normalized_interaction_energies[
                            w_instance, v_instance] -= min_energy

        #Normalize the instance energies.
        for residue in self.residue_variables.values():
            min_energy = min(instance.energyNF
                             for instance in residue.instances.values())
            self.normalized_constant_energy += min_energy
            if min_energy != sys.float_info.max:
                for instance in residue.instances.values():
                    instance.energyNF -= min_energy

    def energy_at_pH(self, pH):
        """Create a representation of the protein energy at the specified pH
        Used primarily for testing."""
        self.interaction_energies_for_ph = self.interaction_energies.copy()

        for residue in self.residue_variables.values():
            residue.instances[
                "DEPROTONATED"].energy_with_ph = residue.instances[
                    "DEPROTONATED"].energy
            residue.instances["PROTONATED"].energy_with_ph = residue.instances[
                "PROTONATED"].energy + math.log(10) * pH

        for v, w in combinations(iter(self.residue_variables.items()), 2):
            v_key, v_residue = v
            w_key, w_residue = w
            is_same_his = (v_key[1:] == w_key[1:]
                           and v_key[0] in ("HId", "HIe")
                           and w_key[0] in ("HId", "HIe"))

            if is_same_his:
                v_prot = v_residue.instances["PROTONATED"]
                w_prot = w_residue.instances["PROTONATED"]
                self.interaction_energies_for_ph[v_prot,
                                                 w_prot] -= (2.0 *
                                                             math.log(10) * pH)
                self.interaction_energies_for_ph[w_prot,
                                                 v_prot] -= (2.0 *
                                                             math.log(10) * pH)
class IBMQProvider(BaseProvider):
    """Provider for remote IBMQ backends with admin features.

    This class is the entry point for handling backends from IBMQ, allowing
    using different accounts.
    """
    def __init__(self):
        super().__init__()

        # dict[credentials_unique_id: IBMQSingleProvider]
        # This attribute stores a reference to the different accounts. The
        # keys are tuples (hub, group, project), as the convention is that
        # that tuple uniquely identifies a set of credentials.
        self._accounts = OrderedDict()

    def backends(self, name=None, filters=None, **kwargs):
        """Return all backends accessible via IBMQ provider, subject to optional filtering.

        Args:
            name (str): backend name to filter by
            filters (callable): more complex filters, such as lambda functions
                e.g. IBMQ.backends(filters=lambda b: b.configuration['n_qubits'] > 5)
            kwargs: simple filters specifying a true/false criteria in the
                backend configuration or backend status or provider credentials
                e.g. IBMQ.backends(n_qubits=5, operational=True, hub='internal')

        Returns:
            list[IBMQBackend]: list of backends available that match the filter

        Raises:
            IBMQAccountError: if no account matched the filter.
        """
        # pylint: disable=arguments-differ

        # Special handling of the credentials filters: match and prune from kwargs
        credentials_filter = {}
        for key in [
                'token', 'url', 'hub', 'group', 'project', 'proxies', 'verify'
        ]:
            if key in kwargs:
                credentials_filter[key] = kwargs.pop(key)
        providers = [
            provider for provider in self._accounts.values()
            if self._credentials_match_filter(provider.credentials,
                                              credentials_filter)
        ]

        # Special handling of the `name` parameter, to support alias resolution.
        if name:
            aliases = self.aliased_backend_names()
            aliases.update(self.deprecated_backend_names())
            name = aliases.get(name, name)

        # Aggregate the list of filtered backends.
        backends = []
        for provider in providers:
            backends = backends + provider.backends(
                name=name, filters=filters, **kwargs)

        return backends

    @staticmethod
    def deprecated_backend_names():
        """Returns deprecated backend names."""
        return {
            'ibmqx_qasm_simulator': 'ibmq_qasm_simulator',
            'ibmqx_hpc_qasm_simulator': 'ibmq_qasm_simulator',
            'real': 'ibmqx1'
        }

    @staticmethod
    def aliased_backend_names():
        """Returns aliased backend names."""
        return {
            'ibmq_5_yorktown': 'ibmqx2',
            'ibmq_5_tenerife': 'ibmqx4',
            'ibmq_16_rueschlikon': 'ibmqx5',
            'ibmq_20_austin': 'QS1_1'
        }

    def enable_account(self, token, url=QE_URL, **kwargs):
        """Authenticate a new IBMQ account and add for use during this session.

        Login into Quantum Experience or IBMQ using the provided credentials,
        adding the account to the current session. The account is not stored
        in disk.

        Args:
            token (str): Quantum Experience or IBM Q API token.
            url (str): URL for Quantum Experience or IBM Q (for IBM Q,
                including the hub, group and project in the URL).
            **kwargs (dict):
                * proxies (dict): Proxy configuration for the API.
                * verify (bool): If False, ignores SSL certificates errors
        """
        credentials = Credentials(token, url, **kwargs)

        self._append_account(credentials)

    def save_account(self, token, url=QE_URL, overwrite=False, **kwargs):
        """Save the account to disk for future use.

        Login into Quantum Experience or IBMQ using the provided credentials,
        adding the account to the current session. The account is stored in
        disk for future use.

        Args:
            token (str): Quantum Experience or IBM Q API token.
            url (str): URL for Quantum Experience or IBM Q (for IBM Q,
                including the hub, group and project in the URL).
            overwrite (bool): overwrite existing credentials.
            **kwargs (dict):
                * proxies (dict): Proxy configuration for the API.
                * verify (bool): If False, ignores SSL certificates errors
        """
        credentials = Credentials(token, url, **kwargs)
        store_credentials(credentials, overwrite=overwrite)

    def active_accounts(self):
        """List all accounts currently in the session.

        Returns:
            list[dict]: a list with information about the accounts currently
                in the session.
        """
        information = []
        for provider in self._accounts.values():
            information.append({
                'token': provider.credentials.token,
                'url': provider.credentials.url,
            })

        return information

    def stored_accounts(self):
        """List all accounts stored to disk.

        Returns:
            list[dict]: a list with information about the accounts stored
                on disk.
        """
        information = []
        stored_creds = read_credentials_from_qiskitrc()
        for creds in stored_creds:
            information.append({
                'token': stored_creds[creds].token,
                'url': stored_creds[creds].url
            })

        return information

    def load_accounts(self, **kwargs):
        """Load IBMQ accounts found in the system into current session,
        subject to optional filtering.

        Automatically load the accounts found in the system. This method
        looks for credentials in the following locations, in order, and
        returns as soon as credentials are found:

        1. in the `Qconfig.py` file in the current working directory.
        2. in the environment variables.
        3. in the `qiskitrc` configuration file

        Raises:
            IBMQAccountError: if no credentials are found.
        """
        for credentials in discover_credentials().values():
            if self._credentials_match_filter(credentials, kwargs):
                self._append_account(credentials)

        if not self._accounts:
            raise IBMQAccountError('No IBMQ credentials found on disk.')

    def disable_accounts(self, **kwargs):
        """Disable accounts in the current session, subject to optional filtering.

        The filter kwargs can be `token`, `url`, `hub`, `group`, `project`.
        If no filter is passed, all accounts in the current session will be disabled.

        Raises:
            IBMQAccountError: if no account matched the filter.
        """
        disabled = False

        # Try to remove from session.
        current_creds = self._accounts.copy()
        for creds in current_creds:
            credentials = Credentials(current_creds[creds].credentials.token,
                                      current_creds[creds].credentials.url)
            if self._credentials_match_filter(credentials, kwargs):
                del self._accounts[credentials.unique_id()]
                disabled = True

        if not disabled:
            raise IBMQAccountError(
                'No matching account to disable in current session.')

    def delete_accounts(self, **kwargs):
        """Delete saved accounts from disk, subject to optional filtering.

        The filter kwargs can be `token`, `url`, `hub`, `group`, `project`.
        If no filter is passed, all accounts will be deleted from disk.

        Raises:
            IBMQAccountError: if no account matched the filter.
        """
        deleted = False

        # Try to delete from disk.
        stored_creds = read_credentials_from_qiskitrc()
        for creds in stored_creds:
            credentials = Credentials(stored_creds[creds].token,
                                      stored_creds[creds].url)
            if self._credentials_match_filter(credentials, kwargs):
                remove_credentials(credentials)
                deleted = True

        if not deleted:
            raise IBMQAccountError('No matching account to delete from disk.')

    def _append_account(self, credentials):
        """Append an account with the specified credentials to the session.

        Args:
            credentials (Credentials): set of credentials.

        Returns:
            IBMQSingleProvider: new single-account provider.
        """
        # Check if duplicated credentials are already in use. By convention,
        # we assume (hub, group, project) is always unique.
        if credentials.unique_id() in self._accounts.keys():
            warnings.warn('Credentials are already in use.')

        single_provider = IBMQSingleProvider(credentials, self)
        self._accounts[credentials.unique_id()] = single_provider

        return single_provider

    def _credentials_match_filter(self, credentials, filter_dict):
        """Return True if the credentials match a filter.

        These filters apply on properties of a Credentials object:
        token, url, hub, group, project, proxies, verify
        Any other filter has no effect.

        Args:
            credentials (Credentials): IBMQ credentials object
            filter_dict (dict): dictionary of filter conditions

        Returns:
            bool: True if the credentials meet all the filter conditions
        """
        return all(
            getattr(credentials, key_, None) == value_
            for key_, value_ in filter_dict.items())
Beispiel #33
0
class ModuleStruct:
    """
    Define a module struct which stores all info. to generate statement.

    Args:
        nd_struct_list (list): A list of node structs.
        init_as_parent (bool): Control init method if the ModuleStruct be init as a parent module struct.
        parent_base (ModuleStruct): The base ModuleStruct the current ModuleStruct to be init as.
    """
    def __init__(self, nd_struct_list, init_as_parent=False, parent_base=None):
        """Init. a module by NodeStructs."""
        self.pattern_id = -1  # pattern num, -1 as Main module
        self.pattern_uid = -1  # unique module id for this pattern
        self.parent_id = None  # parent's pattern num
        self.parent_uid = None  # parent's pattern module unique id
        self.initialized = False
        self.identifier = None
        self.module_name = None
        self.scope_depth = None
        self.head_nd_struct = None
        self.head_nd_struct_index = None
        self.tail_nd_struct = None
        self.tail_nd_struct_index = None
        self._node_structs = list()
        self._module_structs = list()

        self._fragment = None
        self._args_translator = None
        self._parent_module_struct = None
        # only store original formal args name, not global
        self._nodes_structs_formal_args_list = list()

        # define other settings here
        self._node_args_translation_list = list()
        self._var_name_mgr = LocalVarNameMgr()
        self.construct_header_x = OrderedDict(
        )  # key is header x, value is precursors onnx name
        self.inputs_in_construct_header = OrderedDict(
        )  # key is precursors onnx name, value is x in parent construct

        # key is node's onnx name(output provider), value is (provider_succ_name, opt_var_name)
        self.outputs_collection = dict()
        self.matched_inputs = list(
        )  # Matched inputs will can be directly used by code line generation

        # key is ext. succ node onnx name, value is local opt_var
        self.external_successor_local_returns_map = OrderedDict()

        # Define outputs manager, note this will be assigned later by Generator.
        self.outputs_manager = None

        self._global_context = GlobalContext()

        # Define a dict to store the reference for quick searching
        self.rapid_reference = dict()

        # new vars for matcher
        self.inputs_register = OrderedDict()  # reg by sub
        self.outputs_register = OrderedDict()  # reg by sub
        self.internal_outputs_collection = dict()  # reg by sub

        # new vars for shared weights
        self.shared_weights_collection = dict()  # reg by sub
        self.shared_weights_counter = 0  # updated by sub

        if init_as_parent and (parent_base is not None):
            self.reset_as_parent_passed_in(parent_base)
        else:
            # start initialization
            if not self.initialized:
                self._init_module(nd_struct_list)
            else:
                self._update_module(nd_struct_list)

            # assign this module reference to node
            for (_, nd_struct) in nd_struct_list:
                nd_struct.parent_module_struct = self

    def reset_as_parent_passed_in(self, parent_base):
        """
        Reset all attributes and filled as a parent module of the module passed in.

        Args:
            parent_base(ModuleStruct): The base ModuleStruct to be passed in for ModuleStruct init.

        Note:
            This function must be called only if the new ModuleStruct is a parent of parent_base.
        """
        self.identifier = copy.deepcopy(parent_base.identifier)[:-1]
        self.scope_depth = copy.deepcopy(parent_base.scope_depth) - 1
        self.module_name = Scope.scope_to_module_name(self.identifier)
        self.head_nd_struct = parent_base.head_nd_struct
        self.head_nd_struct_index = parent_base.head_nd_struct_index
        self.tail_nd_struct = parent_base.tail_nd_struct
        self.tail_nd_struct_index = parent_base.tail_nd_struct_index
        self._node_structs = list()
        self._module_structs = list()
        self._fragment = None
        self._args_translator = None
        self.initialized = True
        self._set_pattern_id()
        self._find_parent_module()
        self.init_args_translator()
        self._parent_module_struct = None
        self._nodes_structs_formal_args_list = list()
        self._node_args_translation_list = list()

    def _set_pattern_id(self):
        """Set pattern id which matches the module fragment pattern."""
        if not self.initialized:
            return
        if self.scope_depth < 1:
            self.pattern_id = -1
            self.pattern_uid = -1
            return
        self.pattern_id = self.identifier[-1][0]
        self.pattern_uid = self.identifier[-1][1]

    def _init_module(self, nd_struct_list):
        """Init this ModuleStruct by a list of Nodes."""
        (nd_topo_idx, nd_struct) = nd_struct_list[0]
        self.identifier = nd_struct.scope.path
        self.module_name = nd_struct.scope.to_str
        self.scope_depth = nd_struct.scope.depth
        self.head_nd_struct = nd_struct
        self.head_nd_struct_index = nd_topo_idx
        self.tail_nd_struct = nd_struct_list[-1][1]
        self.tail_nd_struct_index = nd_struct_list[-1][0]
        self._node_structs = nd_struct_list
        self.initialized = True
        self._set_pattern_id()
        self._find_parent_module()
        self.init_args_translator()

    def _update_module(self, nd_struct_list):
        """Update the ModuleStruct attributes from a list of Nodes."""
        (nd_topo_idx_head, nd_struct_head) = nd_struct_list[0]
        (nd_topo_idx_tail, nd_struct_tail) = nd_struct_list[-1]
        if self.identifier != nd_struct_head.scope.path:
            raise ValueError(
                "Unable to update this module struct {} due to different identifier {}"
                .format(self.identifier, nd_struct_head.scope.path))
        if nd_topo_idx_head < self.head_nd_struct_index:
            self.head_nd_struct_index = nd_topo_idx_head
            self.head_nd_struct = nd_struct_head
        if nd_topo_idx_tail > self.tail_nd_struct_index:
            self.tail_nd_struct_index = nd_topo_idx_tail
            self.tail_nd_struct = nd_struct_tail
        self._node_structs += nd_struct_list

    def _find_parent_module(self):
        """Set the parent's module pattern and uid."""
        if not self.initialized:
            return
        if self.scope_depth == 0:  # is Main Module
            pass
        elif self.scope_depth == 1:  # parent pattern is Main module
            self.parent_id = -1
            self.parent_uid = -1
        else:  # this is a submodule in a module
            (self.parent_id,
             self.parent_uid) = Scope.get_parent_module_num_and_uid(
                 self.identifier)

    def __repr__(self):
        return str({
            "address":
            hex(id(self)),
            "identifier":
            self.identifier,
            "parent": (self.parent_id, self.parent_uid),
            "name":
            self.module_name,
            "pattern":
            self.pattern_id,
            "scope_depth":
            self.scope_depth,
            "nd_idx_range":
            "{} -> {}".format(self.head_nd_struct_index,
                              self.tail_nd_struct_index),
            "initialized":
            self.initialized
        })

    def init_args_translator(self):
        """Initialize the Args Translator for the module."""
        var_name = self.ms_var_name
        self._args_translator = ArgsTranslation(None, var_name, None)

    def update_module_fragment(self):
        """Update this module's fragment."""
        if self._fragment is None:
            return

        # update input output shape
        self._fragment.input_shape = self.head_nd_struct.fragment.input_shape
        self._fragment.output_shape = self.tail_nd_struct.fragment.output_shape

        # update formal args
        self._fragment.formal_args.update(self._args_translator.formal_args)
        self._fragment.formal_args_value.update(
            self._args_translator.formal_args_values)
        # update actual args
        self._fragment.actual_args.update(self._args_translator.actual_args)
        # update others..

    def add_submodule(self, md_structs):
        """
        Add another module struct(s) to this ModuleStruct.

        Args:
            md_structs ([ModuleStruct, list]): a (list) ModuleStruct to be added in this ModuleStruct.
        """
        tail_md = md_structs
        if isinstance(md_structs, ModuleStruct):
            md_structs.args_translator.take_formal_args_from_nodes_and_submodules(
                md_structs.get_all_sub_translators())
            self._module_structs.append(md_structs)
            md_structs.parent_module_struct = self
        elif isinstance(md_structs, list):
            for md_s in md_structs:
                md_s.args_translator.take_formal_args_from_nodes_and_submodules(
                    md_s.get_all_sub_translators())
                md_s.parent_module_struct = self
            self._module_structs += md_structs
            tail_md = md_structs[-1]
        else:
            raise TypeError(
                "ModuleStruct cannot add an unsupported Type {} to module_structs list."
                .format(type(md_structs)))
        # update tail node and index
        if self.tail_nd_struct_index < tail_md.tail_nd_struct_index:
            self.tail_nd_struct = tail_md.tail_nd_struct
            self.tail_nd_struct_index = tail_md.tail_nd_struct_index

    def _update_formal_args_for_all_nd_structs(self):
        """
        Init nodes' args translator and find formal args.
        And collect nodes' formal args.
        """
        if len(self._node_args_translation_list) != len(self._node_structs):
            raise ValueError(
                "ModuleStruct cannot update nodes' formal args due to length inconsistent."
            )
        for idx, (_, nd_struct) in enumerate(self._node_structs):
            formal_arg_of_this_node = self._node_args_translation_list[idx]
            # update var_name to ensure all node names' are unique in a module.
            nd_struct.update_var_name(idx)
            nd_struct.init_args_translator(formal_arg_of_this_node)
            if nd_struct.args_translator is not None:
                self._nodes_structs_formal_args_list.append(
                    nd_struct.args_translator.formal_args_values)
            else:
                self._nodes_structs_formal_args_list.append(None)

    def update_args_translation_list(self, formal_args):
        """
        Receive a list of args name to be changed to formal args, and change them.

        Args:
            formal_args (list[str]): a list of args name to be changed to formal args.
        """
        self._node_args_translation_list = formal_args
        self._update_formal_args_for_all_nd_structs()

    def get_all_sub_translators(self):
        """
        Return a list of args_translators of submodules / nodes affiliated to this module.

        Note:
            The order of returned list is followed by the actual topological order.

        Returns:
            list, a list of args_translators.
        """
        ret = []
        for (_, struct) in self.get_generate_order():
            if struct.args_translator is not None:
                ret.append(struct.args_translator)
        return ret

    def get_generate_order(self):
        """
        Return the order of generated code by index.

        Return:
            list, a list of reference of node_struct or module_struct.
        """
        ret = list()
        if not self._module_structs:
            return self._node_structs
        # Generate a list of tuple (idx, module_structs)
        for md_struct in self._module_structs:
            ret.append((md_struct.head_nd_struct_index, md_struct))
        if self.node_structs:
            ret += self.node_structs
        ret.sort(key=lambda x: x[0])
        return ret

    def _code_line_init_statement_shared_weights_args(self):
        """Generate the args for shared weights where calling this module."""
        args_list = list()
        for passthrough_w_onnx_name, passthrough_w_var_name in self.shared_weights_collection.items(
        ):
            passthrough_w_var_name_in_parent = \
                self.parent_module_struct.shared_weights_collection.get(passthrough_w_onnx_name)
            if self.parent_module_struct.identifier == []:  # now only consider declaration in main model
                args_list.append(
                    f"{passthrough_w_var_name}=self.{passthrough_w_var_name_in_parent}"
                )
            else:
                args_list.append(
                    f"{passthrough_w_var_name}={passthrough_w_var_name_in_parent}"
                )
        return args_list

    def _code_line_init_generate_shared_w_declaration_for_repeated(self):
        """Force to repeat sub nodes init code line for fulfillment of shared weight declaration in main model."""
        for _, nd_struct in self._node_structs:
            nd_struct.code_line_in_init()

    def code_line_in_init(self):
        """Initialization line of code in module init block."""
        self._code_line_init_generate_shared_w_declaration_for_repeated()
        left = "self.{}".format(self.ms_var_name)
        args_list = list()
        # Load args in init statement.
        if self._args_translator is not None:  # from args_translator
            if self._args_translator.actual_args:  # load actual args
                args_list += self._args_translator.actual_args_to_str_list
            elif self._args_translator.actual_args_backup and self.parent_id == -1:
                # For modules repeated in multiple levels, the module under main model should
                # not use formal args as it is unnecessary -> load from actual args backup
                args_list += self._args_translator.actual_args_backup_to_str_list
            args_list += self._args_translator.formal_args_to_str_list  # load from formal args
        else:
            args_list += self._fragment.actual_args
        args_list += self._code_line_init_statement_shared_weights_args()
        right = f"{self.class_name}({', '.join(args_list)})"
        return left, right

    def code_line_in_construct(self, inputs=None):
        """Construct line of code in module construct block."""
        outputs_edges = list(self.outputs_register.keys())
        num_output = len(outputs_edges)

        # Allocate opt_var_name
        if num_output == 1:  # single output
            left = [f"{self.ms_opt_var_name}"]
        else:
            left = [
                f"{self.ms_opt_var_name}_{num}" for num in range(num_output)
            ]

        inputs = []
        # Update self's outputs mgr
        for idx, edge in enumerate(outputs_edges):
            base_out = self.outputs_manager.get_base_out(edge)
            if base_out.opt_var_name is None:
                print(
                    f"ModuleStruct {self.identifier} has an output {base_out.onnx_edge_name} not has opt_var_name"
                )
                base_out.opt_var_name = left[idx]
            self.parent_module_struct.internal_outputs_collection[
                base_out.onnx_edge_name] = base_out.opt_var_name

        # Take inputs from parent & previous
        for input_edge in self.inputs_register:
            if input_edge in self.parent_module_struct.inputs_register:
                inputs.append(
                    self.parent_module_struct.inputs_register.get(input_edge))
            elif input_edge in self.parent_module_struct.internal_outputs_collection:
                inputs.append(
                    self.parent_module_struct.internal_outputs_collection.get(
                        input_edge))

        right = f"self.{self.ms_var_name}({', '.join(inputs)})"
        left = ", ".join(left)
        return (left, right)

    @property
    def node_structs(self):
        """Return all node structs in this module."""
        return self._node_structs

    @property
    def module_structs(self):
        """Return all module structs in this module."""
        return self._module_structs

    @property
    def parent_module_struct(self):
        """Return this module's parent module struct."""
        return self._parent_module_struct

    @parent_module_struct.setter
    def parent_module_struct(self, ref):
        """Set this modu;e's parent module struct."""
        self._parent_module_struct = ref

    @property
    def args_translator(self):
        """Return the args translator."""
        return self._args_translator

    @property
    def head_nd_struct_precursor_nodes_names(self) -> list:
        """Return head node's precursor nodes names."""
        return self.head_nd_struct.precursor_nodes_names

    @property
    def head_nd_struct_precursor_nodes_structs(self) -> list:
        """Return head node's precursor nodes structs."""
        return self.head_nd_struct.precursor_nodes_structs

    @property
    def tail_nd_struct_successor_nodes_names(self) -> list:
        """Return tail node's successor nodes names."""
        return self.tail_nd_struct.successor_nodes_names

    @property
    def tail_nd_struct_successor_nodes_structs(self) -> list:
        """Return tail node's successor nodes structs."""
        return self.tail_nd_struct.successor_nodes_structs

    @property
    def onnx_names_from_nodes(self) -> list:
        """Return all nodes onnx names in this module."""
        if self._global_context.build_struct_finished and "_onnx_names_from_nodes" in self.rapid_reference:
            return self.rapid_reference["_onnx_names_from_nodes"]
        ret = [node.onnx_name for (_, node) in self.node_structs]
        if self._global_context.build_struct_finished:
            self.rapid_reference["_onnx_names_from_nodes"] = ret
        return ret

    @property
    def onnx_names_from_submodules(self) -> list:
        """Return all nodes onnx names in submodules of this module."""
        if self._global_context.build_struct_finished and "_onnx_names_from_submodules" in self.rapid_reference:
            return self.rapid_reference["_onnx_names_from_submodules"]

        ret = []
        for md_struct in self.module_structs:
            ret += md_struct.onnx_names
        if self._global_context.build_struct_finished:
            self.rapid_reference["_onnx_names_from_submodules"] = ret

        return ret

    @property
    def onnx_names(self) -> list:
        """Return all nodes' onnx names which contained by this module."""
        if self._global_context.build_struct_finished and "_onnx_names" in self.rapid_reference:
            return self.rapid_reference["_onnx_names"]
        ret = self.onnx_names_from_nodes + self.onnx_names_from_submodules
        if self._global_context.build_struct_finished:
            self.rapid_reference["_onnx_names"] = ret
        return ret

    @property
    def external_precursor_nodes_names(self) -> list:
        """Return all precursors nodes names not in this module."""
        ret = []
        for _, struct in self.get_generate_order():
            if isinstance(struct, NodeStruct):
                precursor_nodes_names = struct.precursor_nodes_names

            if isinstance(struct, ModuleStruct):
                precursor_nodes_names = struct.external_precursor_nodes_names

            for p_name in precursor_nodes_names:
                if p_name in self.onnx_names:
                    continue
                ret.append(p_name)
        return ret

    @property
    def external_successor_nodes_names(self) -> list:
        """Return all successor nodes names not in this module."""
        ret = []
        for _, struct in self.get_generate_order():
            if isinstance(struct, NodeStruct):
                successor_nodes_names = struct.successor_nodes_names

            if isinstance(struct, ModuleStruct):
                successor_nodes_names = struct.external_successor_nodes_names

            for s_name in successor_nodes_names:
                if s_name in self.onnx_names:
                    continue
                ret.append(s_name)
        return ret

    @property
    def class_name(self) -> str:
        """Return the class name for generating code of this module."""
        if self.pattern_id == -1:
            return "Model"
        if self._global_context.known_module_name.get("Module{}".format(
                self.pattern_id)) is not None:
            class_name = self._global_context.known_module_name.get(
                "Module{}".format(self.pattern_id))
        else:
            class_name = "Module{}".format(self.pattern_id)
        return class_name

    @property
    def ms_var_name(self) -> str:
        """Return the variable name for generated code statement of this module."""
        if self.pattern_id == -1:
            return "Model"
        return f"{self.class_name}_{self.pattern_uid}".lower()

    @property
    def ms_opt_var_name(self) -> str:
        """Return the variable name for generated code statement of the output of this module."""
        return "{}_opt".format(self.ms_var_name).lower()

    # The following part will be resetting nodes' external inputs for supporting multi-in/out
    # and should be called after generator.recursive_form_modules()

    def set_inputs_in_construct_header(self, header_x,
                                       onnx_precursor_node_name):
        """
        Mark the registered external inputs for code generation.

        Note:
            This function to be called by its parent (ModuleStruct).

        Args:
            header_x (str): The `x` in module construct header.
            onnx_precursor_node_name (str): The original onnx node name.
        """
        if self.inputs_in_construct_header.get(
                onnx_precursor_node_name) is not None:
            raise ValueError(
                "The input from {} has already registered. Check this Module \
                {} has duplicate inputs or not.".format(
                    onnx_precursor_node_name, self.identifier))
        self.inputs_in_construct_header[onnx_precursor_node_name] = header_x

    def allocate_construct_header_x(self, force_x=None):
        """
        Allocate the x in construct header for each external input.

        Args:
            force_x (str): Force the arg name to customized.
        """
        local_x_name = 'x'
        if force_x:  # name of x indicated by external
            local_x_name = force_x

        # set construct_header_x for current module
        allocated = set()
        for prec_name in self.external_precursor_nodes_names:
            if prec_name in allocated:
                continue
            x_name_in_construct_header = self._var_name_mgr.get_name(
                local_x_name)
            self.construct_header_x[x_name_in_construct_header] = prec_name
            allocated.add(prec_name)

        # Assign these inputs to nodes and submodules
        for _, struct in self.get_generate_order():
            if isinstance(struct, NodeStruct):  # register node's ext input
                self.reset_node_external_input_to_local(struct)
                self.register_node_output_to_module(struct)
            if isinstance(struct, ModuleStruct):  # reg module's ext input
                if not struct.construct_header_x:
                    struct.allocate_construct_header_x()
                self.reset_submodule_external_input_to_local(struct)
                self.register_submodule_output_to_module(struct)

        # remove parent module's ext. map if ext nodes in this module (no need return)
        for user_name in self.external_successor_local_returns_map.copy().keys(
        ):
            if user_name in self.onnx_names:
                self.external_successor_local_returns_map.pop(user_name)

    def _match_node_inputs(self, struct):
        """Match node's inputs with its precursor nodes."""
        for output_provider in struct.precursor_nodes_names:
            output_list = self.outputs_collection.get(output_provider)
            if output_list is None:
                # not in this module, check construct header
                for (self_x_name,
                     self_output_provider) in self.construct_header_x.items():
                    if self_output_provider == output_provider:
                        struct.matched_inputs.append(self_x_name)
                continue
            for output in output_list:
                (provider_succ, provider_closet_opt_var) = output
                if provider_closet_opt_var in struct.matched_inputs:
                    continue  # skip repeat
                if provider_succ == struct.onnx_name:
                    struct.matched_inputs.append(provider_closet_opt_var)

    def _match_sub_modules_inputs(self):
        """
        Match current module's submodules' inputs with corresponding outputs registered in current module.

        Description:
            The function matches these inputs by the following steps:
            1. For each submodule in the current module, take submodule's construct header
            2. Check submodule's construct header element requires an input from current module's
                construct header or outputs from other submodules.
            3. If from current module's construct header, assign corresponding x to the submodule.
                If from other submodules, assign required submodule output name to the submodule.
        """
        if not self.outputs_collection:
            return  # skip first node
        for (_, struct) in self.get_generate_order():
            if isinstance(struct, NodeStruct):
                self._match_node_inputs(struct)
                continue  # skip node
            sub_construct_header = struct.construct_header_x
            for (_, output_provider) in sub_construct_header.items():
                # check from outputs collection
                output_list = self.outputs_collection.get(output_provider)
                if output_list is None:
                    # not in this module, need from current module construct header
                    for (self_x_name, self_output_provider
                         ) in self.construct_header_x.items():
                        if self_output_provider == output_provider:
                            struct.matched_inputs.append(self_x_name)
                    continue
                for output in output_list:
                    (provider_succ, provider_closet_opt_var) = output
                    if provider_closet_opt_var in struct.matched_inputs:
                        continue  # skip repeat
                    if provider_succ in struct.onnx_names:
                        struct.matched_inputs.append(provider_closet_opt_var)

    def _append_to_outputs_collection(self, provider_name, val):
        """
        Helper function to add a nodes or submodules outputs to current module return statement.

        Args:
            provider_name (str): The onnx name of the output provider.
            val (list[tuple]): A list of tuple which contains
                the output provider's successor name and its opt_var_name.
        """
        exist_output = self.outputs_collection.get(provider_name)
        if isinstance(val, tuple):
            val = [val]
        if exist_output is None:  # add new entry
            exist_output = list()
        exist_output += (val)
        self.outputs_collection[provider_name] = exist_output

    def collect_returns(self):
        """
        Collect all nodes and submodules' returns in the module.

        Note:
            The logic is to collect the return from nodes and submodules by the order
            of topological index.

            For returns from a node, it will check if the return will be used externally.
            If external (external means the successor a.k.a the return user has different scope with the node),
            add this return to current module's outputs_collection, where
            key is this node's original onnx_name, and value is a list of
            tuple(successor_name, this node's opt_var_name)

            For returns from a submodule, it will check if the submodule has already collected returns,
            If not, do it and then continue the following procedures.
            Now we will check each element in submodule's outputs_collection. Note that we DO NOT check submodule's
            returns should be continued returning, but just return them.
            All these returns from submodules will be changes their original nodes output (a.k.a outputs provider)
            `opt_var_name` to submodules' `opt_var_name`.

            Finally, we match the outputs and inputs in the current module level.
        """
        for (_, struct) in self.get_generate_order():
            if isinstance(struct, NodeStruct):
                outputs_list = []
                # add these successor nodes name to collection for future use
                for succ in struct.successor_nodes_names:
                    outputs_list.append((succ, struct.ms_opt_var_name))
                if outputs_list:
                    self._append_to_outputs_collection(struct.onnx_name,
                                                       outputs_list)
            if isinstance(struct, ModuleStruct):
                # Remove unnecessary returns, succ are all inside current
                if not struct.outputs_collection:
                    struct.collect_returns()
                sub_outputs_collection = struct.outputs_collection
                # check each returns in sub
                for provider_name, outputs_list in sub_outputs_collection.items(
                ):
                    for output in outputs_list:
                        (succ,
                         _) = output  # (succ, provider_opt_var_name) in output
                        new_output = (succ, struct.ms_opt_var_name)
                        self._append_to_outputs_collection(
                            provider_name, new_output)
        self._match_sub_modules_inputs()

    def get_returned_opt_var_name(self) -> list:
        """Return a list of returned output var of this module."""
        idx = 0
        added_to_return = set()
        ret = []
        for ext_successor_requested, opt_var_name_in_this_module in self.external_successor_local_returns_map.items(
        ):
            if ext_successor_requested in added_to_return:
                continue
            ret.append(
                (ext_successor_requested, opt_var_name_in_this_module, idx))
            added_to_return.add(ext_successor_requested)
        return ret

    def reset_node_external_input_to_local(self, nd_struct):
        """
        Reset node's input to module's construct args
        """
        for prec_node_name in nd_struct.precursor_nodes_names_external:
            if prec_node_name in self.onnx_names:  # prec node in current module's.
                continue
            if prec_node_name in self.construct_header_x.values():
                # prec node assigned to construct header to passed in.
                local_x = get_dict_key_by_value(prec_node_name,
                                                self.construct_header_x)
                nd_struct.set_inputs_in_construct_header(
                    local_x, prec_node_name)
            else:  # Extra precursor nodes, raise error
                raise ValueError(
                    "Found external inputs of the Node but the module does not have it."
                )

    def reset_submodule_external_input_to_local(self, md_struct):
        """
        Reset submodule's external input to current module.

        Args:
            md_struct (ModuleStruct): The submodule in the current module.
        """
        # check submodule's input
        for _, submodule_precursor in md_struct.construct_header_x.items():
            if submodule_precursor in self.onnx_names:  # if internal, match with local nodes/submodules return
                # but do nothing here
                continue
            # if external, match with current module construct header x
            if submodule_precursor in self.construct_header_x.values():
                local_x = get_dict_key_by_value(submodule_precursor,
                                                self.construct_header_x)
                md_struct.set_inputs_in_construct_header(
                    local_x, submodule_precursor)
            else:  # Extra precursor nodes, raise error
                raise ValueError(
                    "Found external inputs of the submodule but the module does not have it."
                )

    def register_node_output_to_module(self, nd_struct):
        """Register nodes outputs to this module's return."""
        for succ_node_name in nd_struct.successor_nodes_names_external:
            self.external_successor_local_returns_map[
                succ_node_name] = nd_struct.ms_opt_var_name

    def register_submodule_output_to_module(self, md_struct):
        """Register submodule outputs to this module's return."""
        submodule_returns = md_struct.get_returned_opt_var_name()
        submodule_opt_var_name = md_struct.ms_opt_var_name
        for (submodule_ext_succ, _, ith_output) in submodule_returns:
            self.external_successor_local_returns_map[submodule_ext_succ] = (
                submodule_opt_var_name, ith_output)

    # The following funcs are designated to be invoked by matcher.
    def add_inputs_edge(self, edge_name: str):
        construct_header_length = len(self.inputs_register.values())
        default_x_str = "x"
        if not edge_name in self.inputs_register:
            self.inputs_register[edge_name] = "".join([default_x_str, str(construct_header_length-1)]) \
                if construct_header_length > 0 else default_x_str

    def add_outputs_edge(self, edge_name: str):
        if edge_name in self.outputs_register:
            return  # to be filled during code generation, should from sub's opt_var_name
        self.outputs_register[edge_name] = "<placeholder>"

    def fill_outputs_edge(self, edge_name: str, opt_var_name: str):
        # FILL the outputs edge once you got a opt_var_name of corresponding node!!!
        if not edge_name in self.outputs_register:
            raise ValueError(f"ModuleStruct {self.identifier} does not have edge "\
                             f"{edge_name} and unable to fill its output var name.")
        if self.outputs_register[edge_name] != "<placeholder>":
            raise ValueError(f"The edge has been already filled as {self.outputs_register[edge_name]}" \
                             f" instead of your {opt_var_name}")
        self.outputs_register[edge_name] = opt_var_name
Beispiel #34
0
class RunInfo:
    """A little plaintext file containing very basic info about a pants run.

  Can only be appended to, never edited.
  """
    def __init__(self, info_file):
        self._info_file = info_file
        safe_mkdir_for(self._info_file)
        self._info = OrderedDict()
        if os.path.exists(self._info_file):
            with open(self._info_file, 'r') as infile:
                info = infile.read()
            for m in re.finditer("""^([^:]+):(.*)$""", info, re.MULTILINE):
                self._info[m.group(1).strip()] = m.group(2).strip()

    def path(self):
        return self._info_file

    def get_info(self, key):
        return self._info.get(key, None)

    def __getitem__(self, key):
        ret = self.get_info(key)
        if ret is None:
            raise KeyError(key)
        return ret

    def get_as_dict(self):
        return self._info.copy()

    def add_info(self, key, val, ignore_errors=False):
        """Adds the given info and returns a dict composed of just this added info."""
        self.add_infos((key, val), ignore_errors=ignore_errors)

    def add_infos(self, *keyvals, **kwargs):
        """Adds the given info and returns a dict composed of just this added info."""
        kv_pairs = []
        for key, val in keyvals:
            key = key.strip()
            val = str(val).strip()
            if ':' in key:
                raise ValueError(f'info key "{key}" must not contain a colon.')
            kv_pairs.append((key, val))

        for k, v in kv_pairs:
            if k in self._info:
                raise ValueError(
                    f'info key "{k}" already exists with value {self._info[k]}. '
                    'Cannot add it again with value {v}.')
            self._info[k] = v

        try:
            with open(self._info_file, 'a') as outfile:
                for k, v in kv_pairs:
                    outfile.write('{}: {}\n'.format(k, v))
        except IOError:
            if not kwargs.get('ignore_errors', False):
                raise

    def add_basic_info(self, run_id, timestamp):
        """Adds basic build info."""
        datetime = time.strftime('%A %b %d, %Y %H:%M:%S',
                                 time.localtime(timestamp))
        user = getpass.getuser()
        machine = socket.gethostname()
        buildroot = get_buildroot()
        # TODO: Get rid of the redundant 'path' key once everyone is off it.
        self.add_infos(('id', run_id), ('timestamp', timestamp),
                       ('datetime', datetime), ('user', user),
                       ('machine', machine), ('path', buildroot),
                       ('buildroot', buildroot), ('version', VERSION))

    def add_scm_info(self):
        """Adds SCM-related info."""
        scm = get_scm()
        if not scm:
            return
        revision = scm.commit_id
        branch = scm.branch_name or revision
        self.add_infos(('revision', revision), ('branch', branch))
Beispiel #35
0
def main(run_dct):
    """
    data_sources: list, i.e. [1,2]
        0: dxcluster
        1: WSPRNet
        2: RBN

    loc_sources: list, i.e. ['P','Q']
        P: user Provided
        Q: QRZ.com or HAMCALL
        E: Estimated using prefix
    """

    # Get Variables from run_dct
    sDate = run_dct['sDate']
    eDate = run_dct['eDate']
    params = run_dct['params']
    xkeys = run_dct['xkeys']
    rgc_lim = run_dct['rgc_lim']
    filter_region = run_dct['filter_region']
    filter_region_kind = run_dct['filter_region_kind']
    band_obj = run_dct['band_obj']
    xb_size_min = run_dct['xb_size_min']
    yb_size_km = run_dct['yb_size_km']
    base_dir = run_dct.get('base_dir')
    output_dir = run_dct.get('output_dir')
    reprocess = run_dct.get('reprocess', False)
    loc_sources = run_dct.get('loc_sources')
    data_sources = run_dct.get('data_sources', [1, 2])

    # Define path for saving NetCDF Files
    if output_dir is None:
        tmp = []
        if filter_region is not None:
            tmp.append('{!s}'.format(filter_region))
            tmp.append('{!s}'.format(filter_region_kind))
        tmp.append('{:.0f}-{:.0f}km'.format(rgc_lim[0], rgc_lim[1]))
        tmp.append('dx{:.0f}min'.format(xb_size_min))
        tmp.append('dy{:.0f}km'.format(yb_size_km))
        ncs_path = os.path.join(base_dir, '_'.join(tmp))
    else:
        ncs_path = output_dir

    gl.prep_output({0: ncs_path}, clear=reprocess)

    # Loop through dates
    dates = list(daterange(sDate, eDate))
    if strip_time(sDate) != strip_time(eDate):
        dates = dates[:-1]

    for dt in tqdm.tqdm(dates):
        nc_name = dt.strftime('%Y%m%d') + '.data.nc'
        nc_path = os.path.join(ncs_path, nc_name)

        # Skip processing if file already exists
        if os.path.exists(nc_path + '.bz2'):
            continue

        # Load spots from CSVs
        load_dates = [dt, dt + pd.Timedelta('1D')]

        df = pd.DataFrame()
        for ld_inx, load_date in enumerate(load_dates):
            ld_str = load_date.strftime("%Y-%m-%d")
            dft = gl.load_spots_csv(ld_str,
                                    data_sources=data_sources,
                                    rgc_lim=rgc_lim,
                                    loc_sources=loc_sources,
                                    filter_region=filter_region,
                                    filter_region_kind=filter_region_kind)

            if dft is None:
                print('No data for {!s}'.format(ld_str))
                continue
            for xkey in xkeys:
                dft[xkey] = ld_inx * 24 + dft[xkey]

            df = df.append(dft, ignore_index=True)

        # Set Up Data Storage Containers
        data_das = {}
        for xkey in xkeys:
            data_das[xkey] = {}
            for param in params:
                data_das[xkey][param] = []

        map_das = {}
        for xkey in xkeys:
            map_das[xkey] = []

        src_cnts = {}
        for xkey in xkeys:
            if xkey not in src_cnts:
                src_cnts[xkey] = {}
            for band_inx, (band_key,
                           band) in enumerate(band_obj.band_dict.items()):
                if band_key not in src_cnts[xkey]:
                    src_cnts[xkey][band_key] = {}

        for band_inx, (band_key,
                       band) in enumerate(band_obj.band_dict.items()):
            if len(df) == 0:
                continue

            frame = df.loc[df["band"] == band.get('meters')].copy()

            # Create attrs diction to save with xarray DataArray
            attrs = OrderedDict()
            attrs['sTime'] = dt
            attrs['param'] = param
            attrs['filter_region'] = filter_region
            attrs['filter_region_kind'] = filter_region_kind
            attrs['band'] = band_key
            attrs['band_name'] = band['name']
            attrs['band_fname'] = band['freq_name']

            for xkey in xkeys:
                src_cnts[xkey][band_key] = gl.count_sources(frame)
                for param in params:
                    # Compute General Data
                    attrs['xkey'] = xkey
                    attrs['param'] = param
                    attrs['dx'] = xb_size_min / 60.
                    attrs['xlim'] = (0, 24)
                    attrs['ykey'] = 'dist_Km'
                    attrs['ylim'] = rgc_lim
                    attrs['dy'] = yb_size_km
                    result = calc_histogram(frame, attrs)
                    data_da_result = result
                    data_das[xkey][param].append(result)

                # Compute Map
                time_bins = np.array(data_da_result.coords[xkey])
                map_attrs = attrs.copy()
                del map_attrs['param']
                map_attrs['xkey'] = 'md_long'
                map_attrs['xlim'] = (-180, 180)
                map_attrs['dx'] = 1
                map_attrs['ykey'] = 'md_lat'
                map_attrs['ylim'] = (-90, 90)
                map_attrs['dy'] = 1

                map_tmp = []
                for tb_inx, tb_0 in enumerate(time_bins[:-1]):
                    tb_1 = time_bins[tb_inx + 1]
                    tf = np.logical_and(frame[xkey] >= tb_0,
                                        frame[xkey] < tb_1)
                    tb_frame = frame[tf].copy()
                    result = calc_histogram(tb_frame, map_attrs)
                    result.coords[xkey] = tb_0
                    map_tmp.append(result)
                map_tmp_da = xr.concat(map_tmp, dim=xkey)
                map_das[xkey].append(map_tmp_da)

        # Continue if no data.
        if len(map_das[xkeys[0]]) == 0:
            continue

        # Maps - Concatenate all bands into single DataArray
        for xkey in xkeys:
            map_das[xkey] = xr.concat(map_das[xkey], dim='freq_MHz')

        map_dss = OrderedDict()
        for xkey in xkeys:
            map_ds = xr.Dataset()
            map_ds['spot_density'] = map_das[xkey]
            map_dss[xkey] = map_ds

        # Time Series - Concatenate all bands into single DataArray
        for xkey in xkeys:
            for param in params:
                data_da = data_das[xkey][param] = xr.concat(
                    data_das[xkey][param], dim='freq_MHz')

        data_dss = OrderedDict()
        for xkey in xkeys:
            data_ds = xr.Dataset()
            for param in params:
                d_ds = data_das[xkey][param]
                src_cnt = src_cnts[xkey]
                d_ds.attrs['src_cnt'] = pd.DataFrame(src_cnt).to_json()
                data_ds[param] = d_ds
            data_dss[xkey] = data_ds

        # Save to data file.
        first = True
        for xkey, map_ds in map_dss.items():
            if first:
                mode = 'w'
                first = False
            else:
                mode = 'a'

            group = 'map/{!s}'.format(xkey)
            map_ds.to_netcdf(nc_path, mode=mode, group=group)

        for xkey, data_ds in data_dss.items():
            group = 'time_series/{!s}'.format(xkey)
            data_ds.to_netcdf(nc_path, mode='a', group=group)

        mbz2 = gl.MyBz2(nc_path)
        mbz2.compress()
    return ncs_path
Beispiel #36
0
class Graph(object):
    """
    A graph of ``Components``
    """
    def __init__(self,
                 name=None,
                 default_capacity=10,
                 description=None,
                 metadata=None):
        self.name = name
        self.default_capacity = default_capacity
        self.metadata = metadata or {}
        self.description = None

        # type: OrderedDict[str, rill.engine.component.Component]
        self._components = OrderedDict()

        # exported inports and outports
        # type: OrderedDict[str, rill.engine.inputport.InputPort]
        self.inports = OrderedDict()
        self.inport_metadata = OrderedDict()

        # type: OrderedDict[str, rill.engine.inputport.OutputPort]
        self.outports = OrderedDict()
        self.outport_metadata = OrderedDict()

    def __repr__(self):
        return '{}(name={!r})'.format(self.__class__.__name__, self.name)

    def __getstate__(self):
        # avoid copying InputPorts and OutputPorts
        data = self.__dict__.copy()
        for k in ('inports', 'outports'):
            data.pop(k)

        inports = []
        for name, port in self.inports.items():
            inports.append((name, port.component.name, port._name, port.index))
        data['inports'] = inports

        outports = []
        for name, port in self.outports.items():
            outports.append(
                (name, port.component.name, port._name, port.index))
        data['outports'] = outports
        return data

    def __setstate__(self, data):
        # avoid copying InputPorts and OutputPorts
        inports = data.pop('inports')
        outports = data.pop('outports')
        self.__dict__.update(data)
        self.inports = OrderedDict()
        for name, comp_name, port_name, port_index in inports:
            self.inports[name] = self.get_component_port(
                (comp_name, port_name), index=port_index, kind='in')
        self.outports = OrderedDict()
        for name, comp_name, port_name, port_index in outports:
            self.outports[name] = self.get_component_port(
                (comp_name, port_name), index=port_index, kind='out')

    def copy(self, deep=True):
        """
        Create a deep copy of the graph.

        Returns
        -------
        ``Graph``
        """
        if deep:
            return copy.deepcopy(self)
        else:
            graph = Graph(self.name, self.default_capacity)
            graph.inports = self.inports.copy()
            graph.outports = self.outports.copy()
            graph._components = self._components.copy()
            return graph

    @supports_listeners
    def set_metadata(self, metadata):
        """
        Set graph level metadata

        Parameters
        ----------
        metadata: dict
        """
        merge_metadata(metadata, self.metadata)
        self.set_metadata.event.emit(metadata)
        return self.metadata

    @supports_listeners
    def rename(self, new_name):
        """
        Rename graph

        Parameters
        ----------
        new_name: str
        """
        old_name = self.name
        self.name = new_name
        self.rename.event.emit(old_name, new_name)
        return new_name

    # Components --

    def _get_unique_name(self, basename):
        i = 0
        name = basename
        while name in self._components:
            i += 1
            name = basename + str(i)
        return name

    def add_component(self, *args, **initializations):
        """
        Instantiate a component and add it to the network.

        Parameters
        ----------
        name : str
            name of component
        comp_type : Type[``rill.engine.component.Component``]
            component class to instantiate

        Returns
        -------
        ``rill.engine.component.Component``
        """
        if len(args) == 1:
            arg = args[0]
            if isinstance(arg, Component):
                comp = arg
                name = comp.name
            elif isclass(arg) and issubclass(arg, Component):
                comp_type = arg
                name = comp_type.type_name or comp_type.__name__
                name = self._get_unique_name(name)
                comp = comp_type(name)
            else:
                raise ValueError()
        elif len(args) == 2:
            name, comp_type = args
            if not isclass(comp_type) or not issubclass(comp_type, Component):
                raise TypeError("comp_type must be a sub-class of Component")
            comp = comp_type(name)
        else:
            raise ValueError()

        if name in self._components:
            raise FlowError(
                "Component {} already exists in network".format(name))

        self.put_component(name, comp)

        for name, value in initializations.items():
            receiver = comp.port(name, kind='in')
            if value is None and not receiver.required:
                continue
            if isinstance(value,
                          (OutputPort, OutputArray, InputPort, InputArray)):
                self.connect(value, receiver)
            else:
                self.initialize(value, receiver)
        return comp

    @supports_listeners
    def set_node_metadata(self, node, metadata):
        """
        Set metadata on node

        Parameters
        ----------
        node: ``rill.engine.component.Component``
        metadata: {}
        """
        merge_metadata(metadata, node.metadata)
        self.set_node_metadata.event.emit(node, metadata)
        return node.metadata

    def add_graph(self, name, graph, **initializations):
        """
        Instantiate a component and add it to the network.

        Parameters
        ----------
        name : str
            name of component within the graph
        graph : ``Graph``
            graph to add

        Returns
        -------
        ``rill.engine.component.Component``
        """
        from rill.engine.subnet import make_subgraph
        comp_type = make_subgraph(name, graph)
        return self.add_component(name, comp_type, **initializations)

    @supports_listeners
    def remove_component(self, name):
        """
        Remove component from graph

        Parameters
        ----------
        name: str
        """
        # FIXME: this needs some love
        # assert not self.active
        component = self._components.pop(name)
        for inport in component.inports:
            if inport.is_connected() and not inport.is_initialized():
                for outport in inport._connection.outports:
                    self.disconnect(inport, outport)
        for outport in component.outports:
            if outport.is_connected():
                self.disconnect(outport._connection.inport, outport)
        # FIXME: remove references in self.inports and self.outports
        self.remove_component.event.emit(component)

    @supports_listeners
    def rename_component(self, orig_name, new_name):
        """
        Change name of component

        Parameters
        ----------
        orig_name: str
        new_name: str
        """
        # FIXME: this needs some love
        # assert not self.active
        assert new_name not in self._components
        component = self._components.pop(orig_name)
        self._components[new_name] = component
        self.rename_component.event.emit(orig_name, new_name, component)

    def component(self, name):
        """
        Parameters
        ----------
        name : str
            name of component

        Returns
        -------
        ``rill.engine.component.Component``

        Raises
        ------
        ``rill.engine.exceptions.FlowError`` : if no component found
        """
        comp = self.get_component(name)
        if comp is None:
            raise FlowError("Reference to unknown component " + name)
        return comp

    def get_components(self):
        """
        Get a dictionary of components in this network

        Returns
        -------
        Dict[str, ``rill.enginge.component.Component``]
        """
        return self._components

    def get_component(self, name):
        """
        Returns the requested component in this network if present or None
        if not present.

        Returns
        -------
        ``rill.enginge.component.Component``
        """
        return self._components.get(name)

    # FIXME: remove this and run/move Component._init to __init__ ?
    @supports_listeners
    def put_component(self, name, comp):
        """
        Adds a component and inits it.

        Parameters
        ----------
        name : str
        comp : ``rill.enginge.component.Component``
        """
        # initialize the component's ports so they can be used for network
        # building
        comp._init()
        self._components[name] = comp
        self.put_component.event.emit(name, comp)

    # Ports --

    @supports_listeners
    def export(self, internal_port, external_port_name, metadata=None):
        """
        Exports component port for connecting to other networks
        as a sub network

        Parameters
        ----------
        internal_port : ``rill.engine.outputport.OutputPort`` or
            ``rill.engine.inputport.InputPort`` or str
        external_port_name : str
            name of port that will be exposed
        """
        metadata = metadata or {}

        internal_port = self.get_component_port(internal_port)

        if isinstance(internal_port, InputPort):
            self.inports[external_port_name] = internal_port
            self.inport_metadata[external_port_name] = metadata

        elif isinstance(internal_port, OutputPort):
            self.outports[external_port_name] = internal_port
            self.outport_metadata[external_port_name] = metadata

        self.export.event.emit(internal_port, external_port_name, metadata)

    @supports_listeners
    def remove_inport(self, external_port_name):
        """
        Remove public inport from graph

        Parameters
        ----------
        external_port_name: str
        """
        del self.inports[external_port_name]
        del self.inport_metadata[external_port_name]
        self.remove_inport.event.emit(external_port_name)

    @supports_listeners
    def remove_outport(self, external_port_name):
        """
        Remove public outport from graph

        Parameters
        ----------
        external_port_name: str
        """
        del self.outports[external_port_name]
        del self.outport_metadata[external_port_name]
        self.remove_outport.event.emit(external_port_name)

    @supports_listeners
    def set_inport_metadata(self, external_port_name, metadata):
        """
        Set metadata on inport

        Parameters
        ----------
        external_port_name: str
        metadata: dict
        """
        port_metadata = self.inport_metadata[external_port_name]
        merge_metadata(metadata, port_metadata)
        self.set_inport_metadata.event.emit(external_port_name, metadata)
        return port_metadata

    @supports_listeners
    def set_outport_metadata(self, external_port_name, metadata):
        """
        Set metadata on outport

        Parameters
        ----------
        external_port_name: str
        metadata: dict
        """
        port_metadata = self.outport_metadata[external_port_name]
        merge_metadata(metadata, port_metadata)
        self.set_outport_metadata.event.emit(external_port_name, metadata)
        return port_metadata

    # FIXME: might be better to split this into get_component_inport / get_component_outport
    # the main argument for the current design is if you don't know or care what
    # kind of port you want, but in practice, 'kind' is always provided
    def get_component_port(self, arg, index=None, kind=None):
        """
        Get a port on a component.

        Parameters
        ----------
        arg : Union[``rill.engine.outputport.OutputPort``, ``rill.engine.inputport.InputPort``, str]
        index : Optional[int]
            index of element, if port is an array. If None, the next available
            index is used
        kind : str
            {'in', 'out'}

        Returns
        -------
        Union[``rill.engine.outputport.OutputPort``, ``rill.engine.inputport.InputPort``]
        """
        if isinstance(arg, (OutputPort, OutputArray, InputPort, InputArray)):
            port = arg
            if kind is not None and port.kind != kind:
                raise FlowError("Expected {}port: got {}".format(
                    kind, type(port)))
        else:
            if isinstance(arg, (tuple, list)):
                comp_name, port_name = arg
            elif isinstance(arg, basestring):
                split = arg.split('.')
                comp_name = '.'.join(split[:-1])
                port_name = split[-1]
            else:
                raise TypeError(arg)

            comp = self.component(comp_name)
            port = comp.port(port_name, kind=kind)

        if port.is_array() and index is not False:
            port = port.get_element(index, create=True)

        return port

    @supports_listeners
    def connect(self,
                sender,
                receiver,
                connection_capacity=None,
                count_packets=False,
                metadata=None):
        """
        Connect an output port of one component to an input port of another.

        Parameters
        ----------
        sender : Union[``rill.engine.inputport.InputPort``, str]
        receiver : Union[``rill.engine.outputport.OutputPort``, str]

        Returns
        -------
        ``rill.engine.inputport.InputPort``
        """
        outport = self.get_component_port(sender, kind='out')
        inport = self.get_component_port(receiver, kind='in')
        assert not outport.is_array()
        assert not inport.is_array()

        if connection_capacity is None:
            connection_capacity = self.default_capacity

        if inport._connection is None:
            inport._connection = Connection()
        inport._connection.connect(inport, outport, connection_capacity)

        metadata = metadata or {}
        edge_metadata = inport._connection.metadata.setdefault(outport, {})
        edge_metadata.update(metadata)

        self.connect.event.emit(outport, inport, connection_capacity, metadata)

        return inport

    @supports_listeners
    def disconnect(self, sender, receiver):
        """
        Disconnect an output port of one component from an input port of
        another.

        Parameters
        ----------
        sender : Union[``rill.engine.inputport.InputPort``, str]
        receiver : Union[``rill.engine.outputport.OutputPort``, str]
        """
        outport = self.get_component_port(sender, kind='out')
        inport = self.get_component_port(receiver, kind='in')
        outport._connection = None
        if outport._connections:
            outport._connections.remove(inport._connection)

        inport._connection.outports.remove(outport)
        if not inport._connection.outports:
            inport._connection = None
        else:
            inport._connection.metadata.pop(outport)

        self.disconnect.event.emit(outport, inport)

    @supports_listeners
    def set_edge_metadata(self, sender, receiver, metadata):
        """
        Set metadata on edge

        Parameters
        ----------
        sender : Union[``rill.engine.inputport.InputPort``, str]
        receiver : Union[``rill.engine.outputport.OutputPort``, str]
        metadata: dict
        """
        outport = self.get_component_port(sender, kind='out')
        inport = self.get_component_port(receiver, kind='in')
        edge_metadata = inport._connection.metadata.setdefault(outport, {})

        merge_metadata(metadata, edge_metadata)
        self.set_edge_metadata.event.emit(outport, inport, metadata)
        return edge_metadata

    # FIXME: reverse the order of these args
    @supports_listeners
    def initialize(self, content, receiver):
        """
        Initialize an inport port with a value

        Parameters
        ----------
        content : Any
        receiver : Union[``rill.engine.inputport.InputPort``, str]
        """
        inport = self.get_component_port(receiver, kind='in')
        inport.initialize(content)
        self.initialize.event.emit(inport, content)

    @supports_listeners
    def uninitialize(self, receiver):
        """
        Remove the initialized value of an inport port.

        Parameters
        ----------
        receiver : Union[``rill.engine.inputport.InputPort``, str]

        Returns
        -------
        ``rill.engine.inputport.InitializationConnection``
            The removed initialization connection
        """
        inport = self.get_component_port(receiver, kind='in')
        result = inport.uninitialize()
        self.uninitialize.event.emit(inport)
        return result

    def validate(self):
        """
        Validate the graph.
        """
        errors = []
        for component in self._components.values():
            for port in component.ports:
                try:
                    port.validate()
                except FlowError as e:
                    errors.append(str(e))
        if errors:
            for error in errors:
                logger.error(error)
            raise FlowError("Errors opening ports")

    # Serialization --

    def to_dict(self):
        """
        Serialize graph to dictionary

        Returns
        -------
        definition : dict
            json-serializable representation of graph according to fbp json
            standard
        """
        from rill.events.listeners.memory import fbp_iip, fbp_edge

        definition = {
            'processes': {},
            'connections': [],
            'inports': {},
            'outports': {}
        }
        for (comp_name, component) in self.get_components().items():
            if component.hidden:
                continue

            definition['processes'][comp_name] = {
                "component": component.get_type(),
                "metadata": component.metadata,
            }

            for inport in component.inports:
                if not inport.is_connected():
                    continue

                conn = inport._connection
                if isinstance(conn, InitializationConnection):
                    definition['connections'].append(
                        fbp_iip(inport, self.name, 'process'))
                else:
                    for outport in conn.outports:
                        if outport.component.hidden:
                            continue
                        definition['connections'].append(
                            fbp_edge(outport, inport, self.name,
                                     self.default_capacity, 'process'))

        for (name, inport) in self.inports.items():
            definition['inports'][name] = {
                'process': inport.component.get_name(),
                'port': inport.name,
                'metadata': self.inport_metadata.get(name, {})
            }

        for (name, outport) in self.outports.items():
            definition['outports'][name] = {
                'process': outport.component.get_name(),
                'port': outport.name,
                'metadata': self.outport_metadata.get(name, {})
            }

        return definition

    @classmethod
    def from_dict(cls, definition, component_lookup=None):
        """
        Create network from serialized definition

        Parameters
        ----------
        definition : dict
            network structure according to fbp json standard
        component_lookup : Dict[str, Type[``rill.enginge.component.Component``]]
            maps of component name to component class

        Returns
        -------
        ``rill.enginge.network.Graph``
        """
        from rill.utils import locate_class

        def _port(p, kind):
            return graph.get_component_port((p['process'], p['port']),
                                            index=p.get('index', None),
                                            kind=kind)

        graph = cls()
        for (name, spec) in definition['processes'].items():
            if component_lookup:
                comp_class = component_lookup[spec['component']]
            else:
                comp_class = locate_class(spec['component'].replace('/', '.'))
            component = graph.add_component(name, comp_class)
            if spec.get('metadata'):
                component.metadata.update(spec['metadata'])

        for connection in definition['connections']:
            tgt = _port(connection['tgt'], 'in')
            if 'data' in connection['src']:
                # static initializer
                data = connection['src']['data']
                content = deserialize(data)
                if not tgt.auto_receive:
                    content = Stream(content)
                # content = tgt.type.to_native(content)
                graph.initialize(content, tgt)
            else:
                # connection
                src = _port(connection['src'], 'out')
                if 'cap' in connection.keys():
                    graph.connect(src,
                                  tgt,
                                  connection_capacity=connection['cap'])
                else:
                    graph.connect(src, tgt)

        for (name, inport) in definition['inports'].items():
            graph.export(_port(inport, 'in'), name)

        for (name, outport) in definition['outports'].items():
            graph.export(_port(outport, 'out'), name)

        return graph
Beispiel #37
0
    def build_update(self):
        """
    Draw sample from proposal conditional on last sample. Then accept
    or reject the sample based on the ratio,

    .. math::
      \\text{ratio} = \log p(x, z^{new}) - \log p(x, z^{old}) +
        \log g(z^{new} \mid z^{old}) - \log g(z^{old} \mid z^{new})

    Notes
    -----
    The updates assume each Empirical random variable is directly
    parameterized by tf.Variables().
    """
        old_sample = {
            z: tf.gather(qz.params, tf.maximum(self.t - 1, 0))
            for z, qz in six.iteritems(self.latent_vars)
        }
        old_sample = OrderedDict(old_sample)

        # Form dictionary in order to replace conditioning on prior or
        # observed variable with conditioning on a specific value.
        dict_swap = {}
        for x, qx in six.iteritems(self.data):
            if isinstance(x, RandomVariable):
                if isinstance(qx, RandomVariable):
                    qx_copy = copy(qx, scope='conditional')
                    dict_swap[x] = qx_copy.value()
                else:
                    dict_swap[x] = qx

        dict_swap_old = dict_swap.copy()
        dict_swap_old.update(old_sample)
        scope_old = 'inference_' + str(id(self)) + '/old'
        scope_new = 'inference_' + str(id(self)) + '/new'

        # Draw proposed sample and calculate acceptance ratio.
        new_sample = old_sample.copy()  # copy to ensure same order
        ratio = 0.0
        for z, proposal_z in six.iteritems(self.proposal_vars):
            # Build proposal g(znew | zold).
            proposal_znew = copy(proposal_z, dict_swap_old, scope=scope_old)
            # Sample znew ~ g(znew | zold).
            new_sample[z] = proposal_znew.value()
            # Increment ratio.
            ratio += tf.reduce_sum(proposal_znew.log_prob(new_sample[z]))

        dict_swap_new = dict_swap.copy()
        dict_swap_new.update(new_sample)

        for z, proposal_z in six.iteritems(self.proposal_vars):
            # Build proposal g(zold | znew).
            proposal_zold = copy(proposal_z, dict_swap_new, scope=scope_new)
            # Increment ratio.
            ratio -= tf.reduce_sum(proposal_zold.log_prob(dict_swap_old[z]))

        if self.model_wrapper is None:
            for z in six.iterkeys(self.latent_vars):
                # Build priors p(znew) and p(zold).
                znew = copy(z, dict_swap_new, scope=scope_new)
                zold = copy(z, dict_swap_old, scope=scope_old)
                # Increment ratio.
                ratio += tf.reduce_sum(znew.log_prob(dict_swap_new[z]))
                ratio -= tf.reduce_sum(zold.log_prob(dict_swap_old[z]))

            for x in six.iterkeys(self.data):
                if isinstance(x, RandomVariable):
                    # Build likelihoods p(x | znew) and p(x | zold).
                    x_znew = copy(x, dict_swap_new, scope=scope_new)
                    x_zold = copy(x, dict_swap_old, scope=scope_old)
                    # Increment ratio.
                    ratio += tf.reduce_sum(x_znew.log_prob(dict_swap[x]))
                    ratio -= tf.reduce_sum(x_zold.log_prob(dict_swap[x]))
        else:
            x = self.data
            ratio += self.model_wrapper.log_prob(x, new_sample)
            ratio -= self.model_wrapper.log_prob(x, old_sample)

        # Accept or reject sample.
        u = Uniform().sample()
        accept = tf.log(u) < ratio
        sample_values = tf.cond(accept,
                                lambda: list(six.itervalues(new_sample)),
                                lambda: list(six.itervalues(old_sample)))
        if not isinstance(sample_values, list):
            # ``tf.cond`` returns tf.Tensor if output is a list of size 1.
            sample_values = [sample_values]

        sample = {
            z: sample_value
            for z, sample_value in zip(six.iterkeys(new_sample), sample_values)
        }

        # Update Empirical random variables.
        assign_ops = []
        for z, qz in six.iteritems(self.latent_vars):
            variable = qz.get_variables()[0]
            assign_ops.append(tf.scatter_update(variable, self.t, sample[z]))

        # Increment n_accept (if accepted).
        assign_ops.append(self.n_accept.assign_add(tf.where(accept, 1, 0)))
        return tf.group(*assign_ops)
Beispiel #38
0
class CommandInfo:
    """One command which is provided by one bundy module, it has zero
       or more parameters
    """
    def __init__(self, name, desc=""):
        self.name = name
        self.desc = desc
        self.params = OrderedDict()
        # Set default parameter "help"
        self.add_param(
            ParamInfo("help", desc="Get help for command.", optional=True))

    def __str__(self):
        return str("%s \t(%s)" % (self.name, self.desc))

    def get_name(self):
        return self.name

    def get_desc(self):
        return self.desc

    def add_param(self, paraminfo):
        """Add a ParamInfo object to this CommandInfo"""
        self.params[paraminfo.name] = paraminfo

    def has_param_with_name(self, param_name):
        """Returns true if the parameter with param_name exists"""
        return param_name in self.params

    def get_param_with_name(self, param_name):
        """Returns the ParamInfo with the given name. Raises a
           KeyError if it doesn't exist"""
        return self.params[param_name]

    def get_params(self):
        """Returns a list of all ParamInfo objects for this CommandInfo"""
        return list(self.params.values())

    def get_param_names(self):
        """Returns a list of the names of all parameters for this command"""
        return list(self.params.keys())

    def get_mandatory_param_names(self):
        """Returns a list of the names of all mandatory parameters for
           this command"""
        all_names = self.params.keys()
        return [
            name for name in all_names if not self.params[name].is_optional
        ]

    def get_param_name_by_position(self, pos, param_count):
        """
        Find a proper parameter name for the position 'pos':
        If param_count is equal to the count of mandatory parameters of command,
        and there is some optional parameter, find the first mandatory parameter 
        from the position 'pos' to the end. Else, return the name on position pos.
        (This function will be changed if bundyctl command line syntax is changed
        in the future. )
        """
        if type(pos) != int:
            raise KeyError(str(pos) + " is not an integer")

        else:
            params = self.params.copy()
            del params['help']
            count = len(params)
            if (pos >= count):
                raise KeyError(str(pos) + " out of range")

            mandatory_count = len(self.get_mandatory_param_names())
            param_names = list(params.keys())
            if (param_count == mandatory_count) and (param_count < count):
                while pos < count:
                    if not params[param_names[pos]].is_optional:
                        return param_names[pos]
                    pos += 1

                raise KeyError(str(pos) + "parameters have error")
            else:
                return param_names[pos]

    def command_help(self):
        """Prints the help info for this command to stdout"""
        print("Command ", self)
        print("\t\thelp (Get help for command)")

        params = self.params.copy()
        del params["help"]

        if len(params) == 0:
            print("This command has no parameters")
            return

        print("Parameters:")
        for info in params.values():
            print("    %s" % info.get_basic_info())
            description = info.get_desc()
            if description != "":
                print(
                    textwrap.fill(description,
                                  initial_indent="        ",
                                  subsequent_indent="        ",
                                  width=70))
Beispiel #39
0
class SGANNode(object):
    """
    docstring
    """

    def __init__(self):
        model_path = rospy.get_param("model_path")
        model_name = rospy.get_param("model_name")
        model = os.path.join(model_path, model_name + "_model.pt")
        checkpoint = torch.load(model)
        self.num_samples = rospy.get_param("num_samples", 20)
        self.seq_len = rospy.get_param("seq_len", 8)
        self.visualise = rospy.get_param("visualise", True)
        self.args_ = AttrDict(checkpoint['args'])
        self.generator = self.get_generator(checkpoint)
        self.tracked_persons_sub = rospy.Subscriber(
            "/spencer/perception/tracked_persons", TrackedPersons, self.tracked_persons_cb, queue_size=3)
        # self.predicted_tracks_pub = rospy.Publisher("/sgan/predictions", TBD, queue_size=1)
        self.predictions_marker_pub = rospy.Publisher("/sgan/predictions_marker", MarkerArray, queue_size=1)
        self.tracked_persons = {}

    def get_generator(self, checkpoint):
        generator = TrajectoryGenerator(
            obs_len=self.args_.obs_len,
            pred_len=self.args_.pred_len,
            embedding_dim=self.args_.embedding_dim,
            encoder_h_dim=self.args_.encoder_h_dim_g,
            decoder_h_dim=self.args_.decoder_h_dim_g,
            mlp_dim=self.args_.mlp_dim,
            num_layers=self.args_.num_layers,
            noise_dim=self.args_.noise_dim,
            noise_type=self.args_.noise_type,
            noise_mix_type=self.args_.noise_mix_type,
            pooling_type=self.args_.pooling_type,
            pool_every_timestep=self.args_.pool_every_timestep,
            dropout=self.args_.dropout,
            bottleneck_dim=self.args_.bottleneck_dim,
            neighborhood_size=self.args_.neighborhood_size,
            grid_size=self.args_.grid_size,
            batch_norm=self.args_.batch_norm,
            use_gpu=self.args_.use_gpu)
        generator.load_state_dict(checkpoint['g_state'])
        if self.args_.use_gpu:
            generator.cuda()
        generator.train()
        return generator

    def tracked_persons_cb(self, msg):
        """
        docstring
        """
        for track in msg.tracks:
            if track.twist.twist.linear.x > 0 or track.twist.twist.linear.y > 0:
                if track.track_id in self.tracked_persons:
                    self.tracked_persons[track.track_id].append(track)
                else:
                    self.tracked_persons[track.track_id] = deque([track], maxlen=8)

        current_ids = [track.track_id for track in msg.tracks]
        msg_arr = MarkerArray()
        for k in self.tracked_persons.copy().keys():
            if k not in current_ids:
                if self.visualise:
                    msg = Marker()
                    msg.action = msg.DELETEALL
                    msg.ns = "track_" + str(k)
                    msg_arr.markers.append(copy(msg))

                del self.tracked_persons[k]

        if self.visualise:
            self.predictions_marker_pub.publish(msg_arr)

        if type(self.tracked_persons) is dict:
            self.tracked_persons = OrderedDict(
                sorted(self.tracked_persons.items(), key=lambda t: t[0]))

    def predict_tracks(self):
        """
        docstring
        """
        curr_seq = np.zeros((self.seq_len, len(self.tracked_persons), 2))
        curr_seq_rel = np.zeros(curr_seq.shape)
        valid_tracks = 0

        with torch.no_grad():
            # copy is needed, because the subscriber callback can mutate the dict
            for id, tracks in self.tracked_persons.copy().items():
                if len(tracks) != self.seq_len:
                    continue
                obs_traj = np.array([[track.pose.pose.position.x, track.pose.pose.position.y] for track in tracks])
                obs_traj_rel = np.zeros(obs_traj.shape)
                obs_traj_rel[1:, :] = obs_traj[1:, :] - obs_traj[:-1, :]
                curr_seq[:, valid_tracks, :] = obs_traj
                curr_seq_rel[:, valid_tracks, :] = obs_traj_rel
                valid_tracks = valid_tracks + 1

            if valid_tracks > 0:
                curr_seq = torch.from_numpy(curr_seq[:, :valid_tracks, :]).float()
                curr_seq_rel = torch.from_numpy(curr_seq_rel[:, :valid_tracks, :]).float()
                if self.args_.use_gpu:
                    curr_seq = curr_seq.cuda()
                    curr_seq_rel = curr_seq_rel.cuda()
                l = [valid_tracks]
                cum_start_idx = [0] + np.cumsum(l).tolist()
                seq_start_end = [
                    (start, end)
                    for start, end in zip(cum_start_idx, cum_start_idx[1:])
                ]
                seq_start_end = torch.Tensor(seq_start_end).type(torch.int)
                if self.args_.use_gpu:
                    seq_start_end = seq_start_end.cuda()
                pred_samples = dict.fromkeys(self.tracked_persons.keys(), [])
                track_ids = list(self.tracked_persons.keys())

                for _ in range(self.num_samples):
                    pred_traj_fake_rel = self.generator(curr_seq, curr_seq_rel, seq_start_end)
                    pred_traj_fake = relative_to_abs(pred_traj_fake_rel, curr_seq[-1])
                    for ped_id, pred in enumerate(pred_traj_fake.transpose(1, 0).tolist()):
                        pred_samples[track_ids[ped_id]].extend(
                            [Point(pred[idx][0], pred[idx][1], 0) for idx in range(0, self.seq_len)])

                if self.visualise:
                    self.visualise_predictions(pred_samples)

    def visualise_predictions(self, predictions_samples):
        msg = Marker()
        msg.header.frame_id = "odom"
        msg.ns = "tracks"
        msg.id = 0
        msg.action = msg.ADD
        msg.pose.orientation.w = 1.0
        msg.type = msg.LINE_LIST
        msg.points = []
        msg.scale.x = 0.05
        msg.color.a = 1.
        msg.color.r = 1.
        msg.color.g = 0.
        msg.color.b = 0.

        msg_arr = MarkerArray()

        for k, v in predictions_samples.items():
            msg.points = []
            msg.points.extend(v)
            msg.ns = "track_" + str(k)
            msg_arr.markers.append(copy(msg))

        self.predictions_marker_pub.publish(msg_arr)

    def spin(self):
        """
        docstring
        """
        rate = rospy.Rate(5)
        while not rospy.is_shutdown():
            if self.tracked_persons:
                self.predict_tracks()
            rate.sleep()
Beispiel #40
0
class Workspace:
    def __init__(self, workspace_load_target):
        from .cli_target import WorkspaceLoadTarget

        self._stack = ExitStack()

        self._workspace_load_target = check.opt_inst_param(
            workspace_load_target, "workspace_load_target", WorkspaceLoadTarget
        )
        self._grpc_server_registry = self._stack.enter_context(
            ProcessGrpcServerRegistry(reload_interval=0, heartbeat_ttl=30)
        )
        self._load_workspace()

    def _load_workspace(self):
        repository_location_origins = (
            self._workspace_load_target.create_origins() if self._workspace_load_target else []
        )

        self._location_origin_dict = OrderedDict()
        check.list_param(
            repository_location_origins,
            "repository_location_origins",
            of_type=RepositoryLocationOrigin,
        )

        self._location_dict = {}
        self._location_error_dict = {}
        for origin in repository_location_origins:
            check.invariant(
                self._location_origin_dict.get(origin.location_name) is None,
                'Cannot have multiple locations with the same name, got multiple "{name}"'.format(
                    name=origin.location_name,
                ),
            )

            self._location_origin_dict[origin.location_name] = origin
            self._load_location(origin.location_name)

    # Can be overidden in subclasses that need different logic for loading repository
    # locations from origins
    def create_location_from_origin(self, origin):
        if not self._grpc_server_registry.supports_origin(origin):  # pylint: disable=no-member
            return origin.create_location()
        else:
            endpoint = self._grpc_server_registry.reload_grpc_endpoint(  # pylint: disable=no-member
                origin
            )
            return GrpcServerRepositoryLocation(
                origin=origin,
                server_id=endpoint.server_id,
                port=endpoint.port,
                socket=endpoint.socket,
                host=endpoint.host,
                heartbeat=True,
                watch_server=False,
                grpc_server_registry=self._grpc_server_registry,
            )

    def _load_location(self, location_name):
        if self._location_dict.get(location_name):
            del self._location_dict[location_name]

        if self._location_error_dict.get(location_name):
            del self._location_error_dict[location_name]

        origin = self._location_origin_dict[location_name]
        try:
            location = self.create_location_from_origin(origin)
            self._location_dict[location_name] = location
        except Exception:  # pylint: disable=broad-except
            error_info = serializable_error_info_from_exc_info(sys.exc_info())
            self._location_error_dict[location_name] = error_info
            warnings.warn(
                "Error loading repository location {location_name}:{error_string}".format(
                    location_name=location_name, error_string=error_info.to_string()
                )
            )

    def create_snapshot(self):
        return WorkspaceSnapshot(
            self._location_origin_dict.copy(), self._location_error_dict.copy()
        )

    @property
    def repository_locations(self):
        return list(self._location_dict.values())

    def has_repository_location(self, location_name):
        check.str_param(location_name, "location_name")
        return location_name in self._location_dict

    def get_repository_location(self, location_name):
        check.str_param(location_name, "location_name")
        return self._location_dict[location_name]

    def has_repository_location_error(self, location_name):
        check.str_param(location_name, "location_name")
        return location_name in self._location_error_dict

    def reload_repository_location(self, location_name):
        self._load_location(location_name)

    def reload_workspace(self):
        for location in self.repository_locations:
            location.cleanup()
        self._load_workspace()

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, traceback):
        for location in self.repository_locations:
            location.cleanup()
        self._stack.close()
Beispiel #41
0
def _eqcalculate(dbf,
                 comps,
                 phases,
                 conditions,
                 output,
                 data=None,
                 per_phase=False,
                 callables=None,
                 parameters=None,
                 **kwargs):
    """
    WARNING: API/calling convention not finalized.
    Compute the *equilibrium value* of a property.
    This function differs from `calculate` in that it computes
    thermodynamic equilibrium instead of randomly sampling the
    internal degrees of freedom of a phase.
    Because of that, it's slower than `calculate`.
    This plugs in the equilibrium phase and site fractions
    to compute a thermodynamic property defined in a Model.

    Parameters
    ----------
    dbf : Database
        Thermodynamic database containing the relevant parameters.
    comps : list
        Names of components to consider in the calculation.
    phases : list or dict
        Names of phases to consider in the calculation.
    conditions : dict or (list of dict)
        StateVariables and their corresponding value.
    output : str
        Equilibrium model property (e.g., CPM, HM, etc.) to compute.
        This must be defined as an attribute in the Model class of each phase.
    data : Dataset, optional
        Previous result of call to `equilibrium`.
        Should contain the equilibrium configurations at the conditions of interest.
        If the databases are not the same as in the original calculation,
        the results may be meaningless. If None, `equilibrium` will be called.
        Specifying this keyword argument can save the user some time if several properties
        need to be calculated in succession.
    per_phase : bool, optional
        If True, compute and return the property for each phase present.
        If False, return the total system value, weighted by the phase fractions.
    parameters : dict, optional
        Maps SymPy Symbol to numbers, for overriding the values of parameters in the Database.
    callables : dict
        Callable functions to compute 'output' for each phase.
    kwargs
        Passed to `calculate`.

    Returns
    -------
    Dataset of property as a function of equilibrium conditions
    """
    if data is None:
        data = equilibrium(dbf, comps, phases, conditions)
    active_phases = unpack_phases(phases) or sorted(dbf.phases.keys())
    conds = _adjust_conditions(conditions)
    indep_vars = ['N', 'P', 'T']
    # TODO: Rewrite this to use the coord dict from 'data'
    str_conds = OrderedDict((str(key), value) for key, value in conds.items())
    indep_vals = list([float(x) for x in np.atleast_1d(val)]
                      for key, val in str_conds.items() if key in indep_vars)
    coord_dict = str_conds.copy()
    components = [x for x in sorted(comps)]
    desired_active_pure_elements = [
        list(x.constituents.keys()) for x in components
    ]
    desired_active_pure_elements = [
        el.upper() for constituents in desired_active_pure_elements
        for el in constituents
    ]
    pure_elements = sorted(
        set([x for x in desired_active_pure_elements if x != 'VA']))
    coord_dict['vertex'] = np.arange(
        len(pure_elements) + 1
    )  # +1 is to accommodate the degenerate degree of freedom at the invariant reactions
    grid_shape = np.meshgrid(*coord_dict.values(), indexing='ij',
                             sparse=False)[0].shape
    prop_shape = grid_shape
    prop_dims = list(str_conds.keys()) + ['vertex']

    result = Dataset({output: (prop_dims, np.full(prop_shape, np.nan))},
                     coords=coord_dict)
    # For each phase select all conditions where that phase exists
    # Perform the appropriate calculation and then write the result back
    for phase in active_phases:
        dof = sum([len(x) for x in dbf.phases[phase].constituents])
        current_phase_indices = (data.Phase.values == phase)
        if ~np.any(current_phase_indices):
            continue
        points = data.Y.values[np.nonzero(current_phase_indices)][..., :dof]
        statevar_indices = np.nonzero(current_phase_indices)[:len(indep_vals)]
        statevars = {
            key: np.take(np.asarray(vals), idx)
            for key, vals, idx in zip(indep_vars, indep_vals, statevar_indices)
        }
        statevars.update(kwargs)
        if statevars.get('mode', None) is None:
            statevars['mode'] = 'numpy'
        calcres = calculate(dbf,
                            comps, [phase],
                            output=output,
                            points=points,
                            broadcast=False,
                            callables=callables,
                            parameters=parameters,
                            **statevars)
        result[output].values[np.nonzero(
            current_phase_indices)] = calcres[output].values
    if not per_phase:
        result[output] = (result[output] * data['NP']).sum(dim='vertex',
                                                           skipna=True)
    else:
        result['Phase'] = data['Phase'].copy()
        result['NP'] = data['NP'].copy()
    return result
Beispiel #42
0
def equilibrium(dbf,
                comps,
                phases,
                conditions,
                output=None,
                model=None,
                verbose=False,
                broadcast=True,
                calc_opts=None,
                scheduler='sync',
                parameters=None,
                solver=None,
                callables=None,
                **kwargs):
    """
    Calculate the equilibrium state of a system containing the specified
    components and phases, under the specified conditions.

    Parameters
    ----------
    dbf : Database
        Thermodynamic database containing the relevant parameters.
    comps : list
        Names of components to consider in the calculation.
    phases : list or dict
        Names of phases to consider in the calculation.
    conditions : dict or (list of dict)
        StateVariables and their corresponding value.
    output : str or list of str, optional
        Additional equilibrium model properties (e.g., CPM, HM, etc.) to compute.
        These must be defined as attributes in the Model class of each phase.
    model : Model, a dict of phase names to Model, or a seq of both, optional
        Model class to use for each phase.
    verbose : bool, optional
        Print details of calculations. Useful for debugging.
    broadcast : bool
        If True, broadcast conditions against each other. This will compute all combinations.
        If False, each condition should be an equal-length list (or single-valued).
        Disabling broadcasting is useful for calculating equilibrium at selected conditions,
        when those conditions don't comprise a grid.
    calc_opts : dict, optional
        Keyword arguments to pass to `calculate`, the energy/property calculation routine.
    scheduler : Dask scheduler, optional
        Job scheduler for performing the computation.
        If None, return a Dask graph of the computation instead of actually doing it.
    parameters : dict, optional
        Maps SymPy Symbol to numbers, for overriding the values of parameters in the Database.
    solver : pycalphad.core.solver.SolverBase
        Instance of a solver that is used to calculate local equilibria.
        Defaults to a pycalphad.core.solver.InteriorPointSolver.
    callables : dict, optional
        Pre-computed callable functions for equilibrium calculation.

    Returns
    -------
    Structured equilibrium calculation, or Dask graph if scheduler=None.

    Examples
    --------
    None yet.
    """
    if not broadcast:
        raise NotImplementedError('Broadcasting cannot yet be disabled')
    comps = sorted(unpack_components(dbf, comps))
    phases = unpack_phases(phases) or sorted(dbf.phases.keys())
    # remove phases that cannot be active
    list_of_possible_phases = filter_phases(dbf, comps)
    active_phases = sorted(
        set(list_of_possible_phases).intersection(set(phases)))
    if len(list_of_possible_phases) == 0:
        raise ConditionError(
            'There are no phases in the Database that can be active with components {0}'
            .format(comps))
    if len(active_phases) == 0:
        raise ConditionError(
            'None of the passed phases ({0}) are active. List of possible phases: {1}.'
            .format(phases, list_of_possible_phases))
    if isinstance(comps, (str, v.Species)):
        comps = [comps]
    if len(set(comps) - set(dbf.species)) > 0:
        raise EquilibriumError('Components not found in database: {}'.format(
            ','.join([c.name for c in (set(comps) - set(dbf.species))])))
    calc_opts = calc_opts if calc_opts is not None else dict()
    solver = solver if solver is not None else InteriorPointSolver(
        verbose=verbose)
    parameters = parameters if parameters is not None else dict()
    if isinstance(parameters, dict):
        parameters = OrderedDict(sorted(parameters.items(), key=str))
    models = instantiate_models(dbf,
                                comps,
                                active_phases,
                                model=model,
                                parameters=parameters)
    # Temporary solution until constraint system improves
    if conditions.get(v.N) is None:
        conditions[v.N] = 1
    if np.any(np.array(conditions[v.N]) != 1):
        raise ConditionError('N!=1 is not yet supported, got N={}'.format(
            conditions[v.N]))
    # Modify conditions values to be within numerical limits, e.g., X(AL)=0
    # Also wrap single-valued conditions with lists
    conds = _adjust_conditions(conditions)

    for cond in conds.keys():
        if isinstance(cond,
                      (v.Composition,
                       v.ChemicalPotential)) and cond.species not in comps:
            raise ConditionError(
                '{} refers to non-existent component'.format(cond))
    state_variables = sorted(get_state_variables(models=models, conds=conds),
                             key=str)
    str_conds = OrderedDict((str(key), value) for key, value in conds.items())
    num_calcs = np.prod([len(i) for i in str_conds.values()])
    components = [x for x in sorted(comps)]
    desired_active_pure_elements = [
        list(x.constituents.keys()) for x in components
    ]
    desired_active_pure_elements = [
        el.upper() for constituents in desired_active_pure_elements
        for el in constituents
    ]
    pure_elements = sorted(
        set([x for x in desired_active_pure_elements if x != 'VA']))
    if verbose:
        print('Components:', ' '.join([str(x) for x in comps]))
        print('Phases:', end=' ')
    output = output if output is not None else 'GM'
    output = output if isinstance(output, (list, tuple, set)) else [output]
    output = set(output)
    output |= {'GM'}
    output = sorted(output)
    need_hessians = any(
        type(c) in v.CONDITIONS_REQUIRING_HESSIANS for c in conds.keys())
    phase_records = build_phase_records(dbf,
                                        comps,
                                        active_phases,
                                        conds,
                                        models,
                                        output='GM',
                                        callables=callables,
                                        parameters=parameters,
                                        verbose=verbose,
                                        build_gradients=True,
                                        build_hessians=need_hessians)
    if verbose:
        print('[done]', end='\n')

    # 'calculate' accepts conditions through its keyword arguments
    grid_opts = calc_opts.copy()
    statevar_strings = [str(x) for x in state_variables]
    grid_opts.update({
        key: value
        for key, value in str_conds.items() if key in statevar_strings
    })
    if 'pdens' not in grid_opts:
        grid_opts['pdens'] = 500
    grid = delayed(calculate, pure=False)(dbf,
                                          comps,
                                          active_phases,
                                          model=models,
                                          fake_points=True,
                                          callables=callables,
                                          output='GM',
                                          parameters=parameters,
                                          **grid_opts)
    coord_dict = str_conds.copy()
    coord_dict['vertex'] = np.arange(
        len(pure_elements) + 1
    )  # +1 is to accommodate the degenerate degree of freedom at the invariant reactions
    coord_dict['component'] = pure_elements
    grid_shape = tuple(len(x)
                       for x in conds.values()) + (len(pure_elements) + 1, )
    properties = delayed(starting_point, pure=False)(conds, state_variables,
                                                     phase_records, grid)
    conditions_per_chunk_per_axis = 2
    if num_calcs > 1:
        # Generate slices of 'properties'
        slices = []
        for val in grid_shape[:-1]:
            idx_arr = list(range(val))
            num_chunks = int(np.floor(val / conditions_per_chunk_per_axis))
            if num_chunks > 0:
                cond_slices = [
                    x for x in np.array_split(np.asarray(idx_arr), num_chunks)
                    if len(x) > 0
                ]
            else:
                cond_slices = [idx_arr]
            slices.append(cond_slices)
        chunk_dims = [len(slc) for slc in slices]
        chunk_grid = np.array(
            np.unravel_index(np.arange(np.prod(chunk_dims)), chunk_dims)).T
        res = []
        for chunk in chunk_grid:
            prop_slice = properties[OrderedDict(
                list(
                    zip(str_conds.keys(), [
                        np.atleast_1d(sl)[ch] for ch, sl in zip(chunk, slices)
                    ])))]
            job = delayed(_solve_eq_at_conditions,
                          pure=False)(comps,
                                      prop_slice,
                                      phase_records,
                                      grid,
                                      list(str_conds.keys()),
                                      state_variables,
                                      verbose,
                                      solver=solver)
            res.append(job)
        properties = delayed(_merge_property_slices,
                             pure=False)(properties, chunk_grid, slices,
                                         list(str_conds.keys()), res)
    else:
        # Single-process job; don't create child processes
        properties = delayed(_solve_eq_at_conditions,
                             pure=False)(comps,
                                         properties,
                                         phase_records,
                                         grid,
                                         list(str_conds.keys()),
                                         state_variables,
                                         verbose,
                                         solver=solver)

    # Compute equilibrium values of any additional user-specified properties
    # We already computed these properties so don't recompute them
    output = sorted(set(output) - {'GM', 'MU'})
    for out in output:
        if (out is None) or (len(out) == 0):
            continue
        # TODO: How do we know if a specified property should be per_phase or not?
        # For now, we make a best guess
        if (out == 'degree_of_ordering') or (out == 'DOO'):
            per_phase = True
        else:
            per_phase = False
        eqcal = delayed(_eqcalculate, pure=False)(dbf,
                                                  comps,
                                                  active_phases,
                                                  conditions,
                                                  out,
                                                  data=properties,
                                                  per_phase=per_phase,
                                                  callables=callables,
                                                  parameters=parameters,
                                                  model=models,
                                                  **calc_opts)
        properties = delayed(properties.merge, pure=False)(eqcal,
                                                           compat='equals')
    if scheduler is not None:
        properties = dask.compute(properties, scheduler=scheduler)[0]
    properties.attrs['created'] = datetime.utcnow().isoformat()
    if len(kwargs) > 0:
        warnings.warn(
            'The following equilibrium keyword arguments were passed, but unused:\n{}'
            .format(kwargs))
    return properties
Beispiel #43
0
class NamedObjectMap(object):

    def __init__(self, type):  # @ReservedAssignment
        super(NamedObjectMap, self).__init__()
        self.type = type
        self._data = OrderedDict()

        if not issubclass(type, NamedObject):
            raise TypeError("type must be a NamedObject")

    def clear(self):
        self._data = OrderedDict()

    def copy(self):
        copy = NamedObjectMap(self.type)
        copy._data = self._data.copy()

        return copy

    def __len__(self):
        return len(self._data)

    def __getitem__(self, key):
        if isinstance(key, six.integer_types):
            for i, (_, v) in enumerate(six.iteritems(self._data)):
                if i == key:
                    return v
            raise KeyError(key)
        else:
            return self._data[key]

    def __setitem__(self, key, value):
        if not isinstance(value, self.type):
            raise TypeError("can only add " + self.type.__name__ + " objects")

        if isinstance(key, six.integer_types):
            self._data = OrderedDict([(value.name, value) if i == key else (k, v)
                                      for i, (k, v) in enumerate(six.iteritems(self._data))])
        else:
            if value.name != key:
                raise ValueError(
                    "key does not match name of " + self.type.__name__)

            self._data[key] = value

    def __delitem__(self, key):
        del self._data[key]

    def __iter__(self):
        return iter(self._data.values())

    def __contains__(self, item):
        return item in self._data

    def extend(self, value):
        if isinstance(value, NamedObject):
            self._data[value.name] = value
        elif hasattr(value, "__iter__"):
            for item in value:
                self._data[item.name] = item
        else:
            raise TypeError("can only add " + str(type) + " objects")

    def __add__(self, value):
        self.extend(value)
        return self

    def __iadd__(self, value):
        self.extend(value)
        return self

    def keys(self):
        return self._data.keys()
Beispiel #44
0
class Query:
    """ Helper to conform OData filters """
    _mapping = {
        'from': 'from/emailAddress/address',
        'to': 'toRecipients/emailAddress/address',
        'start': 'start/DateTime',
        'end': 'end/DateTime',
        'flag': 'flag/flagStatus'
    }

    def __init__(self, attribute=None, *, protocol):
        """ Build a query to apply OData filters
        https://docs.microsoft.com/en-us/graph/query-parameters

        :param str attribute: attribute to apply the query for
        :param Protocol protocol: protocol to use for connecting
        """
        self.protocol = protocol() if isinstance(protocol, type) else protocol
        self._attribute = None
        self._chain = None
        self.new(attribute)
        self._negation = False
        self._filters = []  # store all the filters
        self._order_by = OrderedDict()
        self._selects = set()
        self._expands = set()
        self._search = None

    def __str__(self):
        return 'Filter: {}\nOrder: {}\nSelect: {}\nExpand: {}\nSearch: {}'.format(
            self.get_filters(), self.get_order(), self.get_selects(),
            self.get_expands(), self._search)

    def __repr__(self):
        return self.__str__()

    @fluent
    def select(self, *attributes):
        """ Adds the attribute to the $select parameter

        :param str attributes: the attributes tuple to select.
         If empty, the on_attribute previously set is added.
        :rtype: Query
        """
        if attributes:
            for attribute in attributes:
                attribute = self.protocol.convert_case(
                    attribute) if attribute and isinstance(attribute,
                                                           str) else None
                if attribute:
                    if '/' in attribute:
                        # only parent attribute can be selected
                        attribute = attribute.split('/')[0]
                    self._selects.add(attribute)
        else:
            if self._attribute:
                self._selects.add(self._attribute)

        return self

    @fluent
    def expand(self, *relationships):
        """ Adds the relationships (e.g. "event" or "attachments")
        that should be expanded with the $expand parameter
        Important: The ApiComponent using this should know how to handle this relationships.
            eg: Message knows how to handle attachments, and event (if it's an EventMessage).
        Important: When using expand on multi-value relationships a max of 20 items will be returned.
        :param str relationships: the relationships tuple to expand.
        :rtype: Query
        """

        for relationship in relationships:
            if relationship == 'event':
                relationship = '{}/event'.format(
                    self.protocol.get_service_keyword('event_message_type'))
            self._expands.add(relationship)

        return self

    @fluent
    def search(self, text):
        """
        Perform a search.
        Not from graph docs:
         You can currently search only message and person collections.
         A $search request returns up to 250 results.
         You cannot use $filter or $orderby in a search request.
        :param str text: the text to search
        :return: the Query instance
        """
        if text is None:
            self._search = None
        else:
            # filters an order are not allowed
            self.clear_filters()
            self.clear_order()
            self._search = '"{}"'.format(text)

        return self

    def as_params(self):
        """ Returns the filters, orders, select, expands and search as query parameters

        :rtype: dict
        """
        params = {}
        if self.has_filters:
            params['$filter'] = self.get_filters()
        if self.has_order:
            params['$orderby'] = self.get_order()
        if self.has_selects:
            params['$select'] = self.get_selects()
        if self.has_expands:
            params['$expand'] = self.get_expands()
        if self._search:
            params['$search'] = self._search
            params.pop('$filter', None)
            params.pop('$orderby', None)
        return params

    @property
    def has_filters(self):
        """ Whether the query has filters or not

        :rtype: bool
        """
        return bool(self._filters)

    @property
    def has_order(self):
        """ Whether the query has order_by or not

        :rtype: bool
        """
        return bool(self._order_by)

    @property
    def has_selects(self):
        """ Whether the query has select filters or not

        :rtype: bool
        """
        return bool(self._selects)

    @property
    def has_expands(self):
        """ Whether the query has relationships that should be expanded or not

         :rtype: bool
        """
        return bool(self._expands)

    def get_filters(self):
        """ Returns the result filters

        :rtype: str or None
        """
        if self._filters:
            filters_list = self._filters
            if isinstance(filters_list[-1], Enum):
                filters_list = filters_list[:-1]
            return ' '.join([
                fs.value if isinstance(fs, Enum) else fs[1]
                for fs in filters_list
            ]).strip()
        else:
            return None

    def get_order(self):
        """ Returns the result order by clauses

        :rtype: str or None
        """
        # first get the filtered attributes in order as they must appear
        # in the order_by first
        if not self.has_order:
            return None
        filter_order_clauses = OrderedDict([(filter_attr[0], None)
                                            for filter_attr in self._filters
                                            if isinstance(filter_attr, tuple)])

        # any order_by attribute that appears in the filters is ignored
        order_by_dict = self._order_by.copy()
        for filter_oc in filter_order_clauses.keys():
            direction = order_by_dict.pop(filter_oc, None)
            filter_order_clauses[filter_oc] = direction

        filter_order_clauses.update(
            order_by_dict)  # append any remaining order_by clause

        if filter_order_clauses:
            return ','.join([
                '{} {}'.format(attribute,
                               direction if direction else '').strip()
                for attribute, direction in filter_order_clauses.items()
            ])
        else:
            return None

    def get_selects(self):
        """ Returns the result select clause

        :rtype: str or None
        """
        if self._selects:
            return ','.join(self._selects)
        else:
            return None

    def get_expands(self):
        """ Returns the result expand clause

         :rtype: str or None
        """
        if self._expands:
            return ','.join(self._expands)
        else:
            return None

    def _get_mapping(self, attribute):
        if attribute:
            mapping = self._mapping.get(attribute)
            if mapping:
                attribute = '/'.join([
                    self.protocol.convert_case(step)
                    for step in mapping.split('/')
                ])
            else:
                attribute = self.protocol.convert_case(attribute)
            return attribute
        return None

    @fluent
    def new(self, attribute, operation=ChainOperator.AND):
        """ Combine with a new query

        :param str attribute: attribute of new query
        :param ChainOperator operation: operation to combine to new query
        :rtype: Query
        """
        if isinstance(operation, str):
            operation = ChainOperator(operation)
        self._chain = operation
        self._attribute = self._get_mapping(attribute) if attribute else None
        self._negation = False
        return self

    def clear_filters(self):
        """ Clear filters """
        self._filters = []

    def clear_order(self):
        """ Clears any order commands """
        self._order_by = OrderedDict()

    @fluent
    def clear(self):
        """ Clear everything

        :rtype: Query
        """
        self._filters = []
        self._order_by = OrderedDict()
        self._selects = set()
        self._negation = False
        self._attribute = None
        self._chain = None
        self._search = None

        return self

    @fluent
    def negate(self):
        """ Apply a not operator

        :rtype: Query
        """
        self._negation = not self._negation
        return self

    @fluent
    def chain(self, operation=ChainOperator.AND):
        """ Start a chain operation

        :param ChainOperator operation: how to combine with a new one
        :rtype: Query
        """
        if isinstance(operation, str):
            operation = ChainOperator(operation)
        self._chain = operation
        return self

    @fluent
    def on_attribute(self, attribute):
        """ Apply query on attribute, to be used along with chain()

        :param str attribute: attribute name
        :rtype: Query
        """
        self._attribute = self._get_mapping(attribute)
        return self

    def remove_filter(self, filter_attr):
        """ Removes a filter given the attribute name """
        filter_attr = self._get_mapping(filter_attr)
        new_filters = []
        remove_chain = False

        for flt in self._filters:
            if isinstance(flt, tuple):
                if flt[0] == filter_attr:
                    remove_chain = True
                else:
                    new_filters.append(flt)
            else:
                # this is a ChainOperator
                if remove_chain is False:
                    new_filters.append(flt)
                else:
                    remove_chain = False

        self._filters = new_filters

    def _add_filter(self, *filter_data):
        if self._attribute:
            if self._filters and not isinstance(self._filters[-1],
                                                ChainOperator):
                self._filters.append(self._chain)
            self._filters.append(
                (self._attribute, filter_data[0], filter_data[1]))
        else:
            raise ValueError(
                'Attribute property needed. call on_attribute(attribute) '
                'or new(attribute)')

    def _parse_filter_word(self, word):
        """ Converts the word parameter into the correct format """
        if isinstance(word, str):
            word = "'{}'".format(word)
        elif isinstance(word, dt.date):
            if isinstance(word, dt.datetime):
                if word.tzinfo is None:
                    # if it's a naive datetime, localize the datetime.
                    word = self.protocol.timezone.localize(
                        word)  # localize datetime into local tz
                if word.tzinfo != pytz.utc:
                    word = word.astimezone(
                        pytz.utc)  # transform local datetime to utc
            if '/' in self._attribute:
                # TODO: this is a fix for the case when the parameter
                #  filtered is a string instead a dateTimeOffset
                #  but checking the '/' is not correct, but it will
                #  differentiate for now the case on events:
                #  start/dateTime (date is a string here) from
                #  the case on other dates such as
                #  receivedDateTime (date is a dateTimeOffset)
                word = "'{}'".format(
                    word.isoformat())  # convert datetime to isoformat.
            else:
                word = "{}".format(
                    word.isoformat())  # convert datetime to isoformat
        elif isinstance(word, bool):
            word = str(word).lower()
        return word

    @staticmethod
    def _prepare_sentence(attribute, operation, word, negation=False):
        negation = 'not' if negation else ''
        attrs = (negation, attribute, operation, word)
        return '{} {} {} {}'.format(negation, attribute, operation,
                                    word).strip(), attrs

    @fluent
    def logical_operator(self, operation, word):
        """ Apply a logical operator

        :param str operation: how to combine with a new one
        :param word: other parameter for the operation
         (a = b) would be like a.logical_operator('eq', 'b')
        :rtype: Query
        """
        word = self._parse_filter_word(word)
        self._add_filter(*self._prepare_sentence(self._attribute, operation,
                                                 word, self._negation))
        return self

    @fluent
    def equals(self, word):
        """ Add a equals check

        :param str word: word to compare with
        :rtype: Query
        """
        return self.logical_operator('eq', word)

    @fluent
    def unequal(self, word):
        """ Add a unequals check

        :param str word: word to compare with
        :rtype: Query
        """
        return self.logical_operator('ne', word)

    @fluent
    def greater(self, word):
        """ Add a greater than check

        :param str word: word to compare with
        :rtype: Query
        """
        return self.logical_operator('gt', word)

    @fluent
    def greater_equal(self, word):
        """ Add a greater than or equal to check

        :param str word: word to compare with
        :rtype: Query
        """
        return self.logical_operator('ge', word)

    @fluent
    def less(self, word):
        """ Add a less than check

        :param str word: word to compare with
        :rtype: Query
        """
        return self.logical_operator('lt', word)

    @fluent
    def less_equal(self, word):
        """ Add a less than or equal to check

        :param str word: word to compare with
        :rtype: Query
        """
        return self.logical_operator('le', word)

    @staticmethod
    def _prepare_function(function_name, attribute, word, negation=False):
        negation = 'not' if negation else ''
        attrs = (negation, attribute, function_name, word)
        return "{} {}({}, {})".format(negation, function_name, attribute,
                                      word).strip(), attrs

    @fluent
    def function(self, function_name, word):
        """ Apply a function on given word

        :param str function_name: function to apply
        :param str word: word to apply function on
        :rtype: Query
        """
        word = self._parse_filter_word(word)

        self._add_filter(*self._prepare_function(
            function_name, self._attribute, word, self._negation))
        return self

    @fluent
    def contains(self, word):
        """ Adds a contains word check

        :param str word: word to check
        :rtype: Query
        """
        return self.function('contains', word)

    @fluent
    def startswith(self, word):
        """ Adds a startswith word check

        :param str word: word to check
        :rtype: Query
        """
        return self.function('startswith', word)

    @fluent
    def endswith(self, word):
        """ Adds a endswith word check

        :param str word: word to check
        :rtype: Query
        """
        return self.function('endswith', word)

    @fluent
    def iterable(self,
                 iterable_name,
                 *,
                 collection,
                 attribute,
                 word,
                 func=None,
                 operation=None):
        """ Performs a filter with the OData 'iterable_name' keyword
        on the collection

        For example:
        q.iterable('any', collection='email_addresses', attribute='address',
        operation='eq', word='*****@*****.**')

        will transform to a filter such as:
        emailAddresses/any(a:a/address eq '*****@*****.**')

        :param str iterable_name: the OData name of the iterable
        :param str collection: the collection to apply the any keyword on
        :param str attribute: the attribute of the collection to check
        :param str word: the word to check
        :param str func: the logical function to apply to the attribute inside
         the collection
        :param str operation: the logical operation to apply to the attribute
         inside the collection
        :rtype: Query
        """

        if func is None and operation is None:
            raise ValueError('Provide a function or an operation to apply')
        elif func is not None and operation is not None:
            raise ValueError(
                'Provide either a function or an operation but not both')

        current_att = self._attribute
        self._attribute = iterable_name

        word = self._parse_filter_word(word)
        collection = self._get_mapping(collection)
        attribute = self._get_mapping(attribute)

        if func is not None:
            sentence = self._prepare_function(func, attribute, word)
        else:
            sentence = self._prepare_sentence(attribute, operation, word)

        filter_str, attrs = sentence

        filter_data = '{}/{}(a:a/{})'.format(collection, iterable_name,
                                             filter_str), attrs
        self._add_filter(*filter_data)

        self._attribute = current_att

        return self

    @fluent
    def any(self, *, collection, attribute, word, func=None, operation=None):
        """ Performs a filter with the OData 'any' keyword on the collection

        For example:
        q.any(collection='email_addresses', attribute='address',
        operation='eq', word='*****@*****.**')

        will transform to a filter such as:

        emailAddresses/any(a:a/address eq '*****@*****.**')

        :param str collection: the collection to apply the any keyword on
        :param str attribute: the attribute of the collection to check
        :param str word: the word to check
        :param str func: the logical function to apply to the attribute
         inside the collection
        :param str operation: the logical operation to apply to the
         attribute inside the collection
        :rtype: Query
        """

        return self.iterable('any',
                             collection=collection,
                             attribute=attribute,
                             word=word,
                             func=func,
                             operation=operation)

    @fluent
    def all(self, *, collection, attribute, word, func=None, operation=None):
        """ Performs a filter with the OData 'all' keyword on the collection

        For example:
        q.any(collection='email_addresses', attribute='address',
        operation='eq', word='*****@*****.**')

        will transform to a filter such as:

        emailAddresses/all(a:a/address eq '*****@*****.**')

        :param str collection: the collection to apply the any keyword on
        :param str attribute: the attribute of the collection to check
        :param str word: the word to check
        :param str func: the logical function to apply to the attribute
         inside the collection
        :param str operation: the logical operation to apply to the
         attribute inside the collection
        :rtype: Query
        """

        return self.iterable('all',
                             collection=collection,
                             attribute=attribute,
                             word=word,
                             func=func,
                             operation=operation)

    @fluent
    def order_by(self, attribute=None, *, ascending=True):
        """ Applies a order_by clause

        :param str attribute: attribute to apply on
        :param bool ascending: should it apply ascending order or descending
        :rtype: Query
        """
        attribute = self._get_mapping(attribute) or self._attribute
        if attribute:
            self._order_by[attribute] = None if ascending else 'desc'
        else:
            raise ValueError(
                'Attribute property needed. call on_attribute(attribute) '
                'or new(attribute)')
        return self
class ConfiguredObject(object):
    """Superclass for classes that take configuration from YAML files"""

    # Roles of defined objects and the category they belong to
    # NB the double underscore is deliberate - attribute must be hidden from subclasses
    __content_roles = []

    # Procedure names - placeholder.
    # Will be replaced by a set in any subclasses that can contain procedures
    # Note that _procedure_names may *not* be set if it is already set in a superclass
    _procedure_names = None

    def __init__(self, name):

        self.name = name

        self._objects = OrderedDict((role, None) for role in self.all_roles)

    def _init(self):
        """Object initialisation - executed *before* loading contents"""
        pass

    def init(self):
        """Object initialisation - executed *after* loading contents"""
        pass

    def replace_object(self, role, new_object):
        """Replace already defined Object with a new one - for runtime use

        Args:
            role (text_str): Role name of contained Object
            new_object (Optional[ConfiguredObject]): New contained Object
        """
        if role in self._objects:
            self._objects[role] = new_object
        else:
            raise ValueError("Unknown contained Object role: %s" % role)

    # NB this function must be re-implemented in nested subclasses
    @property
    def all_roles(self):
        """Tuple of all content object roles, indefinition and loading order

        Returns:
            tuple[text_str, ...]
        """
        return tuple(self.__content_roles)

    @property
    def all_objects_by_role(self):
        """All contained Objects mapped by role (in specification order).

            Includes objects defined in subclasses.

        Returns:
            OrderedDict[text_str, ConfiguredObject]:

        """
        return self._objects.copy()

    @property
    def procedures(self):
        """Procedures attached to this object  mapped by name (in specification order).

        Returns:
            OrderedDict[text_str, ConfiguredObject]:

        """
        procedure_names = self.__class__._procedure_names
        result = OrderedDict()
        if procedure_names:
            for name in procedure_names:
                procedure = getattr(self, name)
                if procedure is not None:
                    result[name] = procedure

        return result
Beispiel #46
0
class RecalculateSubsectionGradeTest(ModuleStoreTestCase):
    """
    Ensures that the recalculate subsection grade task functions as expected when run.
    """
    def setUp(self):
        super(RecalculateSubsectionGradeTest, self).setUp()
        self.user = UserFactory()
        PersistentGradesEnabledFlag.objects.create(enabled_for_all_courses=True, enabled=True)

    def set_up_course(self, enable_subsection_grades=True):
        """
        Configures the course for this test.
        """
        # pylint: disable=attribute-defined-outside-init,no-member
        self.course = CourseFactory.create(
            org='edx',
            name='course',
            run='run',
        )
        if not enable_subsection_grades:
            PersistentGradesEnabledFlag.objects.create(enabled=False)

        self.chapter = ItemFactory.create(parent=self.course, category="chapter", display_name="Chapter")
        self.sequential = ItemFactory.create(parent=self.chapter, category='sequential', display_name="Sequential1")
        self.problem = ItemFactory.create(parent=self.sequential, category='problem', display_name='Problem')

        self.frozen_now_datetime = datetime.now().replace(tzinfo=pytz.UTC)
        self.frozen_now_timestamp = to_timestamp(self.frozen_now_datetime)

        self.problem_weighted_score_changed_kwargs = OrderedDict([
            ('weighted_earned', 1.0),
            ('weighted_possible', 2.0),
            ('user_id', self.user.id),
            ('course_id', unicode(self.course.id)),
            ('usage_id', unicode(self.problem.location)),
            ('only_if_higher', None),
            ('modified', self.frozen_now_datetime),
        ])

        create_new_event_transaction_id()

        self.recalculate_subsection_grade_kwargs = OrderedDict([
            ('user_id', self.user.id),
            ('course_id', unicode(self.course.id)),
            ('usage_id', unicode(self.problem.location)),
            ('only_if_higher', None),
            ('expected_modified_time', self.frozen_now_timestamp),
            ('score_deleted', False),
            ('event_transaction_id', unicode(get_event_transaction_id())),
            ('event_transaction_type', u'edx.grades.problem.submitted'),
        ])

        # this call caches the anonymous id on the user object, saving 4 queries in all happy path tests
        _ = anonymous_id_for_user(self.user, self.course.id)
        # pylint: enable=attribute-defined-outside-init,no-member

    @contextmanager
    def mock_get_score(self, score=MagicMock(grade=1.0, max_grade=2.0)):
        """
        Mocks the scores needed by the SCORE_PUBLISHED signal
        handler. By default, sets the returned score to 1/2.
        """
        with patch("lms.djangoapps.grades.tasks.get_score", return_value=score):
            yield

    def test_problem_weighted_score_changed_queues_task(self):
        """
        Ensures that the PROBLEM_WEIGHTED_SCORE_CHANGED signal enqueues the correct task.
        """
        self.set_up_course()
        send_args = self.problem_weighted_score_changed_kwargs
        local_task_args = self.recalculate_subsection_grade_kwargs.copy()
        local_task_args['event_transaction_type'] = u'edx.grades.problem.submitted'
        with self.mock_get_score() and patch(
            'lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.apply_async',
            return_value=None
        ) as mock_task_apply:
            PROBLEM_WEIGHTED_SCORE_CHANGED.send(sender=None, **send_args)
            mock_task_apply.assert_called_once_with(kwargs=local_task_args)

    @patch('lms.djangoapps.grades.signals.signals.SUBSECTION_SCORE_CHANGED.send')
    def test_subsection_update_triggers_signal(self, mock_subsection_signal):
        """
        Ensures that the subsection update operation triggers a signal.
        """
        self.set_up_course()
        self._apply_recalculate_subsection_grade()
        self.assertTrue(mock_subsection_signal.called)

    @ddt.data(
        (ModuleStoreEnum.Type.mongo, 1, 23),
        (ModuleStoreEnum.Type.split, 3, 22),
    )
    @ddt.unpack
    def test_subsection_grade_updated(self, default_store, num_mongo_calls, num_sql_calls):
        with self.store.default_store(default_store):
            self.set_up_course()
            self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
            with check_mongo_calls(num_mongo_calls):
                with self.assertNumQueries(num_sql_calls):
                    self._apply_recalculate_subsection_grade()

    @patch('lms.djangoapps.grades.signals.signals.SUBSECTION_SCORE_CHANGED.send')
    def test_other_inaccessible_subsection(self, mock_subsection_signal):
        self.set_up_course()
        accessible_seq = ItemFactory.create(parent=self.chapter, category='sequential')
        inaccessible_seq = ItemFactory.create(parent=self.chapter, category='sequential', visible_to_staff_only=True)

        # Update problem to have 2 additional sequential parents.
        # So in total, 3 sequential parents, with one inaccessible.
        for sequential in (accessible_seq, inaccessible_seq):
            sequential.children = [self.problem.location]
            modulestore().update_item(sequential, self.user.id)  # pylint: disable=no-member

        # Make sure the signal is sent for only the 2 accessible sequentials.
        self._apply_recalculate_subsection_grade()
        self.assertEquals(mock_subsection_signal.call_count, 2)
        sequentials_signalled = {
            args[1]['subsection_grade'].location
            for args in mock_subsection_signal.call_args_list
        }
        self.assertSetEqual(
            sequentials_signalled,
            {self.sequential.location, accessible_seq.location},
        )

    def test_single_call_to_create_block_structure(self):
        self.set_up_course()
        self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
        with patch(
            'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_cache',
            return_value=None,
        ) as mock_block_structure_create:
            self._apply_recalculate_subsection_grade()
            self.assertEquals(mock_block_structure_create.call_count, 1)

    # TODO (TNL-6225) Fix the number of SQL queries so they
    # don't grow linearly with the number of sequentials.
    @ddt.data(
        (ModuleStoreEnum.Type.mongo, 1, 46),
        (ModuleStoreEnum.Type.split, 3, 45),
    )
    @ddt.unpack
    def test_query_count_does_not_change_with_more_content(self, default_store, num_mongo_calls, num_sql_calls):
        with self.store.default_store(default_store):
            self.set_up_course()
            self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id))

            num_problems = 10
            for _ in range(num_problems):
                ItemFactory.create(parent=self.sequential, category='problem')

            num_sequentials = 10
            for _ in range(num_sequentials):
                ItemFactory.create(parent=self.chapter, category='sequential')

            with check_mongo_calls(num_mongo_calls):
                with self.assertNumQueries(num_sql_calls):
                    self._apply_recalculate_subsection_grade()

    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
    def test_subsection_grades_not_enabled_on_course(self, default_store):
        with self.store.default_store(default_store):
            self.set_up_course(enable_subsection_grades=False)
            self.assertFalse(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
            with check_mongo_calls(0):
                with self.assertNumQueries(0):
                    self._apply_recalculate_subsection_grade()

    @skip("Pending completion of TNL-5089")
    @ddt.data(
        (ModuleStoreEnum.Type.mongo, True),
        (ModuleStoreEnum.Type.split, True),
        (ModuleStoreEnum.Type.mongo, False),
        (ModuleStoreEnum.Type.split, False),
    )
    @ddt.unpack
    def test_query_counts_with_feature_flag(self, default_store, feature_flag):
        PersistentGradesEnabledFlag.objects.create(enabled=feature_flag)
        with self.store.default_store(default_store):
            self.set_up_course()
            with check_mongo_calls(0):
                with self.assertNumQueries(3 if feature_flag else 2):
                    self._apply_recalculate_subsection_grade()

    @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry')
    @patch('lms.djangoapps.grades.new.subsection_grade.SubsectionGradeFactory.update')
    def test_retry_subsection_update_on_integrity_error(self, mock_update, mock_retry):
        """
        Ensures that tasks will be retried if IntegrityErrors are encountered.
        """
        self.set_up_course()
        mock_update.side_effect = IntegrityError("WHAMMY")
        self._apply_recalculate_subsection_grade()
        self._assert_retry_called(mock_retry)

    @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry')
    def test_retry_subsection_grade_on_update_not_complete(self, mock_retry):
        self.set_up_course()
        self._apply_recalculate_subsection_grade(
            mock_score=MagicMock(modified=datetime.utcnow().replace(tzinfo=pytz.UTC) - timedelta(days=1))
        )
        self._assert_retry_called(mock_retry)

    @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry')
    def test_retry_subsection_grade_on_no_score(self, mock_retry):
        self.set_up_course()
        self._apply_recalculate_subsection_grade(mock_score=None)
        self._assert_retry_called(mock_retry)

    @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry')
    def test_retry_subsection_grade_on_no_sub_score(self, mock_retry):
        self.set_up_course()
        with patch('lms.djangoapps.grades.tasks.sub_api.get_score') as mock_sub_score:
            mock_sub_score.return_value = None
            self._apply_recalculate_subsection_grade(
                mock_score=MagicMock(module_type='openassessment')
            )
        self._assert_retry_called(mock_retry)

    @patch('lms.djangoapps.grades.signals.signals.SUBSECTION_SCORE_CHANGED.send')
    @patch('lms.djangoapps.grades.new.subsection_grade.SubsectionGradeFactory.update')
    def test_retry_first_time_only(self, mock_update, mock_course_signal):
        """
        Ensures that a task retry completes after a one-time failure.
        """
        self.set_up_course()
        mock_update.side_effect = [IntegrityError("WHAMMY"), None]
        self._apply_recalculate_subsection_grade()
        self.assertEquals(mock_course_signal.call_count, 1)

    def _apply_recalculate_subsection_grade(
            self,
            mock_score=MagicMock(modified=datetime.utcnow().replace(tzinfo=pytz.UTC) + timedelta(days=1))
    ):
        """
        Calls the recalculate_subsection_grade task with necessary
        mocking in place.
        """
        with self.mock_get_score(mock_score):
            recalculate_subsection_grade_v2.apply(kwargs=self.recalculate_subsection_grade_kwargs)

    def _assert_retry_called(self, mock_retry):
        """
        Verifies the task was retried and with the correct
        number of arguments.
        """
        self.assertTrue(mock_retry.called)
        self.assertEquals(len(mock_retry.call_args[1]['kwargs']), len(self.recalculate_subsection_grade_kwargs))
Beispiel #47
0
class MultiRemoteTest(unittest.TestCase):

    def setUp(self):
        self.servers = OrderedDict()
        self.users = {}
        for i in range(3):
            test_server = TestServer()
            self.servers["remote%d" % i] = test_server
            self.users["remote%d" % i] = [("lasote", "mypass")]

        self.client = TestClient(servers=self.servers, users=self.users)

    def predefine_remote_test(self):
        files = cpp_hello_conan_files("Hello0", "0.1", build=False)
        self.client.save(files)
        self.client.run("export . lasote/stable")
        self.client.run("upload Hello0/0.1@lasote/stable -r=remote0")
        self.client.run("upload Hello0/0.1@lasote/stable -r=remote1")
        self.client.run("upload Hello0/0.1@lasote/stable -r=remote2")
        self.client.run('remove "*" -f')
        self.client.run("remote add_ref Hello0/0.1@lasote/stable remote1")
        self.client.run("install Hello0/0.1@lasote/stable --build=missing")
        self.assertIn("Hello0/0.1@lasote/stable: Retrieving from predefined remote 'remote1'",
                      self.client.user_io.out)
        self.client.run("remote list_ref")
        self.assertIn("Hello0/0.1@lasote/stable: remote1", self.client.user_io.out)

    def upload_test(self):
        conan_reference = ConanFileReference.loads("Hello0/0.1@lasote/stable")
        files = cpp_hello_conan_files("Hello0", "0.1", build=False)
        self.client.save(files)
        self.client.run("export . lasote/stable")
        self.client.run("upload %s" % str(conan_reference))

        self.client.run("info %s" % str(conan_reference))
        self.assertIn("remote0=http://", self.client.user_io.out)

        # The remote, once fixed does not change
        self.client.run("upload %s -r=remote1" % str(conan_reference))
        self.client.run("info %s" % str(conan_reference))
        self.assertIn("remote0=http://", self.client.user_io.out)

        # Now install it in other machine from remote 0
        client2 = TestClient(servers=self.servers, users=self.users)
        client2.run("install %s --build=missing" % str(conan_reference))
        client2.run("info %s" % str(conan_reference))
        self.assertIn("remote0=http://", client2.user_io.out)

        # Now install it in other machine from remote 1
        servers = self.servers.copy()
        servers.pop("remote0")
        client3 = TestClient(servers=servers, users=self.users)
        client3.run("install %s --build=missing" % str(conan_reference))
        client3.run("info %s" % str(conan_reference))
        self.assertIn("remote1=http://", client3.user_io.out)

    def fail_when_not_notfound_test(self):
        """
        If a remote fails with a 404 it has to keep looking in the next remote, but if it fails by
        any other reason it has to stop
        """
        servers = OrderedDict()
        servers["s0"] = TestServer()
        servers["s1"] = TestServer()
        servers["s2"] = TestServer()

        client = TestClient(servers=servers, users=self.users)
        files = cpp_hello_conan_files("MyLib", "0.1", build=False)
        client.save(files)
        client.run("create . lasote/testing")
        client.run("user lasote -p mypass -r s1")
        client.run("upload MyLib* -r s1 -c")

        servers["s1"].fake_url = "http://asdlhaljksdhlajkshdljakhsd"  # Do not exist
        client2 = TestClient(servers=servers, users=self.users)
        err = client2.run("install MyLib/0.1@conan/testing --build=missing", ignore_error=True)
        self.assertTrue(err)
        self.assertIn("MyLib/0.1@conan/testing: Trying with 's0'...", client2.out)
        self.assertIn("MyLib/0.1@conan/testing: Trying with 's1'...", client2.out)
        self.assertIn("Unable to connect to s1=http://asdlhaljksdhlajkshdljakhsd", client2.out)
        # s2 is not even tried
        self.assertNotIn("MyLib/0.1@conan/testing: Trying with 's2'...", client2.out)

    def install_from_remotes_test(self):
        for i in range(3):
            conan_reference = ConanFileReference.loads("Hello%d/0.1@lasote/stable" % i)
            files = cpp_hello_conan_files("Hello%d" % i, "0.1", build=False)
            self.client.save(files)
            self.client.run("export . lasote/stable")
            self.client.run("upload %s -r=remote%d" % (str(conan_reference), i))

            self.client.run("info %s" % str(conan_reference))
            self.assertIn("remote%d=http://" % i, self.client.user_io.out)

        # Now install it in other machine from remote 0
        client2 = TestClient(servers=self.servers, users=self.users)
        files = cpp_hello_conan_files("HelloX", "0.1", deps=["Hello0/0.1@lasote/stable",
                                                             "Hello1/0.1@lasote/stable",
                                                             "Hello2/0.1@lasote/stable"])
        files["conanfile.py"] = files["conanfile.py"].replace("def build(", "def build2(")
        client2.save(files)
        client2.run("install . --build=missing")
        self.assertIn("Hello0/0.1@lasote/stable from remote0", client2.user_io.out)
        self.assertIn("Hello1/0.1@lasote/stable from remote1", client2.user_io.out)
        self.assertIn("Hello2/0.1@lasote/stable from remote2", client2.user_io.out)
        client2.run("info .")
        self.assertIn("Remote: remote0=http://", client2.user_io.out)
        self.assertIn("Remote: remote1=http://", client2.user_io.out)
        self.assertIn("Remote: remote2=http://", client2.user_io.out)
Beispiel #48
0
class DAGDependency:
    """Object to represent a quantum circuit as a directed acyclic graph
    via operation dependencies (i.e. lack of commutation).

    The nodes in the graph are operations represented by quantum gates.
    The edges correspond to non-commutation between two operations
    (i.e. a dependency). A directed edge from node A to node B means that
    operation A does not commute with operation B.
    The object's methods allow circuits to be constructed.

    The nodes in the graph have the following attributes:
    'operation', 'successors', 'predecessors'.

    **Example:**

    Bell circuit with no measurement.

    .. parsed-literal::

              ┌───┐
        qr_0: ┤ H ├──■──
              └───┘┌─┴─┐
        qr_1: ─────┤ X ├
                   └───┘

    The dependency DAG for the above circuit is represented by two nodes.
    The first one corresponds to Hadamard gate, the second one to the CNOT gate
    as the gates do not commute there is an edge between the two nodes.

    **Reference:**

    [1] Iten, R., Moyard, R., Metger, T., Sutter, D. and Woerner, S., 2020.
    Exact and practical pattern matching for quantum circuit optimization.
    `arXiv:1909.05270 <https://arxiv.org/abs/1909.05270>`_

    """

    def __init__(self):
        """
        Create an empty DAGDependency.
        """
        # Circuit name
        self.name = None

        # Circuit metadata
        self.metadata = None

        # Directed multigraph whose nodes are operations(gates) and edges
        # represent non-commutativity between two gates.
        self._multi_graph = rx.PyDAG()

        # Map of qreg/creg name to Register object.
        self.qregs = OrderedDict()
        self.cregs = OrderedDict()

        # List of all Qubit/Clbit wires.
        self.qubits = []
        self.clbits = []

        self._global_phase = 0
        self._calibrations = defaultdict(dict)

        self.duration = None
        self.unit = "dt"

        self.comm_checker = CommutationChecker()

    @property
    def global_phase(self):
        """Return the global phase of the circuit."""
        return self._global_phase

    @global_phase.setter
    def global_phase(self, angle):
        """Set the global phase of the circuit.

        Args:
            angle (float, ParameterExpression)
        """
        from qiskit.circuit.parameterexpression import ParameterExpression  # needed?

        if isinstance(angle, ParameterExpression):
            self._global_phase = angle
        else:
            # Set the phase to the [0, 2π) interval
            angle = float(angle)
            if not angle:
                self._global_phase = 0
            else:
                self._global_phase = angle % (2 * math.pi)

    @property
    def calibrations(self):
        """Return calibration dictionary.

        The custom pulse definition of a given gate is of the form
            {'gate_name': {(qubits, params): schedule}}
        """
        return dict(self._calibrations)

    @calibrations.setter
    def calibrations(self, calibrations):
        """Set the circuit calibration data from a dictionary of calibration definition.

        Args:
            calibrations (dict): A dictionary of input in the format
                {'gate_name': {(qubits, gate_params): schedule}}
        """
        self._calibrations = defaultdict(dict, calibrations)

    def to_networkx(self):
        """Returns a copy of the DAGDependency in networkx format."""
        # For backwards compatibility, return networkx structure from terra 0.12
        # where DAGNodes instances are used as indexes on the networkx graph.
        warnings.warn(
            "The to_networkx() method is deprecated and will be removed in a future release.",
            DeprecationWarning,
            stacklevel=2,
        )

        try:
            import networkx as nx
        except ImportError as ex:
            raise MissingOptionalLibraryError(
                libname="Networkx",
                name="DAG dependency",
                pip_install="pip install networkx",
            ) from ex
        dag_networkx = nx.MultiDiGraph()

        for node in self.get_nodes():
            dag_networkx.add_node(node)
        for node in self.topological_nodes():
            for source_id, dest_id, edge in self.get_in_edges(node.node_id):
                dag_networkx.add_edge(self.get_node(source_id), self.get_node(dest_id), **edge)
        return dag_networkx

    def to_retworkx(self):
        """Returns the DAGDependency in retworkx format."""
        return self._multi_graph

    def size(self):
        """Returns the number of gates in the circuit"""
        return len(self._multi_graph)

    def depth(self):
        """Return the circuit depth.
        Returns:
            int: the circuit depth
        """
        depth = rx.dag_longest_path_length(self._multi_graph)
        return depth if depth >= 0 else 0

    def add_qubits(self, qubits):
        """Add individual qubit wires."""
        if any(not isinstance(qubit, Qubit) for qubit in qubits):
            raise DAGDependencyError("not a Qubit instance.")

        duplicate_qubits = set(self.qubits).intersection(qubits)
        if duplicate_qubits:
            raise DAGDependencyError("duplicate qubits %s" % duplicate_qubits)

        self.qubits.extend(qubits)

    def add_clbits(self, clbits):
        """Add individual clbit wires."""
        if any(not isinstance(clbit, Clbit) for clbit in clbits):
            raise DAGDependencyError("not a Clbit instance.")

        duplicate_clbits = set(self.clbits).intersection(clbits)
        if duplicate_clbits:
            raise DAGDependencyError("duplicate clbits %s" % duplicate_clbits)

        self.clbits.extend(clbits)

    def add_qreg(self, qreg):
        """Add qubits in a quantum register."""
        if not isinstance(qreg, QuantumRegister):
            raise DAGDependencyError("not a QuantumRegister instance.")
        if qreg.name in self.qregs:
            raise DAGDependencyError("duplicate register %s" % qreg.name)
        self.qregs[qreg.name] = qreg
        existing_qubits = set(self.qubits)
        for j in range(qreg.size):
            if qreg[j] not in existing_qubits:
                self.qubits.append(qreg[j])

    def add_creg(self, creg):
        """Add clbits in a classical register."""
        if not isinstance(creg, ClassicalRegister):
            raise DAGDependencyError("not a ClassicalRegister instance.")
        if creg.name in self.cregs:
            raise DAGDependencyError("duplicate register %s" % creg.name)
        self.cregs[creg.name] = creg
        existing_clbits = set(self.clbits)
        for j in range(creg.size):
            if creg[j] not in existing_clbits:
                self.clbits.append(creg[j])

    def _add_multi_graph_node(self, node):
        """
        Args:
            node (DAGDepNode): considered node.

        Returns:
            node_id(int): corresponding label to the added node.
        """
        node_id = self._multi_graph.add_node(node)
        node.node_id = node_id
        return node_id

    def get_nodes(self):
        """
        Returns:
            generator(dict): iterator over all the nodes.
        """
        return iter(self._multi_graph.nodes())

    def get_node(self, node_id):
        """
        Args:
            node_id (int): label of considered node.

        Returns:
            node: corresponding to the label.
        """
        return self._multi_graph.get_node_data(node_id)

    def _add_multi_graph_edge(self, src_id, dest_id, data):
        """
        Function to add an edge from given data (dict) between two nodes.

        Args:
            src_id (int): label of the first node.
            dest_id (int): label of the second node.
            data (dict): data contained on the edge.

        """
        self._multi_graph.add_edge(src_id, dest_id, data)

    def get_edges(self, src_id, dest_id):
        """
        Edge enumeration between two nodes through method get_all_edge_data.

        Args:
            src_id (int): label of the first node.
            dest_id (int): label of the second node.

        Returns:
            List: corresponding to all edges between the two nodes.
        """
        return self._multi_graph.get_all_edge_data(src_id, dest_id)

    def get_all_edges(self):
        """
        Enumeration of all edges.

        Returns:
            List: corresponding to the label.
        """

        return [
            (src, dest, data)
            for src_node in self._multi_graph.nodes()
            for (src, dest, data) in self._multi_graph.out_edges(src_node.node_id)
        ]

    def get_in_edges(self, node_id):
        """
        Enumeration of all incoming edges for a given node.

        Args:
            node_id (int): label of considered node.

        Returns:
            List: corresponding incoming edges data.
        """
        return self._multi_graph.in_edges(node_id)

    def get_out_edges(self, node_id):
        """
        Enumeration of all outgoing edges for a given node.

        Args:
            node_id (int): label of considered node.

        Returns:
            List: corresponding outgoing edges data.
        """
        return self._multi_graph.out_edges(node_id)

    def direct_successors(self, node_id):
        """
        Direct successors id of a given node as sorted list.

        Args:
            node_id (int): label of considered node.

        Returns:
            List: direct successors id as a sorted list
        """
        return sorted(list(self._multi_graph.adj_direction(node_id, False).keys()))

    def direct_predecessors(self, node_id):
        """
        Direct predecessors id of a given node as sorted list.

        Args:
            node_id (int): label of considered node.

        Returns:
            List: direct predecessors id as a sorted list
        """
        return sorted(list(self._multi_graph.adj_direction(node_id, True).keys()))

    def successors(self, node_id):
        """
        Successors id of a given node as sorted list.

        Args:
            node_id (int): label of considered node.

        Returns:
            List: all successors id as a sorted list
        """
        return self._multi_graph.get_node_data(node_id).successors

    def predecessors(self, node_id):
        """
        Predecessors id of a given node as sorted list.

        Args:
            node_id (int): label of considered node.

        Returns:
            List: all predecessors id as a sorted list
        """
        return self._multi_graph.get_node_data(node_id).predecessors

    def topological_nodes(self):
        """
        Yield nodes in topological order.

        Returns:
            generator(DAGNode): node in topological order.
        """

        def _key(x):
            return x.sort_key

        return iter(rx.lexicographical_topological_sort(self._multi_graph, key=_key))

    def add_op_node(self, operation, qargs, cargs):
        """Add a DAGDepNode to the graph and update the edges.

        Args:
            operation (qiskit.circuit.Instruction): operation as a quantum gate.
            qargs (list[Qubit]): list of qubits on which the operation acts
            cargs (list[Clbit]): list of classical wires to attach to.
        """
        directives = ["measure"]
        if not getattr(operation, "_directive", False) and operation.name not in directives:
            qindices_list = []
            for elem in qargs:
                qindices_list.append(self.qubits.index(elem))
            if getattr(operation, "condition", None):
                for clbit in self.clbits:
                    if clbit in operation.condition[0]:
                        initial = self.clbits.index(clbit)
                        final = self.clbits.index(clbit) + operation.condition[0].size
                        cindices_list = range(initial, final)
                        break
            else:
                cindices_list = []
        else:
            qindices_list = []
            cindices_list = []

        new_node = DAGDepNode(
            type="op",
            op=operation,
            name=operation.name,
            qargs=qargs,
            cargs=cargs,
            successors=[],
            predecessors=[],
            qindices=qindices_list,
            cindices=cindices_list,
        )
        self._add_multi_graph_node(new_node)
        self._update_edges()

    def _gather_pred(self, node_id, direct_pred):
        """Function set an attribute predecessors and gather multiple lists
        of direct predecessors into a single one.

        Args:
            node_id (int): label of the considered node in the DAG
            direct_pred (list): list of direct successors for the given node

        Returns:
            DAGDependency: A multigraph with update of the attribute ['predecessors']
            the lists of direct successors are put into a single one
        """
        gather = self._multi_graph
        gather.get_node_data(node_id).predecessors = []
        for d_pred in direct_pred:
            gather.get_node_data(node_id).predecessors.append([d_pred])
            pred = self._multi_graph.get_node_data(d_pred).predecessors
            gather.get_node_data(node_id).predecessors.append(pred)
        return gather

    def _gather_succ(self, node_id, direct_succ):
        """
        Function set an attribute successors and gather multiple lists
        of direct successors into a single one.

        Args:
            node_id (int): label of the considered node in the DAG
            direct_succ (list): list of direct successors for the given node

        Returns:
            MultiDiGraph: with update of the attribute ['predecessors']
            the lists of direct successors are put into a single one
        """
        gather = self._multi_graph
        for d_succ in direct_succ:
            gather.get_node_data(node_id).successors.append([d_succ])
            succ = gather.get_node_data(d_succ).successors
            gather.get_node_data(node_id).successors.append(succ)
        return gather

    def _list_pred(self, node_id):
        """
        Use _gather_pred function and merge_no_duplicates to construct
        the list of predecessors for a given node.

        Args:
            node_id (int): label of the considered node
        """
        direct_pred = self.direct_predecessors(node_id)
        self._multi_graph = self._gather_pred(node_id, direct_pred)
        self._multi_graph.get_node_data(node_id).predecessors = list(
            merge_no_duplicates(*(self._multi_graph.get_node_data(node_id).predecessors))
        )

    def _update_edges(self):
        """
        Function to verify the commutation relation and reachability
        for predecessors, the nodes do not commute and
        if the predecessor is reachable. Update the DAGDependency by
        introducing edges and predecessors(attribute)
        """
        max_node_id = len(self._multi_graph) - 1
        max_node = self._multi_graph.get_node_data(max_node_id)

        for current_node_id in range(0, max_node_id):
            self._multi_graph.get_node_data(current_node_id).reachable = True
        # Check the commutation relation with reachable node, it adds edges if it does not commute
        for prev_node_id in range(max_node_id - 1, -1, -1):
            prev_node = self._multi_graph.get_node_data(prev_node_id)
            if prev_node.reachable and not self.comm_checker.commute(
                prev_node.op,
                prev_node.qargs,
                prev_node.cargs,
                max_node.op,
                max_node.qargs,
                max_node.cargs,
            ):
                self._multi_graph.add_edge(prev_node_id, max_node_id, {"commute": False})
                self._list_pred(max_node_id)
                list_predecessors = self._multi_graph.get_node_data(max_node_id).predecessors
                for pred_id in list_predecessors:
                    self._multi_graph.get_node_data(pred_id).reachable = False

    def _add_successors(self):
        """
        Use _gather_succ and merge_no_duplicates to create the list of successors
        for each node. Update DAGDependency 'successors' attribute. It has to
        be used when the DAGDependency() object is complete (i.e. converters).
        """
        for node_id in range(len(self._multi_graph) - 1, -1, -1):
            direct_successors = self.direct_successors(node_id)

            self._multi_graph = self._gather_succ(node_id, direct_successors)

            self._multi_graph.get_node_data(node_id).successors = list(
                merge_no_duplicates(*self._multi_graph.get_node_data(node_id).successors)
            )

    def copy(self):
        """
        Function to copy a DAGDependency object.
        Returns:
            DAGDependency: a copy of a DAGDependency object.
        """

        dag = DAGDependency()
        dag.name = self.name
        dag.cregs = self.cregs.copy()
        dag.qregs = self.qregs.copy()

        for node in self.get_nodes():
            dag._multi_graph.add_node(node.copy())
        for edges in self.get_all_edges():
            dag._multi_graph.add_edge(edges[0], edges[1], edges[2])
        return dag

    def draw(self, scale=0.7, filename=None, style="color"):
        """
        Draws the DAGDependency graph.

        This function needs `pydot <https://github.com/erocarrera/pydot>`, which in turn needs
        Graphviz <https://www.graphviz.org/>` to be installed.

        Args:
            scale (float): scaling factor
            filename (str): file path to save image to (format inferred from name)
            style (str): 'plain': B&W graph
                         'color' (default): color input/output/op nodes

        Returns:
            Ipython.display.Image: if in Jupyter notebook and not saving to file,
                otherwise None.
        """
        from qiskit.visualization.dag_visualization import dag_drawer

        return dag_drawer(dag=self, scale=scale, filename=filename, style=style)
Beispiel #49
0
class GraphView(urwid.WidgetPlaceholder):
    """
    A class responsible for providing the application's interface and
    graph display.
    """
    def __init__(self, controller, args):

        self.controller = controller
        self.custom_temp = args.custom_temp
        self.custom_fan = args.custom_fan
        self.args = args
        self.hline = urwid.AttrWrap(urwid.SolidFill(u'_'), 'line')
        self.mode_buttons = []
        self.refresh_rate_ctrl = urwid.Edit(('bold text', u'Refresh[s]:'),
                                            self.controller.refresh_rate)

        self.visible_graphs = {}
        self.graph_place_holder = urwid.WidgetPlaceholder(urwid.Pile([]))

        self.main_window_w = []

        self.stress_menu = StressMenu(self.on_menu_close)
        self.help_menu = HelpMenu(self.on_menu_close)
        self.about_menu = AboutMenu(self.on_menu_close)
        self.temp_sensors_menu = TempSensorsMenu(self.on_sensors_menu_close)
        self.global_data = GlobalData(is_admin)

        self.stress_menu.sqrt_workers = str(self.global_data.num_cpus)
        self.left_margin = 0
        self.top_margin = 0
        self.v_relative = 50
        self.h_relative = 50

        urwid.WidgetPlaceholder.__init__(self, self.main_window())
        urwid.connect_signal(self.refresh_rate_ctrl, 'change',
                             self.update_refresh_rate)

    def update_refresh_rate(self, edit, new_refresh_rate):
        try:
            if float(new_refresh_rate) <= 0.001:
                pass
            else:
                self.controller.refresh_rate = new_refresh_rate
        except:
            self.controller.refresh_rate = '1.0'

    def update_displayed_information(self):
        """ Update all the graphs that are being displayed """

        for key, val in self.summaries.items():
            val.source.update()

        for g in self.visible_graphs.values():
            g.update_displayed_graph_data()

        for s in self.available_summaries.values():
            s.update()

    def on_reset_button(self, w):
        """Reset graph data and display empty graph"""
        for g in self.visible_graphs.values():
            g.reset()
        for g in self.graphs.values():
            try:
                g.source.reset()
            except (NotImplementedError):
                pass
        self.update_displayed_information()

    def on_menu_close(self):
        """Return to main screen"""
        self.original_widget = self.main_window_w

    def on_sensors_menu_close(self):
        """Return to main screen and update sensor"""
        if self.temp_sensors_menu.current_active_mode:
            logging.info("State is not None")
            self.args.custom_temp = self.temp_sensors_menu.current_active_mode
            self.__init__(self.controller, self.args)
            logging.info("Temp sensor updated to " + self.args.custom_temp)
        else:
            logging.info("Temp sensor is None")

        self.original_widget = self.main_window_w

    def on_stress_menu_open(self, w):
        """Open stress options"""
        self.original_widget = urwid.Overlay(self.stress_menu.main_window,
                                             self.original_widget,
                                             ('relative', self.left_margin),
                                             self.stress_menu.get_size()[1],
                                             ('relative', self.top_margin),
                                             self.stress_menu.get_size()[0])

    def on_help_menu_open(self, w):
        """Open Help menu"""
        self.original_widget = urwid.Overlay(self.help_menu.main_window,
                                             self.original_widget,
                                             ('relative', self.left_margin),
                                             self.help_menu.get_size()[1],
                                             ('relative', self.top_margin),
                                             self.help_menu.get_size()[0])

    def on_about_menu_open(self, w):
        """Open About menu"""
        self.original_widget = urwid.Overlay(self.about_menu.main_window,
                                             self.original_widget,
                                             ('relative', self.left_margin),
                                             self.about_menu.get_size()[1],
                                             ('relative', self.top_margin),
                                             self.about_menu.get_size()[0])

    def on_temp_sensors_menu_open(self, w):
        """Open About menu"""
        self.original_widget = urwid.Overlay(
            self.temp_sensors_menu.main_window, self.original_widget,
            ('relative', self.left_margin),
            self.temp_sensors_menu.get_size()[1],
            ('relative', self.top_margin),
            self.temp_sensors_menu.get_size()[0])

    def on_mode_button(self, button, state):
        """Notify the controller of a new mode setting."""
        if state:
            # The new mode is the label of the button
            self.controller.set_mode(button.get_label())
            self.controller.start_stress()

    def on_mode_change(self, m):
        """Handle external mode change by updating radio buttons."""
        for rb in self.mode_buttons:
            if rb.get_label() == m:
                rb.set_state(True, do_callback=False)
                break

    def on_unicode_checkbox(self, w, state):
        """Enable smooth edges if utf-8 is supported"""
        logging.debug("unicode State is " + str(state))
        if state:
            self.hline = urwid.AttrWrap(
                urwid.SolidFill(u'\N{LOWER ONE QUARTER BLOCK}'), 'line')
        else:
            self.hline = urwid.AttrWrap(urwid.SolidFill(u'_'), 'line')

        for g_name, g in self.graphs.items():
            g.set_smooth_colors(state)

        self.show_graphs()

    def exit_program(self, w=None):
        """ Kill all stress operations upon exit"""
        try:
            kill_child_processes(self.controller.mode.get_stress_process())
        except:
            logging.debug('Could not kill process')
        raise urwid.ExitMainLoop()

    def graph_controls(self):
        """ Dislplay sidebar controls. i.e. buttons, and controls"""
        modes = self.controller.get_modes()
        # setup mode radio buttons
        group = []
        for m in modes:
            rb = radio_button(group, m, self.on_mode_button)
            self.mode_buttons.append(rb)

        # Create list of buttons
        control_options = [button("Reset", self.on_reset_button)]
        if stress_installed:
            control_options.append(
                button('Stress Options', self.on_stress_menu_open))
        control_options.append(
            button('Temp Sensors', self.on_temp_sensors_menu_open))
        control_options.append(button('Help', self.on_help_menu_open))
        control_options.append(button('About', self.on_about_menu_open))

        # Create the menu
        animate_controls = urwid.GridFlow(control_options, 18, 2, 0, 'center')

        if urwid.get_encoding_mode() == "utf8":
            unicode_checkbox = urwid.CheckBox(
                "Smooth Graph",
                state=False,
                on_state_change=self.on_unicode_checkbox)
        else:
            unicode_checkbox = urwid.Text("UTF-8 encoding not detected")

        install_stress_message = urwid.Text("")
        if not stress_installed:
            install_stress_message = urwid.Text(
                ('button normal', u"(N/A) install stress"))

        graph_checkboxes = [
            urwid.CheckBox(x.get_graph_name(),
                           state=True,
                           on_state_change=lambda w, state, x=x: self.
                           change_checkbox_state(x, state))
            for x in self.available_graphs.values()
        ]
        unavalable_graphs = [
            urwid.Text(("[N/A] " + x.get_graph_name()))
            for x in self.graphs.values()
            if x.source.get_is_available() == False
        ]
        graph_checkboxes += unavalable_graphs

        buttons = [
            urwid.Text(('bold text', u"Modes"), align="center"),
        ] + self.mode_buttons + [
            install_stress_message,
            urwid.Divider(),
            urwid.Text(('bold text', u"Control Options"), align="center"),
            animate_controls,
            urwid.Divider(),
            self.refresh_rate_ctrl,
            urwid.Divider(),
            urwid.LineBox(urwid.Pile(graph_checkboxes)),
            urwid.LineBox(unicode_checkbox),
            urwid.Divider(),
            button("Quit", self.exit_program),
        ]

        return buttons

    def change_checkbox_state(self, x, state):

        if state:
            self.visible_graphs[x.get_graph_name()] = x
        else:
            del self.visible_graphs[x.get_graph_name()]
        self.show_graphs()

    def show_graphs(self):
        """Show a pile of the graph selected for dislpay"""
        elements = itertools.chain.from_iterable(
            ([graph, ('fixed', 1, self.hline)]
             for graph in self.visible_graphs.values()))
        self.graph_place_holder.original_widget = urwid.Pile(elements)

    def cpu_stats(self):
        """Read and display processor name """
        cpu_name = urwid.Text("CPU Name N/A", align="center")
        try:
            cpu_name = urwid.Text(get_processor_name().strip(), align="center")
        except:
            logging.info("CPU name not available")
        cpu_stats = [cpu_name, urwid.Divider()]
        return cpu_stats

    def graph_stats(self):

        fixed_stats = []
        for key, val in self.available_summaries.items():
            fixed_stats += val.get_text_item_list()

        return fixed_stats

    def main_window(self):

        user_config_path = None
        if not user_config_dir_exists():
            user_config_path = make_user_config_dir()
        else:
            user_config_path = get_user_config_path()

        script_hooks_enabled = True
        if user_config_path is None:
            logging.warn(
                'Failed to find or create scripts directory, proceeding without scripting support'
            )
            script_hooks_enabled = False
        else:
            self.script_loader = ScriptHookLoader(user_config_path)

        # initiating the graphs
        self.graphs = OrderedDict()
        self.summaries = OrderedDict()

        # TODO: Update to find sensors automatically

        freq_source = FreqSource(is_admin)
        self.graphs[freq_source.get_source_name()] = StuiBarGraph(
            freq_source, 'freq light', 'freq dark', 'freq light smooth',
            'freq dark smooth')
        self.summaries[freq_source.get_source_name()] = SummaryTextList(
            freq_source)

        util_source = UtilSource()
        self.graphs[util_source.get_source_name()] = StuiBarGraph(
            util_source, 'util light', 'util dark', 'util light smooth',
            'util dark smooth')
        self.summaries[util_source.get_source_name()] = SummaryTextList(
            util_source)

        temp_source = TemperatureSource(self.custom_temp)

        if script_hooks_enabled:
            temp_source.add_edge_hook(
                self.script_loader.load_script(temp_source.__class__.__name__,
                                               30000)
            )  # Invoke threshold script every 30s while threshold is exceeded.

        alert_colors = [
            'high temp light', 'high temp dark', 'high temp light smooth',
            'high temp dark smooth'
        ]
        self.graphs[temp_source.get_source_name()] = StuiBarGraph(
            temp_source,
            'temp light',
            'temp dark',
            'temp light smooth',
            'temp dark smooth',
            alert_colors=alert_colors)
        self.summaries[temp_source.get_source_name()] = SummaryTextList(
            temp_source, 'high temp txt')

        rapl_power_source = RaplPowerSource()
        self.graphs[rapl_power_source.get_source_name()] = StuiBarGraph(
            rapl_power_source, 'power dark', 'power light',
            'power dark smooth', 'power light smooth')
        self.summaries[rapl_power_source.get_source_name()] = SummaryTextList(
            rapl_power_source)

        fan_source = FanSource(self.custom_fan)
        self.summaries[fan_source.get_source_name()] = SummaryTextList(
            fan_source)

        # only interested in available graph
        self.available_graphs = OrderedDict(
            (key, val) for key, val in self.graphs.items()
            if val.get_is_available())
        self.available_summaries = OrderedDict(
            (key, val) for key, val in self.summaries.items()
            if val.get_is_available())

        self.visible_graphs = self.available_graphs.copy()
        self.show_graphs()

        cpu_stats = self.cpu_stats()
        graph_controls = self.graph_controls()
        graph_stats = self.graph_stats()

        text_col = ViListBox(
            urwid.SimpleListWalker(cpu_stats + graph_controls +
                                   [urwid.Divider()] + graph_stats))

        vline = urwid.AttrWrap(urwid.SolidFill(u'\u2502'), 'line')
        w = urwid.Columns([
            ('fixed', 20, text_col),
            ('fixed', 1, vline),
            ('weight', 2, self.graph_place_holder),
        ],
                          dividechars=1,
                          focus_column=0)

        w = urwid.Padding(w, ('fixed left', 1), ('fixed right', 0))
        w = urwid.AttrWrap(w, 'body')
        w = urwid.LineBox(w)
        w = urwid.AttrWrap(w, 'line')
        self.main_window_w = w
        return self.main_window_w
Beispiel #50
0
class Serializer(BaseSerializer):
    """
    Serializer class.

    """
    def to_internal_value(self, data):
        """
        Data transformation to python object.

        :param dict data: Data for transformation.

        :return: Transformed data.
        :rtype: dict

        :raise ValidationError: If not valid data.

        """
        return self._field_validation(self.fields, data)

    def to_representation(self, instance):
        """
        Transformation an object to a valid JSON object.

        :param object instance: The object to transformation.

        :return: Transformed data.
        :rtype: object

        """
        res = OrderedDict()  # Attributes storage.

        for field_name, field_val in six.iteritems(self.fields):
            # TODO: mini hack
            if not isinstance(field_val, SerializerMethodField):
                # We try to get the attribute.
                try:
                    attribute = field_val.get_attribute(instance)
                except SkipError:
                    # TODO: That thing, throw an error, if the attribute of the object is not found, or skip?
                    continue
            else:
                attribute = instance

            # We try to turn it into a JSON valid format.
            res[field_name] = field_val.to_representation(attribute)

        # Return.
        return res

    def _manual_validate_method(self, field_name, validated_value):
        """
        Manual validation of a specific field.

        :param str field_name: Field name.
        :param object validated_value: Field value.

        :return: Validation value.
        :rtype: object

        """
        # We look, if there is a method of manual validation, we call it..
        manual_validate_method = getattr(self, 'validate_' + field_name, None)
        if callable(manual_validate_method):
            validated_value = manual_validate_method(validated_value)
        return validated_value

    def _field_validation(self, fields_dict, data):
        """
        Validation add fields

        :param dict fields_dict: Dictionary with initialized fields that we validate.
        :param dict data: Data that is validated.

        :return: Validated and transformed data.
        :raise ValidationError: If errors occurred during validation.

        """
        validated_data, errors = OrderedDict(), OrderedDict()
        # Running through the fields.
        for field_name, field_obj in six.iteritems(fields_dict):
            try:
                # Transform to python type and validate each field.
                validated_val = field_obj.run_validation(
                    data.get(field_name, None))

                # Now manual validation.
                validated_val = self._manual_validate_method(
                    field_name, validated_val)

                # And if there was a field in the incoming data, then we save it in the converted form.
                if field_name in data:
                    validated_data[field_name] = validated_val
                elif field_obj.default:
                    validated_data[field_name] = field_obj.default

            except ValidationError as e:
                # If not passed validation, save the error.
                errors[field_name] = e.detail
            except (AttributeError, TypeError, ValueError):
                # If data not valid format.
                errors[
                    field_name] = 'Could not parse data for field `{}`.'.format(
                        field_name)

        if any(errors):
            raise ValidationError(errors)

        return validated_data

    def is_valid(self, raise_exception=False):
        """
        Validates the data that came into the serializer.

        :param bool raise_exception: Whether to throw an exception if validation failed?

        :return: Validation result.
        :rtype: bool

        """
        if not hasattr(self, 'initial_data'):
            raise AssertionError(
                'Cannot call `.is_valid()` as no `data=` keyword argument '
                'was passed when instantiating the serializer instance.')

        # Preparing storage for results.
        self._errors, self._validated_data = OrderedDict(), OrderedDict()

        # Validated all fields.
        try:
            self._validated_data = self._field_validation(
                self.fields, self.initial_data)
        except ValidationError as e:
            self._errors = e.detail

        # Now run the full manual validation method.
        try:
            self._validated_data = self.validate(
                self._validated_data
            ) if not self._errors else self._validated_data
        except ValidationError as e:
            self._errors['errors'] = e.detail

        # If you need to throw an error, we throw.
        if self._errors and raise_exception:
            self._validated_data = OrderedDict()
            raise ValidationError(self._errors)

        # Return validation result.
        return not bool(self._errors)

    def run_validation(self, data):
        """
        Runs validation on the current serializer..

        :param object data: Data for validation.

        :return: Transformed and validated data.
        :rtype: dict

        """
        # We first check to see if an empty field has arrived?
        is_empty, data = self.validate_empty_values(data)

        # Transformed to python type.
        value = self.to_internal_value(data)

        # Validating validators.
        try:
            self.run_validators(value)
            value = self.validate(value)
            assert value is not None, '`.validate ()` should return a valid value.'

        except ValidationError as e:
            raise ValidationError(detail=e.detail)

        # We return the validated and transformed value.
        return value

    @property
    def validated_data(self):
        """
        Validated data.

        :return: Validated data.
        :rtype: dict

        """
        if not hasattr(self, '_validated_data'):
            raise AssertionError(
                'You must call `.is_valid()` before accessing `.validated_data`.'
            )
        return self._validated_data.copy()

    @property
    def errors(self):
        """
        Errors during validation.

        :return: Errors during validation.
        :rtype: dict

        """
        if not hasattr(self, '_errors'):
            raise AssertionError(
                'You must call `.is_valid()` before accessing `.errors`.')
        return self._errors.copy()
Beispiel #51
0
class Section:
    """
    This class holds a set of settings.

    To add settings and sections to a dictionary of sections we can use
    ``append_to_sections``:

    >>> sections = {}
    >>> append_to_sections(sections,
    ...                    'test1',
    ...                    'val',
    ...                    'origin',
    ...                    section_name='all')
    >>> 'all' in sections
    True
    >>> len(sections)
    1
    >>> str(sections)
    "{'all': <Section object(... contents=OrderedDict([('test1', ..."

    We can also add settings that can be appended to other settings. Basically
    it takes the default value of the setting which resides in the defaults of
    the section and appends the value of the setting in the second and returns
    the value of the setting:

    >>> append_to_sections(sections,
    ...                    'test1',
    ...                    'val2',
    ...                    'origin',
    ...                    section_name='all.python',
    ...                    to_append=True)

    When the section has no defaults:

    >>> str(sections['all.python']['test1'])
    'val2'

    After assigning defaults:

    >>> sections['all.python'].set_default_section(sections)
    >>> str(sections['all.python']['test1'])
    'val, val2'
    """

    @staticmethod
    def __prepare_key(key):
        return str(key).lower().strip()

    def __init__(self,
                 name,
                 defaults=None):
        if defaults is not None and not isinstance(defaults, Section):
            raise TypeError('defaults has to be a Section object or None.')
        if defaults is self:
            raise ValueError('defaults may not be self for non-recursivity.')

        self.name = str(name)
        self.defaults = defaults
        self.contents = OrderedDict()
        self.aspects = None
        self.language = None

    def bear_dirs(self):
        bear_dirs = path_list(self.get('bear_dirs', ''))
        for bear_dir in bear_dirs:
            sys.path.append(bear_dir)
        bear_dir_globs = [
            os.path.join(glob_escape(bear_dir), '**')
            for bear_dir in bear_dirs]
        bear_dir_globs += [
            os.path.join(glob_escape(bear_dir), '**')
            for bear_dir in collect_registered_bears_dirs('coalabears')]
        return bear_dir_globs

    def is_enabled(self, targets):
        """
        Checks if this section is enabled or, if targets is not empty, if it is
        included in the targets list.

        :param targets: List of target section names, all lower case.
        :return:        True or False
        """
        if len(targets) == 0:
            return bool(self.get('enabled', 'true'))

        return self.name.lower() in targets

    def append(self, setting, custom_key=None):
        if not isinstance(setting, Setting):
            raise TypeError
        if custom_key is None:
            key = self.__prepare_key(setting.key)
        else:
            key = self.__prepare_key(custom_key)

        # Setting asserts key != "" for us
        self.contents[key] = setting

    def add_or_create_setting(self,
                              setting,
                              custom_key=None,
                              allow_appending=True):
        """
        Adds the value of the setting to an existing setting if there is
        already a setting  with the key. Otherwise creates a new setting.
        """
        if custom_key is None:
            key = setting.key
        else:
            key = custom_key

        if self.__contains__(key, ignore_defaults=True) and allow_appending:
            val = self[key]
            val.value = str(val._value) + '\n' + setting._value
            self.append(val, custom_key=key)
        else:
            self.append(setting, custom_key=key)

    @enforce_signature
    def __setitem__(self, key: str, value: (str, Setting)):
        """
        Creates a Setting object from the given value if needed and assigns the
        setting to the key:

        >>> section = Section('section_name')
        >>> section['key'] = 'value'
        >>> section['key'].value
        'value'

        :param key:   Argument whose value is to be set
        :param value: The value of the given key
        :return:      Returns nothing.
        """
        if isinstance(value, Setting):
            self.append(value, custom_key=key)
        else:  # It must be a string since signature is enforced
            self.append(Setting(key, value))

    def __iter__(self, ignore_defaults=False):
        joined = self.contents.copy()
        if self.defaults is not None and not ignore_defaults:
            # Since we only return the iterator of joined (which doesnt contain
            # values) it's ok to override values here
            joined.update(self.defaults.contents)

        return iter(joined)

    def __contains__(self, item, ignore_defaults=False):
        try:
            self.__getitem__(item, ignore_defaults)

            return True
        except IndexError:
            return False

    def __getitem__(self, item, ignore_defaults=False):
        key = self.__prepare_key(item)
        if key == '':
            raise IndexError('Empty keys are invalid.')

        res = copy.deepcopy(self.contents.get(key, None))
        if res is not None:
            if res.to_append and self.defaults and res.key in self.defaults:
                res.value = self.defaults[key]._value + ', ' + res._value
            res.to_append = False
            return res

        if self.defaults is None or ignore_defaults:
            raise IndexError('Required index is unavailable.')

        return self.defaults[key]

    def __str__(self):
        value_list = ', '.join(key + ' : ' + repr(str(self[key]))
                               for key in self.contents)
        return self.name + ' {' + value_list + '}'

    def get(self, key, default='', ignore_defaults=False):
        """
        Retrieves the item without raising an exception. If the item is not
        available an appropriate Setting will be generated from your provided
        default value.

        :param key:             The key of the setting to return.
        :param default:         The default value
        :param ignore_defaults: Whether or not to ignore the default section.
        :return:                The setting.
        """
        try:
            return self.__getitem__(key, ignore_defaults)
        except IndexError:
            return Setting(key, str(default))

    def copy(self):
        """
        :return: a deep copy of this object
        """
        result = copy.copy(self)
        result.contents = copy.deepcopy(self.contents)
        if self.defaults is not None:
            result.defaults = self.defaults.copy()

        return result

    def update(self, other_section, ignore_defaults=False):
        """
        Incorporates all keys and values from the other section into this one.
        Values from the other section override the ones from this one.

        Default values from the other section override the default values from
        this only.

        :param other_section:   Another Section
        :param ignore_defaults: If set to true, do not take default values from
                                other
        :return:                self
        """
        if not isinstance(other_section, Section):
            raise TypeError('other_section has to be a Section')

        self.contents.update(other_section.contents)

        if not ignore_defaults and other_section.defaults is not None:
            if self.defaults is None:
                self.defaults = other_section.defaults.copy()
            else:
                self.defaults.update(other_section.defaults)

        return self

    def update_setting(self,
                       key,
                       new_key=None,
                       new_value=None):
        """
        Updates a setting with new values.
        :param key:       The old key string.
        :param new_key:   The new key string.
        :param new_value: The new value for the setting
        """
        if new_key is not None:
            self.contents[key].key = new_key
            self.contents = update_ordered_dict_key(self.contents,
                                                    key,
                                                    new_key)
        if new_value is not None:
            if new_key is not None:
                self.contents[new_key].value = new_value
            else:
                self.contents[key].value = new_value

    def delete_setting(self, key):
        """
        Delete a setting
        :param key: The key of the setting to be deleted
        """
        del self.contents[key]

    def set_default_section(self, sections, section_name=None):
        """
        Find and set the defaults of a section from a dictionary of sections.
        The defaults are found on the basis of '.' in section names:

        >>> sections = {'all': Section('all')}
        >>> section = Section('all.python')
        >>> section.set_default_section(sections)
        >>> section.defaults.name
        'all'
        >>> section = Section('all.python.syntax')
        >>> section.set_default_section(sections)
        >>> section.defaults.name
        'all'

        This works case insensitive. The key of the sections dict
        is expected to be lowered though!

        >>> sections = {'c': Section('C'), 'cpp': Section('Cpp'),
        ...             'c.something': Section('C.something')}
        >>> section = Section('C.something')
        >>> section.set_default_section(sections)
        >>> section.defaults.name
        'C'
        >>> section = Section('C.SOMETHING.else')
        >>> section.set_default_section(sections)
        >>> section.defaults.name
        'C.something'
        >>> section = Section('Cpp.SOMETHING.else')
        >>> section.set_default_section(sections)
        >>> section.defaults.name
        'Cpp'

        :param sections:     A dictionary of sections.
        :param section_name: Optional section name argument to find the default
                             section for. If not given then use member section
                             name.
        """
        default_section = '.'.join(
            (section_name or self.name).split('.')[:-1]
        ).lower()

        if default_section:
            if default_section in sections:
                self.defaults = sections[default_section]
            else:
                self.set_default_section(sections, default_section)
        elif 'cli' in sections and self.name.lower() != 'cli':
            # CLI section is now default
            self.defaults = sections['cli']
Beispiel #52
0
class PolygonPore(object):
    def __init__(self, poly, name="protein", **params):
        self.name = name
        self.protein = Polygon(poly)
        self.polygons = OrderedDict({name: self.protein})
        self.balls = OrderedDict()
        self.params = Params(params)
        self.boundaries = OrderedDict()

        # cs ... protein crosssections for partition of boundary
        self.add_proteinpartition()
        self.add_molecule_ball()

    def add_molecule_ball(self):
        if "x0" in self.params and self.params.x0 is not None:
            self.molecule = True
            lc = self.params.lcMolecule if "lcMolecule" in self.params else 1.
            molecule = Ball(self.params.x0, self.params.rMolecule, lc)
        else:
            self.molecule = False
            molecule = EmptySet()
        self.balls.update(molecule=molecule)

    def add_balls(self, **balls):
        for pname, p in balls.items():
            if not isinstance(p, Ball):
                x0, r = p
                balls[pname] = Ball(x0, r)
        self.balls.update(balls)

    def build_polygons(self):
        R = self.params.R
        if not "Hbot" in self.params:
            H = self.params.H
            self.params["Htop"] = H / 2.
            self.params["Hbot"] = H / 2.
        Hbot = self.params.Hbot
        Htop = self.params.Htop

        self.add_membrane()

        sections = self.add_poresections()
        sections = self.add_poreregions(sections)
        self.add_bulkfluids(R, Htop, Hbot, sections)
        for ball in self.balls.values():
            self.add_molecule(ball)

        # for easier handling of alls domains
        self.domains = self.balls.copy()
        self.domains.update(self.polygons)
        return self.polygons

    def build_boundaries(self):
        boundaries = self.boundaries

        # membrane boundary
        mem = self.polygons["membrane"]
        if not isempty(mem):
            memb = mem.edgerange(mem.b, mem.c) | mem.edgerange(mem.d, mem.a)
            boundaries.update(memb=memb)

        # upper, lower, side
        btop = self.polygons["bulkfluid_top"]
        bbot = self.polygons["bulkfluid_bottom"]
        upperb = btop.edgerange(btop.b, btop.c)
        lowerb = bbot.edgerange(bbot.d, bbot.a)
        sideb = btop.edgerange(btop.c, btop.d) | bbot.edgerange(bbot.c, bbot.d)

        boundaries.update(upperb=upperb, lowerb=lowerb, sideb=sideb)
        boundaries.update(self.compute_proteinboundary())

        # molecule
        for bname, ball in self.balls.items():
            boundaries.update({bname + "b": ball.boundary(self.params.dim)})

        return boundaries

    def molecule_intersects(self, z):
        eps = 0.5
        x0 = self.params.x0 if "x0" in self.params else None
        if x0 is None:
            return False
        z0 = x0[-1]
        r = self.params.rMolecule
        return z0 - r - eps <= z <= z0 + r + eps

    def where_is_molecule(self, ball=None):
        "name of domain where ball lies (or None)"
        # TODO: currently this is only based on z position!!
        if ball is None:
            ball = self.balls["molecule"]
        if isempty(ball):
            return None

        domains = ["pore%d" % i for i in range(self.nsections)]
        if self.params.poreregion:
            domains = ["poreregion_bottom"] + domains + ["poreregion_top"]
        else:
            domains = ["bulkfluid_bottom"] + domains + ["bulkfluid_top"]
        i0 = bisect(self.cs, ball.x0[-1])
        return domains[i0]

    def add_molecule(self, ball):
        # do nothing if empty
        if isempty(ball):
            return

        # check in which domain ball is contained
        domstr = self.where_is_molecule(ball)
        domain = self.polygons[domstr]

        # in 3D, add ball as hole in containing domain
        if "dim" in self.params and self.params.dim == 3:
            if not hasattr(domain, "holes"):
                domain.holes = []
            domain.holes.append(ball)
            return

        # in 2D, modify containing domain
        x1, x2, x3 = tuple(ball.nodes)
        domain.left_intersection(x1[1])
        domain.left_intersection(x3[1])
        i = domain.edges.index((x1, x3))
        domain.edges[i] = (x1, x2, x3)

    def add_membrane(self):
        R = self.params.R
        if not ("no_membrane" in self.params
                and self.params.no_membrane) and (self.protein.rmax()[0] < R):
            hmem = self.params.hmem
            zmem = self.params.zmem
            zbot = zmem - 0.5 * hmem
            ztop = zmem + 0.5 * hmem
            self.membrane = self.protein.clip_from_right(zbot, ztop, R)
        else:
            self.membrane = EmptySet()
        self.polygons["membrane"] = self.membrane
        return self.membrane

    def add_poresections(self, remove_intersections=True):
        # cs ... crosssections
        cs = self.params.cs if "cs" in self.params else None
        ztop = max(x[1] for x in self.protein.nodes)
        zbot = min(x[1] for x in self.protein.nodes)
        if cs is None:
            cs = [zbot + i / 3. * (ztop - zbot) for i in [1, 2]]
        else:
            cs = list(cs)
        assert all(zbot < z < ztop for z in cs)
        cs = [zbot] + sorted(cs) + [ztop]
        # do not add crosssections that intersect with molecule
        if remove_intersections:
            for z in list(cs):
                if self.molecule_intersects(z):
                    cs.remove(z)

        pairs = zip(cs[:-1], cs[1:])
        sections = tuple(
            [self.protein.clip_from_left(a, b, 0) for a, b in pairs])
        names = ["pore%d" % i for i in range(len(sections))]
        self.nsections = len(sections)
        self.lpore = ztop - zbot
        self.cs = cs
        self.polygons.update({name: s for name, s in zip(names, sections)})
        return sections

    def add_poreregions(self, sections):
        if not "poreregion" in self.params or not self.params.poreregion:
            self.params.poreregion = False
            self.polygons["poreregion_top"] = EmptySet()
            self.polygons["poreregion_bottom"] = EmptySet()
            return sections
        if not "H0" in self.params or self.params.H0 is None:
            self.params["H0"] = 0.5 * self.params.H + self.protein.zmax()[1]
        if not "R0" in self.params or self.params.R0 is None:
            self.params["R0"] = self.protein.rmax()[0]
        H0 = self.params.H0
        R0 = self.params.R0
        H0top = H0bot = H0 / 2.
        #print self.protein.zmax()[1], H0top, self.params.H*0.5

        center = MultiPolygon(self.protein, self.membrane, *sections)
        prtop = center.clip_from("top", R0, 0, H0top)
        prbot = center.clip_from("bottom", 0, R0, -H0bot)

        # OLD VERSION: doesn't work bc pore region and memb can overlap
        #        section = sections[-1]
        #        a = section.b
        #        d = self.protein.top_intersection(R0)
        #        upper = self.protein.nrange(d, section.c, -1) + [a]
        #        b, c = (0, H0top), (R0, H0top)
        #        nodes = [b, c] + upper
        #        prtop = Polygon(nodes)
        #        prtop.set_corners(a, b, c, d)
        #
        #        section = sections[0]
        #        b = section.a
        #        c = self.protein.bottom_intersection(R0)
        #        lower = [b] + self.protein.nrange(section.d, c, -1)
        #        a, d = (0, -H0bot), (R0, -H0bot)
        #        nodes = lower + [d, a]
        #        prbot = Polygon(nodes)
        #        prbot.set_corners(a, b, c, d)

        self.polygons["poreregion_top"] = prtop
        self.polygons["poreregion_bottom"] = prbot
        sections = [prbot] + list(sections) + [prtop]
        return sections

    def add_bulkfluids(self, R, Htop, Hbot, sections):
        # sections ... pore sections that remain after adding molecule
        polygons = list(sections) + [self.protein, self.membrane]

        upper = compute_upper_boundary(*polygons)
        b, c = (0, Htop), (R, Htop)
        nodes = [b, c] + upper
        btop = Polygon(nodes)
        btop.a = upper[-1]
        btop.b = b
        btop.c = c
        btop.d = upper[0]

        lower = compute_lower_boundary(*polygons)
        d, a = (R, -Hbot), (0, -Hbot)
        nodes = lower + [d, a]
        bbot = Polygon(nodes)
        bbot.a = a
        bbot.b = lower[0]
        bbot.c = lower[-1]
        bbot.d = d

        self.polygons["bulkfluid_top"] = btop
        self.polygons["bulkfluid_bottom"] = bbot
        return btop, bbot

    def add_proteinpartition(self):
        # do at the beginning
        # insert intersection points with cs in polygon
        cs = self.params.proteincs if "proteincs" in self.params else None
        if cs is None or len(cs) == 0:
            self.proteinpartition = False
            return
        protein = self.protein
        ztop = max(x[1] for x in protein.nodes)
        zbot = min(x[1] for x in protein.nodes)
        assert all(zbot < z < ztop for z in cs)
        for z in cs:
            protein.all_intersections(z)
        self.proteincs = cs
        self.proteinpartition = True

    def compute_proteinboundary(self):
        # do after modifying edges!!
        protein = self.protein
        if not self.proteinpartition:
            dic = {"%sb" % self.name: set(self.protein.edges)}

        else:
            cs = self.proteincs
            ztop = max(x[1] for x in protein.nodes)
            zbot = min(x[1] for x in protein.nodes)
            cs = [zbot] + sorted(cs) + [ztop]
            npart = len(cs) - 1
            sets = [set() for i in range(npart)]

            # partition edges
            for edge in protein.edges:
                x = min(edge, key=lambda t: t[1])
                ix = bisect(cs, x[1]) - 1
                if ix == npart:
                    ix -= 1
                sets[ix].add(edge)

            dic = {"%sb%d" % (self.name, i): sets[i] for i in range(npart)}

        # subtract edges on protein-membrane interface
        mem = self.polygons["membrane"]
        if not isempty(mem):
            pmemb = protein.edgerange(mem.b, mem.a)
            for s in dic.values():
                s -= pmemb

        return dic

    def radius_at(self, z):
        "return z-dependent pore radius"
        I = self.protein.intersections(z)
        return min(x[0] for x in I.keys())
Beispiel #53
0
class MultiPolygonPore(PolygonPore):
    """For pores consisting of multiple subdomains, e.g. Wei-Rant pore,
    but without further pore bisection (proteinpartition).
    Assumes that polygons are disjoint and their union simply connected."""
    def __init__(self, **params):
        self.polygons = OrderedDict()
        self.balls = OrderedDict()
        self.params = Params(params)
        self.boundaries = OrderedDict()
        self.names = []
        self.add_molecule_ball()

    def add_polygons(self, **polygons):
        "enables adding polygons in defined order"
        for pname, p in polygons.items():
            if not isinstance(p, Polygon):
                polygons[pname] = Polygon(p)
        self.polygons.update(polygons)
        self.names.extend(polygons.keys())

    def build_polygons(self):
        # read global height, width params
        R = self.params.R
        if not "Hbot" in self.params:
            H = self.params.H
            self.params["Htop"] = H / 2.
            self.params["Hbot"] = H / 2.
        Hbot = self.params.Hbot
        Htop = self.params.Htop

        # first build encompassing polygon out of existing.
        self.protein = MultiPolygon(*self.polygons.values())
        self.protein.cut_from_right(R)
        self.polygons = OrderedDict()

        bnames = ["%sb" % name for name in self.names]
        self.proteinb = OrderedDict(zip(bnames, self.protein.boundaries))

        self.add_membrane()
        sections = self.add_poresections()
        sections = self.add_poreregions(sections)
        self.add_bulkfluids(R, Htop, Hbot, sections)
        self.polygons.update(zip(self.names, self.protein.polygons))
        for ball in self.balls.values():
            self.add_molecule(ball)

        # for easier handling of alls domains
        self.domains = self.balls.copy()
        self.domains.update(self.polygons)
        return self.polygons

    def build_boundaries(self):
        boundaries = self.boundaries
        proteinb = self.proteinb

        # upper, lower, side
        btop = self.polygons["bulkfluid_top"]
        bbot = self.polygons["bulkfluid_bottom"]
        upperb = btop.edgerange(btop.b, btop.c)
        lowerb = bbot.edgerange(bbot.d, bbot.a)
        sideb = btop.edgerange(btop.c, btop.d) | bbot.edgerange(bbot.c, bbot.d)
        boundaries.update(upperb=upperb, lowerb=lowerb, sideb=sideb)

        fluid_polys = [btop, bbot] + \
            [self.polygons["poreregion_top"],
             self.polygons["poreregion_bottom"]] + \
            [self.polygons["pore%d" % i] for i in range(self.nsections)]
        fluid_edges = set([e[::-1] for p in fluid_polys for e in p.edges])

        # membrane-fluid boundary
        mem = self.polygons["membrane"]
        if not isempty(mem):
            memb = fluid_edges & (mem.edgerange(mem.b, mem.c)
                                  | mem.edgerange(mem.d, mem.a))
            boundaries.update(memb=memb)
        else:
            boundaries.update(memb=set())
        # protein-fluid boundary, divided between protein parts
        for bname in proteinb:
            proteinb[bname] &= fluid_edges
        boundaries.update(proteinb)

        # molecule
        for bname, ball in self.balls.items():
            boundaries.update({bname + "b": ball.boundary(self.params.dim)})

        return boundaries
Beispiel #54
0
class MoveScaleOverlay(Overlay):
    """Viewer overlays for GES.VideoSource transformations."""
    def __init__(self, stack, action_log, source):
        Overlay.__init__(self, stack, source)

        self.__clicked_handle = None
        self.__click_diagonal_sign = None
        self.__box_hovered = False

        self.__action_log = action_log
        self.hovered_handle = None

        # Corner handles need to be ordered for drawing.
        self.corner_handles = OrderedDict([
            ((Edge.top, Edge.left), CornerHandle(self)),
            ((Edge.bottom, Edge.left), CornerHandle(self)),
            ((Edge.bottom, Edge.right), CornerHandle(self)),
            ((Edge.top, Edge.right), CornerHandle(self))
        ])

        self.handles = self.corner_handles.copy()
        for edge in range(1, 5):
            self.handles[(edge, )] = EdgeHandle(self)

        for key in self.handles:
            self.handles[key].set_placement(key)

        self._source.connect("deep-notify", self.__source_property_changed_cb)
        self.update_from_source()

    def __get_source_property(self, prop):
        if self.__source_property_keyframed(prop):
            binding = self._source.get_control_binding(prop)
            res, position = self.__get_pipeline_position()
            if res:
                start = self._source.props.start
                in_point = self._source.props.in_point
                duration = self._source.props.duration
                # If the position is outside of the clip, take the property
                # value at the start/end (whichever is closer) of the clip.
                source_position = max(0, min(position - start,
                                             duration - 1)) + in_point
                value = binding.get_value(source_position)
                res = value is not None
                return res, value

        return self._source.get_child_property(prop)

    def __set_source_property(self, prop, value):
        if self.__source_property_keyframed(prop):
            control_source = self._source.get_control_binding(
                prop).props.control_source
            res, timestamp = self.__get_pipeline_position()
            if not res:
                return
            source_timestamp = timestamp - self._source.props.start + self._source.props.in_point
            control_source.set(source_timestamp, value)
        else:
            self._source.set_child_property(prop, value)

    def __source_property_keyframed(self, prop):
        binding = self._source.get_control_binding(prop)
        return binding is not None

    def __get_pipeline_position(self):
        project = self.stack.app.project_manager.current_project
        if project:
            pipeline = project.pipeline
            try:
                position = pipeline.getPosition()
                return True, position
            except PipelineError:
                pass
        return False, None

    def __get_source_position(self):
        res_x, x = self.__get_source_property("posx")
        res_y, y = self.__get_source_property("posy")
        assert res_x and res_y
        return numpy.array([x, y])

    def __get_source_size(self):
        res_x, x = self.__get_source_property("width")
        res_y, y = self.__get_source_property("height")
        assert res_x and res_y
        return numpy.array([x, y])

    def __get_normalized_source_position(self):
        return self.__get_source_position() / self.project_size

    def __set_source_position(self, position):
        self.__set_source_property("posx", int(position[0]))
        self.__set_source_property("posy", int(position[1]))

    def __set_source_size(self, size):
        self.__set_source_property("width", int(size[0]))
        self.__set_source_property("height", int(size[1]))

    def __get_size(self):
        return numpy.array([self.__get_width(), self.__get_height()])

    def __get_size_stream(self):
        return self.__get_size() * self.project_size

    def __get_height(self):
        return self.handles[(Edge.bottom,
                             Edge.left)].position[1] - self.handles[
                                 (Edge.top, Edge.left)].position[1]

    def __get_width(self):
        return self.handles[(Edge.bottom,
                             Edge.right)].position[0] - self.handles[
                                 (Edge.bottom, Edge.left)].position[0]

    def __set_size(self, size):
        self.handles[(Edge.top, Edge.left)].position = numpy.array([0, 0])
        self.handles[(Edge.bottom,
                      Edge.left)].position = numpy.array([0, size[1]])
        self.handles[(Edge.bottom,
                      Edge.right)].position = numpy.array([size[0], size[1]])
        self.handles[(Edge.top,
                      Edge.right)].position = numpy.array([size[0], 0])
        self.__update_edges_from_corners()

    def __set_position(self, position):
        for handle in self.handles.values():
            handle.set_translation(position)

    def __reset_handle_sizes(self):
        for handle in self.handles.values():
            handle.reset_size()
        self.__update_handle_sizes()

    def __update_handle_sizes(self):
        size = self.__get_size() * self.stack.window_size
        smaller_size = numpy.amin(size)

        for handle in self.handles.values():
            handle.restrict_radius_to_size(smaller_size)

    def __update_edges_from_corners(self):
        half_w = numpy.array([self.__get_width() * 0.5, 0])
        half_h = numpy.array([0, self.__get_height() * 0.5])

        self.handles[(Edge.left, )].set_position(self.handles[
            (Edge.top, Edge.left)].position + half_h)
        self.handles[(Edge.right, )].set_position(self.handles[
            (Edge.top, Edge.right)].position + half_h)
        self.handles[(Edge.bottom, )].set_position(self.handles[
            (Edge.bottom, Edge.left)].position + half_w)
        self.handles[(Edge.top, )].set_position(self.handles[
            (Edge.top, Edge.right)].position - half_w)

    def __draw_rectangle(self, cr):
        for handle in self.corner_handles.values():
            cr.line_to(*handle.get_window_position())
        cr.line_to(*self.handles[(Edge.top, Edge.left)].get_window_position())

    def get_center(self):
        diagonal = self.handles[(Edge.bottom,
                                 Edge.right)].position - self.handles[
                                     (Edge.top, Edge.left)].position
        return self.handles[(Edge.top, Edge.left)].position + (diagonal / 2)

    def get_aspect_ratio(self):
        size = self.__get_size()
        return size[0] / size[1]

    def on_button_press(self):
        disconnectAllByFunc(self._source, self.__source_property_changed_cb)
        self.click_source_position = self.__get_source_position()
        self.__clicked_handle = None

        if self.hovered_handle or self.__box_hovered:
            self.__action_log.begin(
                "Video position change",
                finalizing_action=CommitTimelineFinalizingAction(
                    self._source.get_timeline().get_parent()),
                toplevel=True)

        if self.hovered_handle:
            self.hovered_handle.on_click()
            self.__clicked_handle = self.hovered_handle
        elif self.__box_hovered:
            self._select()
            self.stack.set_cursor("grabbing")
            self.stack.selected_overlay = self
        elif self._is_selected():
            self._deselect()
            self.hovered_handle = None

    def on_button_release(self, cursor_position):
        self.click_source_position = None
        self.update_from_source()
        self.on_hover(cursor_position)

        self.__action_log.commit("Video position change")
        if self.__clicked_handle:
            if not self.__clicked_handle.hovered:
                self.stack.reset_cursor()
            self.__clicked_handle.on_release()
            self.__clicked_handle = None
        elif self._is_hovered():
            self.stack.set_cursor("grab")

        self._source.connect("deep-notify", self.__source_property_changed_cb)
        self.queue_draw()

    def __source_property_changed_cb(self, unused_source, unused_gstelement,
                                     unused_pspec):
        self.update_from_source()

    def on_motion_notify(self, cursor_pos):
        click_to_cursor = self.stack.get_normalized_drag_distance(cursor_pos)
        if self.__clicked_handle:
            # Resize Box / Use Handle
            self.__clicked_handle.on_drag(click_to_cursor)
            self.__update_edges_from_corners()

            # We only need to change translation coordinates in the source for resizing
            # when handle does not return NULL for get_source_position
            source_position = self.__clicked_handle.get_source_position()
            if isinstance(source_position, numpy.ndarray):
                self.__set_source_position(source_position)

            self.__set_source_size(self.__get_size_stream())
            self.__update_handle_sizes()
        else:
            # Move Box
            stream_position = self.click_source_position + click_to_cursor * self.project_size
            self.__set_position(stream_position / self.project_size)
            self.__set_source_position(stream_position)
        self.queue_draw()
        self._commit()

    def on_hover(self, cursor_pos):
        if not self.is_visible():
            return
        # handles hover check
        self.hovered_handle = None
        if self._is_selected():
            for handle in self.handles.values():
                handle.on_hover(cursor_pos)
                if handle.hovered:
                    self.hovered_handle = handle
            if self.hovered_handle:
                self._hover()
                self.queue_draw()
                return True

        # box hover check
        source = self.__get_normalized_source_position()
        cursor = self.stack.get_normalized_cursor_position(cursor_pos)

        self.__box_hovered = False
        if (source < cursor).all() and (cursor <
                                        source + self.__get_size()).all():
            self.__box_hovered = True
            self.stack.set_cursor("grab")
            self._hover()
        else:
            self.__box_hovered = False
            self.unhover()

        self.queue_draw()
        return self.__box_hovered

    def update_from_source(self):
        self.__set_size(self.__get_source_size() / self.project_size)
        self.__set_position(self.__get_source_position() / self.project_size)
        self.__reset_handle_sizes()
        self.queue_draw()

    def do_draw(self, cr):
        if not self._is_selected() and not self._is_hovered():
            return

        cr.save()
        # clear background
        cr.set_operator(cairo.OPERATOR_OVER)
        cr.set_source_rgba(0.0, 0.0, 0.0, 0.0)
        cr.paint()

        if self.__box_hovered:
            brightness = 0.65
        else:
            brightness = 0.3

        # clip away outer mask
        self.__draw_rectangle(cr)
        cr.clip()
        cr.set_source_rgba(brightness, brightness, brightness, 0.6)
        self.__draw_rectangle(cr)

        cr.set_line_width(16)
        cr.stroke()
        cr.restore()

        if self._is_selected():
            for handle in self.handles.values():
                handle.draw(cr)
Beispiel #55
0
class DRSPath(object):
    """
    Handler providing methods to deal with paths.

    """
    # Default DRS tree version
    # Modified on the ProcessingContext instance
    TREE_VERSION = 'v{}'.format(datetime.now().strftime('%Y%m%d'))

    def __init__(self, parts):
        # Retrieve the dataset directory parts
        self.d_parts = OrderedDict(
            parts.items()[:parts.keys().index('version')])
        # Retrieve the file directory parts
        self.f_parts = OrderedDict(
            parts.items()[parts.keys().index('version') + 1:])
        # Retrieve the upgrade version
        self.v_upgrade = DRSPath.TREE_VERSION
        # If the dataset path is not equivalent to the file diretcory (e.g., CMIP5 like)
        # Get the physical files version
        self.v_latest = self.get_latest_version()
        if self.path(f_part=False) != self.path():
            self.v_files = OrderedDict({
                'variable':
                '{}_{}'.format(self.get('variable'), self.v_upgrade[1:])
            })
        else:
            self.v_files = OrderedDict(
                {'data': 'd{}'.format(self.v_upgrade[1:])})

    def get(self, key):
        """
        Returns the attribute value corresponding to the key.
        The submitted key can refer to the DRS dataset parts of the DRS file parts.

        :param str key: The key
        :returns: The value
        :rtype: *str* or *list* or *dict* depending on the key
        :raises Error: If unknown key

        """
        if key in self.d_parts:
            return self.d_parts[key]
        elif key in self.f_parts.keys():
            return self.f_parts[key]
        else:
            raise KeyNotFound(key, self.d_parts.keys() + self.f_parts.keys())

    def items(self,
              d_part=True,
              f_part=True,
              version=True,
              file_folder=False,
              latest=False,
              root=False):
        """
        Itemizes the facet values along the DRS path.
        Flags can be combine to obtain different behaviors.

        :param boolean d_part: True to append the dataset facets
        :param boolean f_part: True to append the file facets
        :param boolean version: True to append the version facet
        :param boolean file_folder: True to append the folder for physical files
        :param boolean latest: True to switch from upgrade to latest version
        :param boolean root: True to prepend the DRS root directory
        :return: The corresponding facet values
        :rtype: *list*

        """
        parts = OrderedDict()
        if d_part:
            parts = self.d_parts.copy()
        if version:
            if latest:
                parts.update(OrderedDict({'version': self.v_latest}))
            else:
                parts.update(OrderedDict({'version': self.v_upgrade}))
            if file_folder:
                parts.update(OrderedDict({'version': 'files'}))
        if f_part:
            parts.update(self.f_parts)
        if file_folder:
            parts.update(self.v_files)
        if not root and 'root' in parts.keys():
            del parts['root']
        return parts.values()

    def path(self, **kwargs):
        """
        Convert a list of facet values into path.
        The arguments are the same as :func:`esgprep.drs.handler.DRSPath.items`

        :returns: The path
        :rtype: *str*

        """
        return os.path.join(*self.items(**kwargs))

    def get_latest_version(self):
        """
        Get the current latest dataset version if exists.

        :returns: The latest dataset version properly formatted
        :rtype: *str*
        :raises Error: If latest version exists and is the same as upgrade version

        """
        # Test if dataset path already exists
        dset_path = self.path(f_part=False, version=False, root=True)
        if os.path.isdir(dset_path):
            # Get and sort all existing dataset versions
            versions = sorted([
                v for v in os.listdir(dset_path)
                if re.compile(r'v[\d]+').search(v)
            ])
            # Upgrade version should not already exist
            if self.v_upgrade in versions:
                raise DuplicatedDataset(dset_path, self.v_upgrade)
            # Pickup respectively previous, next and latest version
            return versions[-1]
        else:
            return None
Beispiel #56
0
    def replace(self, replacements):
        """Replace certain variables in the computation graph.

        Parameters
        ----------
        replacements : dict
            The mapping from variables to be replaced to the corresponding
            substitutes.

        Examples
        --------
        >>> import theano
        >>> from theano import tensor, function
        >>> x = tensor.scalar('x')
        >>> y = x + 2
        >>> z = y + 3
        >>> a = z + 5

        Let's suppose we have dependent replacements like

        >>> replacements = {y: x * 2, z: y * 3}
        >>> cg = ComputationGraph([a])
        >>> theano.pprint(a)  # doctest: +NORMALIZE_WHITESPACE
        '(((x + TensorConstant{2}) + TensorConstant{3}) +
        TensorConstant{5})'
        >>> cg_new = cg.replace(replacements)
        >>> theano.pprint(
        ...     cg_new.outputs[0])  # doctest: +NORMALIZE_WHITESPACE
        '(((x * TensorConstant{2}) * TensorConstant{3}) +
        TensorConstant{5})'

        First two sums turned into multiplications

        >>> float(function(cg_new.inputs, cg_new.outputs)(3.)[0])
        23.0

        """
        # Due to theano specifics we have to make one replacement in time
        replacements = OrderedDict(replacements)

        outputs_cur = self.outputs

        # `replacements` with previous replacements applied. We have to track
        # variables in the new graph corresponding to original replacements.
        replacement_keys_cur = []
        replacement_vals_cur = []
        # Sort `replacements` in topological order
        # variables in self.variables are in topological order
        remaining_replacements = replacements.copy()
        for variable in self.variables:
            if variable in replacements:
                if has_roles(variable, [AUXILIARY]):
                    warnings.warn(
                        "replace method was asked to replace a variable ({}) "
                        "that is an auxiliary variable.".format(variable))
                replacement_keys_cur.append(variable)
                # self.variables should not contain duplicates,
                # otherwise pop() may fail.
                replacement_vals_cur.append(
                    remaining_replacements.pop(variable))

        # if remaining_replacements is not empty
        if remaining_replacements:
            warnings.warn(
                "replace method was asked to replace a variable(s) ({}) "
                "that is not a part of the computational "
                "graph.".format(str(remaining_replacements.keys())))

        # Replace step-by-step in topological order
        while replacement_keys_cur:
            replace_what = replacement_keys_cur[0]
            replace_by = replacement_vals_cur[0]
            # We also want to make changes in future replacements
            outputs_new = theano.clone(outputs_cur + replacement_keys_cur[1:] +
                                       replacement_vals_cur[1:],
                                       replace={replace_what: replace_by})
            # Reconstruct outputs, keys, and values
            outputs_cur = outputs_new[:len(outputs_cur)]
            replacement_keys_cur = outputs_new[len(outputs_cur
                                                   ):len(outputs_cur) +
                                               len(replacement_keys_cur) - 1]
            replacement_vals_cur = outputs_new[len(outputs_cur) +
                                               len(replacement_keys_cur):]

        return ComputationGraph(outputs_cur)
Beispiel #57
0
class NamedElementStack(Mapping):
    def __init__(self, init_elements, valid_element=callable):
        self._queue = OrderedDict()
        for element in reversed(init_elements):
            if valid_element(element):
                self.add(element)
            else:
                self.add(*element)

    def add(self, element, name=None):
        if name is None:
            name = element

        if name in self._queue:
            if name is element:
                raise ValueError(
                    "You can't add the same un-named instance twice")
            else:
                raise ValueError(
                    "You can't add the same name again, use replace instead")

        self._queue[name] = element

    def clear(self):
        self._queue.clear()

    def replace(self, old, new):
        if old not in self._queue:
            raise ValueError(
                "You can't replace unless one already exists, use add instead")
        to_be_replaced = self._queue[old]
        if to_be_replaced is old:
            # re-insert with new name in old slot
            self._replace_with_new_name(old, new)
        else:
            self._queue[old] = new
        return to_be_replaced

    def remove(self, old):
        if old not in self._queue:
            raise ValueError(
                "You can only remove something that has been added")
        del self._queue[old]

    def _replace_with_new_name(self, old, new):
        self._queue[new] = new
        found_old = False
        for key in list(self._queue.keys()):
            if not found_old:
                if key == old:
                    found_old = True
                continue
            elif key != new:
                self._queue.move_to_end(key)
        del self._queue[old]

    def __iter__(self):
        elements = self._queue.values()
        if not isinstance(elements, Sequence):
            elements = list(elements)
        return iter(reversed(elements))

    def __add__(self, other):
        if not isinstance(other, NamedElementStack):
            raise NotImplementedError(
                "You can only combine with another Stack")
        combined = self._queue.copy()
        combined.update(other._queue)
        return NamedElementStack(combined.items())

    def __contains__(self, element):
        return element in self._queue

    def __getitem__(self, element):
        return self._queue[element]

    def __len__(self):
        return len(self._queue)

    def __reversed__(self):
        elements = self._queue.values()
        if not isinstance(elements, Sequence):
            elements = list(elements)
        return iter(elements)
Beispiel #58
0
class AnsibleConfig(object):

    def __init__(
            self,
            ansible_file,
            playbook,
            log_dir=None,
            environmental_config=None,
            remote_user=None,
            extra_vars=[]):
        self._ansible_file = ansible_file
        self._playbook = playbook
        self._env = OrderedDict()
        self._log_dir = log_dir
        self._remote_user = remote_user
        self._extra_vars = extra_vars

        self._env['ANSIBLE_CONFIG'] = self._ansible_file
        if remote_user:
            self._env['ANSIBLE_REMOTE_USER'] = self._remote_user
        if environmental_config:
            self._env.update(self._environment_config_to_dict(environmental_config))

        logger.debug('ansible_file: {f}'.format(f=self.ansible_file))
        logger.debug('playbook: {pb}'.format(pb=self.playbook))
        logger.debug('remoteuser: {ru}'.format(ru=self.remote_user))
        logger.debug('environmental_config: {envcfg}'.format(envcfg=environmental_config))
        logger.debug('env_str: {env_str}'.format(env_str=self.env_str))
        logger.debug('extra_vars: {ev}'.format(ev=self.extra_vars))

    @property
    def env(self):
        return self._env.copy()

    @property
    def env_str(self):
        return ' '.join(['{k}={v}'.format(k=k, v=v) for k, v in self._env.items()])

    @property
    def ansible_file(self):
        return self._ansible_file

    @property
    def playbook(self):
        return self._playbook

    @property
    def remote_user(self):
        return self._remote_user

    @property
    def extra_vars(self):
        return self._extra_vars

    @property
    def log_dir(self):
        return self._log_dir

    @property
    def log_path(self):
        return self._env.get('ANSIBLE_LOG_PATH', None)

    @staticmethod
    def _environment_config_to_dict(configs):
        result = OrderedDict()
        for item in configs:
            try:
                key, value = item.split('=')
                result[key] = value
            except ValueError:
                raise Exception(
                    "The ansible configuration '{item}' is mall-formated".format(item=item))
        return result
Beispiel #59
0
class NicosCmdClient(NicosClient):

    def __init__(self, conndata):
        NicosClient.__init__(self, self.put_error)
        # connection data as an object
        self.conndata = conndata
        # whether to suppress printing history and other info on connection
        self.quiet_connect = False
        # various state variables
        self.in_question = False
        self.in_editing = False
        self.tip_shown = False
        # number of automatic reconnect tries before giving up
        self.reconnect_count = 0
        self.reconnect_time = 0
        # current script, line within it and filename of script
        self.current_script = ['']
        self.current_line = -1
        self.current_filename = ''
        # pending requests (i.e. scripts) in the daemon
        self.pending_requests = OrderedDict()
        # filename of last edited/simulated script
        self.last_filename = ''
        # instrument name from NICOS, pre-filled with server name
        self.instrument = conndata.host.split('.')[0]
        # script directory from NICOS
        self.scriptpath = '.'
        # execution mode of the NICOS session
        self.current_mode = MASTER
        # messages queueing up while the editor is running
        self.message_queue = []
        # whether a stop is pending
        self.stop_pending = False
        # whether we are in debugging mode
        self.debug_mode = False
        # whether we are in spy mode (entering commands disabled)
        self.spy_mode = False
        # detected text-mode browser for help display
        self.browser = None
        # used for determining how much history to print by default
        self.tsize = terminalSize()
        # output stream to print to
        self.out = sys.stdout
        # uuid of the last simulation
        self.simuuid = ''
        # whether we display timestamps with subsecond precision
        self.subsec_ts = False
        # current ETA information
        self.cur_eta = ''

        # set up readline
        for line in DEFAULT_BINDINGS.splitlines():
            readline.parse_and_bind(line)
        readline.set_completer(self.completer)
        readline.set_history_length(10000)
        self.histfile = os.environ.get('NICOS_HISTORY_FILE',
                                       path.expanduser('~/.nicoshistory'))
        if path.isfile(self.histfile):
            readline.read_history_file(self.histfile)
        self.completions = []

        # set up "wakeup" pipe to notify readline of output and changed prompt
        self.wakeup_pipe_r, self.wakeup_pipe_w = os.pipe()

        # pre-set prompt to sane default
        self.set_status('disconnected')

    # -- low-level terminal input/output routines

    def readline(self, prompt, add_history=True):
        """Read a line from the user.

        This function basically reimplements the readline module's
        readline_until_enter_or_signal C function, with the addition
        that we set new prompts and update the display periodically.

        Thanks to ctypes this is possible without a custom C module.
        """
        # pylint: disable=global-statement
        global readline_result
        term_encoding = sys.stdout.encoding or 'utf-8'
        librl.rl_callback_handler_install(to_encoding(prompt, term_encoding),
                                          c_readline_finish_callback)
        readline_result = Ellipsis
        while readline_result is Ellipsis:
            try:
                res = select.select([sys.stdin, self.wakeup_pipe_r], [], [], 1)[0]
            except select.error as e:
                if e.args[0] == errno.EINTR:
                    continue
                librl.rl_callback_handler_remove()
                raise
            except:
                librl.rl_callback_handler_remove()
                raise
            if sys.stdin in res:
                librl.rl_callback_read_char()
            if self.wakeup_pipe_r in res:
                os.read(self.wakeup_pipe_r, 1)
                if not self.in_question:
                    # question has an alternate prompt that never changes
                    librl.rl_set_prompt(to_encoding(self.prompt, term_encoding))
                librl.rl_forced_update_display()
        if readline_result:
            # add to history, but only if requested and not the same as the
            # previous history entry
            if add_history and readline.get_history_item(
                    readline.get_current_history_length()) != readline_result:
                librl.add_history(readline_result)
        elif readline_result is None:
            raise EOFError
        elif readline_result is False:
            raise StateChange
        return readline_result.decode(term_encoding, 'ignore')

    def put(self, string):
        """Put a line of output, preserving the prompt afterwards."""
        self.out.write('\r\x1b[K%s\n' % string)
        self.out.flush()
        os.write(self.wakeup_pipe_w, b' ')

    def put_error(self, string):
        """Put a client error message."""
        self.put(colorize('red', '# ERROR: ' + string))

    def put_client(self, string):
        """Put a client info message."""
        self.put(colorize('bold', '# ' + string))

    def ask_passwd(self, question):
        """Prompt user for a password."""
        return getpass.getpass(colorize('bold', '# %s ' % question))

    def ask_question(self, question, chars='', default='', on_intr=''):
        """Prompt user for input to a question."""
        # add hints of what can be entered
        if chars:
            question += ' (%s)' % ('/'.join(chars.upper()))
        if default:
            question += ' [%s]' % default
        self.in_question = True
        try:
            try:
                # see set_status() for an explanation of the special chars here
                ans = self.readline('\x01\r\x1b[K' + colorize('bold',
                                    '\x02# ' + question + ' \x01') + '\x02',
                                    add_history=False)
            except (KeyboardInterrupt, EOFError):
                return on_intr
            if chars:
                # we accept any string; if it's beginning with one of the chars,
                # that is the result, otherwise it's the default value
                ans = ans.lower()
                for char in chars:
                    if ans.startswith(char):
                        return char
                return default
            if not ans:
                ans = default
            return ans
        finally:
            self.in_question = False

    def ask_input(self, question):
        """Prompt user for a line of input."""
        self.in_question = True
        try:
            try:
                # see set_status() for an explanation of the special chars here
                ans = self.readline('\x01\r\x1b[K' + colorize('bold',
                                    '\x02# ' + question + ' \x01') + '\x02',
                                    add_history=False)
            except (KeyboardInterrupt, EOFError):
                return ''
            return ans
        finally:
            self.in_question = False

    # -- event (signal) handlers

    def initial_update(self):
        """Called after connection is established."""
        # request current full status
        state = self.ask('getstatus')
        if state is None:
            return
        if not self.quiet_connect:
            self.put_client(
                'Connected to %s:%s as %s. '
                'Replaying output (enter "/log" to see more)...' %
                (self.host, self.port, self.conndata.user))
            output = self.ask('getmessages', str(self.tsize[1] - 3), default=[])
            for msg in output:
                self.put_message(msg)
            if not self.tip_shown:
                self.put_client('Loaded setups: %s. Enter "/help" for help '
                                'with the client commands.' %
                                (', '.join(state['setups'][1]) or '(none)'))
                self.tip_shown = True
            else:
                self.put_client('Loaded setups: %s.' %
                                (', '.join(state['setups'][1]) or '(none)'))
        else:
            self.put_client('Connected to %s:%s as %s. ' %
                            (self.host, self.port, self.conndata.user))
        self.signal('processing', {'script': state['script'], 'reqid': '0'})
        self.signal('status', state['status'])
        self.signal('eta', state['eta'])
        self.current_mode = state['mode']
        self.scriptpath = self.eval('session.experiment.scriptpath', '.')
        self.instrument = self.eval('session.instrument.instrument',
                                    self.instrument)
        for req in state['requests']:
            self.pending_requests[req['reqid']] = req
        self.set_status(self.status)

    stcolmap  = {'idle': 'blue',
                 'running': 'fuchsia',
                 'paused': 'red',
                 'disconnected': 'darkgray'}
    modemap   = {MASTER: '',
                 SLAVE:  'slave,',
                 SIMULATION: 'simmode,',
                 MAINTENANCE: 'maintenance,'}

    def set_status(self, status):
        """Update the current execution status, and set a new prompt."""
        self.status = status
        if self.stop_pending:
            pending = ' (stop pending)'
        elif self.pending_requests:
            pending = ' (%d pending)' % len(self.pending_requests)
        else:
            pending = ''
        # \x01/\x02 are markers recognized by readline as "here come"
        # zero-width control characters; ESC[K means "clear whole line"
        self.prompt = '\x01' + colorize(
            self.stcolmap[status],
            '\r\x1b[K\x02# ' + (self.instrument or '') + '[%s%s]%s %s \x01' %
            (self.modemap[self.current_mode], status, pending,
             self.spy_mode and 'spy>' or '>>')) + '\x02'
        os.write(self.wakeup_pipe_w, b' ')

    def showhelp(self, html):
        """Handles the "showhelp" signal.

        Uses html2text to get nicely-formatted text from the html.
        """
        htmlconv = HTML2Text()
        htmlconv.ignore_links = True
        self.put(htmlconv.handle(html))

    def handle_eta(self, data):
        """Handles the "eta" signal."""
        state, eta = data

        if state in (SIM_STATES['pending'], SIM_STATES['running']):
            self.cur_eta = '<calculation in progress>'
        elif state == SIM_STATES['failed']:
            self.cur_eta = '<calculation failed>'
        elif state == SIM_STATES['success'] and eta > currenttime():
            self.cur_eta = formatEndtime(eta - currenttime())

    def put_message(self, msg, sim=False):
        """Handles the "message" signal."""
        if msg[0] == 'nicos':
            namefmt = ''
        else:
            namefmt = '%-10s: ' % msg[0]
        levelno = msg[2]
        if levelno == ACTION:
            action = namefmt + msg[3].rstrip()
            self.out.write('\x1b]0;NICOS%s\x07' %
                           (action and ' (%s)' % action or ''))
            return
        else:
            if self.subsec_ts:
                timesuf = '.%.06d] ' % ((msg[1] % 1) * 1000000)
            else:
                timesuf = '] '
            if levelno <= DEBUG:
                timefmt = strftime('[%H:%M:%S', localtime(msg[1]))
                newtext = colorize('lightgray', timefmt + timesuf) + \
                    colorize('darkgray', namefmt + msg[3].rstrip())
            elif levelno <= INFO:
                timefmt = strftime('[%H:%M:%S', localtime(msg[1]))
                newtext = colorize('lightgray', timefmt + timesuf) + \
                    namefmt + msg[3].rstrip()
            elif levelno == INPUT:
                newtext = colorize('darkgreen', msg[3].rstrip())
            elif levelno <= WARNING:
                timefmt = strftime('[%Y-%m-%d %H:%M:%S', localtime(msg[1]))
                newtext = colorize('purple', timefmt + timesuf + namefmt +
                                   levels[levelno] + ': ' + msg[3].rstrip())
            else:
                timefmt = strftime('[%Y-%m-%d %H:%M:%S', localtime(msg[1]))
                newtext = colorize('red', timefmt + timesuf + namefmt +
                                   levels[levelno] + ': ' + msg[3].rstrip())
        if sim:
            newtext = '(sim) ' + newtext
        self.put(newtext)

    # pylint: disable=W0221
    def signal(self, name, data=None, exc=None):
        """Handles any kind of signal/event sent by the daemon."""
        try:
            # try to order the elifs by frequency
            if name == 'message':
                if self.in_editing:
                    self.message_queue.append(data)
                else:
                    self.put_message(data)
            elif name == 'status':
                status, line = data  # pylint:disable=W0633
                if status == STATUS_IDLE or status == STATUS_IDLEEXC:
                    new_status = 'idle'
                    self.stop_pending = False
                elif status != STATUS_INBREAK:
                    new_status = 'running'
                else:
                    new_status = 'paused'
                if status != self.status:
                    self.set_status(new_status)
                if line != self.current_line:
                    self.current_line = line
            elif name == 'cache':
                if data[1].endswith('/scriptpath'):
                    self.scriptpath = self.eval(
                        'session.experiment.scriptpath', '.')
            elif name == 'processing':
                script = data.get('script')
                if script is None:
                    return
                self.current_filename = data.get('name') or ''
                script = script.splitlines() or ['']
                if script != self.current_script:
                    self.current_script = script
                self.pending_requests.pop(data['reqid'], None)
                self.set_status(self.status)
            elif name == 'request':
                if 'script' in data:
                    self.pending_requests[data['reqid']] = data
                self.set_status(self.status)
            elif name == 'blocked':
                removed = [_f for _f in (self.pending_requests.pop(reqid, None)
                                         for reqid in data) if _f]
                if removed:
                    self.put_client('%d script(s) or command(s) removed from '
                                    'queue.' % len(removed))
                    self.show_pending()
                self.set_status(self.status)
            elif name == 'updated':
                if 'script' in data:
                    self.pending_requests[data['reqid']] = data
            elif name == 'rearranged':
                old_reqs = self.pending_requests.copy()
                self.pending_requests.clear()
                for reqid in data:
                    self.pending_requests[reqid] = old_reqs[reqid]
            elif name == 'connected':
                self.reconnect_count = 0
                self.initial_update()
            elif name == 'disconnected':
                self.put_client('Disconnected from server, use /reconnect to '
                                'try reconnecting.')
                self.current_mode = MASTER
                self.debug_mode = False
                self.pending_requests.clear()
                self.set_status('disconnected')
            elif name == 'showhelp':
                self.showhelp(data[1])
            elif name == 'simmessage':
                if data[5] in [self.simuuid, '0']:
                    if not self.in_editing:
                        self.put_message(data, sim=True)
            elif name == 'simresult':
                if data and data[2] in [self.simuuid, '0']:
                    timing, devinfo, _ = data  # pylint: disable=W0633
                    if timing < 0:
                        self.put_client('Dry run resulted in an error.')
                        return
                    self.put_client('Simulated minimum runtime: %s '
                                    '(finishes approximately %s). Device ranges:' %
                                    (formatDuration(timing, precise=False),
                                     formatEndtime(timing)))
                    if devinfo:
                        dnwidth = max(map(len, devinfo))
                        sorteditems = sorted(iteritems(devinfo),
                                             key=lambda x: x[0].lower())
                        for devname, (_, dmin, dmax, aliases) in sorteditems:
                            aliascol = 'aliases: ' + ', '.join(aliases) if aliases else ''
                            self.put('#   %-*s: %10s  <->  %-10s %s' %
                                     (dnwidth, devname, dmin, dmax, aliascol))
            elif name == 'mode':
                self.current_mode = data
                self.set_status(self.status)
            elif name == 'setup':
                self.scriptpath = self.eval('session.experiment.scriptpath',
                                            '.')
                self.instrument = self.eval('session.instrument.instrument',
                                            self.instrument)
            elif name == 'debugging':
                self.debug_mode = data
                readline_finish_callback(False)
            elif name == 'plugplay':
                if data[0] == 'added':
                    self.put_client('new sample environment detected: load '
                                    'setup %r to activate' % data[1])
                elif data[0] == 'removed':
                    self.put_client('sample environment removed: unload '
                                    'setup %r to clear devices' % data[1])
            elif name == 'eta':
                self.handle_eta(data)
            elif name == 'broken':
                self.put_error(data)
                self.reconnect_count = self.RECONNECT_TRIES
                self.reconnect_time = self.RECONNECT_INTERVAL_SHORT
                self.schedule_reconnect()
            elif name == 'failed':
                if self.reconnect_count:
                    self.schedule_reconnect()
                else:
                    self.put_error(data)
            elif name == 'error':
                self.put_error(data)
            # and we ignore all other signals
        except Exception as e:
            self.put_error('In %s event handler: %s.' % (name, e))

    # -- reconnect handling

    def schedule_reconnect(self):
        def reconnect():
            if self.reconnect_count:
                self.connect(self.conndata, eventmask=EVENTMASK)
        self.reconnect_count -= 1
        if self.reconnect_count <= self.RECONNECT_TRIES_LONG:
            self.reconnect_time = self.RECONNECT_INTERVAL_LONG
        threading.Timer(self.reconnect_time / 1000., reconnect).start()

    # -- command handlers

    def ask_connect(self, ask_all=True):
        hostport = '%s:%s' % (self.conndata.host, self.conndata.port)
        if hostport in (':', ':1301') or ask_all:
            default = '' if hostport in (':', ':1301') else hostport
            default = default or 'localhost'
            server = self.ask_question('Server host:port?', default=default)
            if not server:
                return
            try:
                host, port = server.split(':', 1)
                port = int(port)
            except ValueError:
                host = server
                port = DEFAULT_PORT
            self.conndata.host = host
            self.conndata.port = port
        if not self.conndata.user or ask_all:
            user = self.ask_question('User name?',
                                     default=self.conndata.user or 'guest')
            self.conndata.user = user
        if self.conndata.password is None or ask_all:
            password = self.ask_passwd('Password?')
            self.conndata.password = password
        self.instrument = self.conndata.host.split('.')[0]
        try:
            self.connect(self.conndata, eventmask=EVENTMASK)
        except RuntimeError as err:
            self.put_error('Cannot connect: %s.' % err)

    def help(self, arg):
        """Implements the "/help" command."""
        if not arg:
            arg = 'main'
        if arg not in HELP:
            arg = 'main'
        helptext = HELP[arg]
        for line in helptext.splitlines():
            self.put('# ' + line)

    def edit_file(self, arg):
        """Implements the "/edit" command."""
        if not arg:
            if path.isfile(self.current_filename):
                arg = self.current_filename
        if not arg:
            self.put_error('Need a file name as argument.')
            return
        fpath = path.join(self.scriptpath, path.expanduser(arg))
        if not os.environ.get('EDITOR'):
            os.environ['EDITOR'] = 'vi'
        self.in_editing = True
        cwd = os.getcwd()
        if path.isdir(self.scriptpath):
            os.chdir(self.scriptpath)
        try:
            ret = os.system('$EDITOR "' + fpath + '"')
        finally:
            os.chdir(cwd)
            self.in_editing = False
            for msg in self.message_queue:
                self.put_message(msg)
            self.message_queue = []
        if ret != 0 or not path.isfile(fpath):
            return
        # if the editor exited successfully (and the file exists) we try to be
        # smart about offering the user a choice of running, simulating or
        # updating the current script
        self.last_filename = fpath
        if self.status == 'running':
            if fpath == self.current_filename:
                # current script edited: most likely we want to update it
                if self.ask_question('Update running script?', chars='yn',
                                     default='n') == 'y':
                    return self.command('update', fpath)
            else:
                # another script edited: updating will likely fail
                reply = self.ask_question('Queue or dry-run file?',
                                          chars='qdn')
                if reply == 'q':
                    # this will automatically queue
                    return self.command('run!', fpath)
                elif reply == 'd':
                    return self.command('sim', fpath)
        else:
            # no script is running at the moment: offer to run it
            reply = self.ask_question('Run or dry-run file?', chars='rdn')
            if reply == 'r':
                return self.command('run', fpath)
            elif reply == 'd':
                return self.command('sim', fpath)

    def print_where(self):
        """Implements the "/where" command."""
        if self.status in ('running', 'paused'):
            self.put_client('Printing current script.')
            for i, line in enumerate(self.current_script):
                if i+1 == self.current_line:
                    self.put(colorize('darkgreen', '---> ' + line))
                else:
                    self.put('     ' + line)
            self.put_client('End of script.')
            if self.cur_eta:
                self.put_client('Estimated finishing time: ' + self.cur_eta)
        else:
            self.put_client('No script is running.')

    def _iter_pending(self):
        for reqid, script in iteritems(self.pending_requests):
            if 'name' in script and script['name']:
                short = script['name']
            elif 'script' in script:
                lines = script['script'].splitlines()
                if len(lines) == 1:
                    short = lines[0]
                else:
                    short = lines[0] + ' ...'
            else:
                short = '(stop)'
            yield reqid, short

    def show_pending(self):
        if not self.pending_requests:
            self.put_client('No scripts or commands are pending.')
            return
        self.put_client('Showing pending scripts or commands. '
                        'Use "/cancel" to remove one or more.')
        for _reqid, short in self._iter_pending():
            self.put('#   %s' % short)
        self.put_client('End of pending list.')

    def cancel_menu(self, arg):
        if not self.pending_requests:
            self.put_client('No scripts or commands are pending.')
            return
        if arg == '*':
            self.tell('unqueue', '*')
            return
        self.put_client('Showing pending scripts or commands.')
        indices = {}
        for index, (reqid, short) in enumerate(self._iter_pending(), start=1):
            indices[index] = reqid
            self.put('#   %s  %s' % (colorize('blue', '%2d' % index), short))
        res = self.ask_question('Which script to cancel ("*" for all)?')
        if res == '*':
            self.tell('unqueue', '*')
            return
        try:
            reqid = indices[int(res)]
        except (ValueError, KeyError):
            self.put_error('Invalid selection.')
            return
        self.tell('unqueue', reqid)

    def debug_repl(self):
        """Called to handle remote debugging via Rpdb."""
        self.in_question = True  # suppress prompt changes
        try:
            while self.debug_mode:
                try:
                    cmd = self.readline('\x01\r\x1b[K' + colorize('darkred',
                                        '\x02# (Rpdb) \x01') + '\x02') + '\n'
                except (EOFError, KeyboardInterrupt):
                    cmd = ''
                except StateChange:
                    if not self.debug_mode:
                        return
                self.tell('debuginput', cmd)
        finally:
            self.in_question = False

    def plot_data(self, xterm_mode):
        try:
            xs, ys, _, names = self.eval(
                '__import__("nicos").commands.analyze._getData()[:4]')
            plotlines = txtplot(xs, ys, names[0], names[1], xterm_mode)
        except Exception as err:
            self.put_error('Could not plot: %s.' % str(err))
        else:
            for line in plotlines:
                self.put(line)

    def stop_query(self, how):
        """Called on Ctrl-C (if running) or when "/stop" is entered."""
        self.put_client('== %s ==' % how)
        self.put('# Please enter how to proceed:')
        self.put('# <I> ignore this interrupt')
        self.put('# <H> stop after current scan point')
        self.put('# <L> stop after current command')
        self.put('# <S> immediate stop')
        res = self.ask_question('Your choice?', chars='ihls').upper()
        if res == 'I':
            return
        elif res == 'H':
            # this is basically "stop at any well-defined breakpoint"
            self.tell('stop', BREAK_AFTER_STEP)
            self.stop_pending = True
            self.set_status(self.status)
        elif res == 'L':
            # this is "everywhere after a command in the script"
            self.tell('stop', BREAK_AFTER_LINE)
            self.stop_pending = True
            self.set_status(self.status)
        else:
            self.tell('emergency')

    def command(self, cmd, arg):
        """Called when a "/foo" command is entered at the prompt."""
        # try to order elif cases by frequency
        if cmd in ('cmd', 'exec'):
            if cmd == 'cmd' and self.spy_mode:
                return self.command('eval', arg)
            # this is not usually entered as "/cmd foo", but only "foo"
            if self.status in ('running', 'paused'):
                reply = self.ask_question('A script is already running, '
                                          'queue or execute anyway?', chars='qxn')
                if reply == 'x':
                    if self.status != 'idle':
                        self.tell('exec', arg)
                    else:
                        self.run(arg)
                elif reply == 'q':
                    self.run(arg)
                    self.put_client('Command queued.')
            else:
                self.run(arg)
        elif cmd in ('r', 'run', 'run!'):
            if not arg:
                # since we remember the last edited file, we can offer
                # running it here
                if self.last_filename:
                    reply = self.ask_question('Run last used file %r?' %
                                              path.basename(self.last_filename),
                                              chars='yn', default='y')
                    if reply == 'y':
                        self.command('run', self.last_filename)
                        return
                self.put_error('Need a file name as argument.')
                return
            fpath = path.join(self.scriptpath, path.expanduser(arg))
            try:
                code = open(fpath).read()
            except Exception as e:
                self.put_error('Unable to open file: %s.' % e)
                return
            if self.status in ('running', 'paused') and cmd != 'run!':
                if self.ask_question('A script is already running, '
                                     'queue script?', chars='yn',
                                     default='y') == 'y':
                    self.run(code, fpath)
            else:
                self.run(code, fpath)
        elif cmd == 'update':
            if not arg:
                # always take the current filename, if it still exists
                if path.isfile(self.current_filename):
                    arg = self.current_filename
            if not arg:
                self.put_error('Need a file name as argument.')
                return
            fpath = path.join(self.scriptpath, path.expanduser(arg))
            try:
                code = open(fpath).read()
            except Exception as e:
                self.put_error('Unable to open file: %s.' % e)
                return
            reason = self.ask_input('Reason for updating:')
            self.tell('update', code, reason)
        elif cmd in ('sim', 'simulate'):
            if not arg:
                self.put_error('Need a file name or code as argument.')
                return
            fpath = path.join(self.scriptpath, path.expanduser(arg))
            self.last_filename = fpath
            # detect whether we have a filename or potential Python code
            if path.isfile(fpath) or fpath.endswith(('.py', '.txt')):
                try:
                    code = open(fpath).read()
                except Exception as e:
                    self.put_error('Unable to open file: %s.' % e)
                    return
                self.simulate(fpath, code)
            else:
                self.simulate('', arg)
        elif cmd in ('e', 'edit'):
            self.edit_file(arg)
        elif cmd == 'break':
            self.tell('break', BREAK_AFTER_STEP)
        elif cmd in ('cont', 'continue'):
            self.tell('continue')
        elif cmd in ('s', 'stop'):
            if self.status == 'running':
                self.stop_query('Stop request')
            else:
                self.tell('emergency')
        elif cmd in ('fin', 'finish'):
            self.tell('finish')
        elif cmd == 'pending':
            self.show_pending()
        elif cmd == 'cancel':
            self.cancel_menu(arg)
        elif cmd == 'disconnect':
            if self.isconnected:
                self.disconnect()
        elif cmd == 'connect':
            self.reconnect_count = 0
            if self.isconnected:
                self.put_error('Already connected. Use /disconnect first.')
            else:
                self.ask_connect()
        elif cmd in ('re', 'reconnect'):
            self.reconnect_count = 0   # no automatic reconnect
            if self.isconnected:
                self.disconnect()
            self.ask_connect(ask_all=False)
        elif cmd in ('q', 'quit'):
            if self.isconnected:
                self.disconnect()
            return 0   # i.e. exit with success
        elif cmd in ('h', 'help', '?'):
            self.help(arg)
        elif cmd == 'log':
            if arg:
                n = str(int(arg))  # make sure it's an integer
            else:
                n = '*'  # as a slice index, this means "unlimited"
            # this can take a while to transfer, but we don't want to cache
            # messages in this client just for this command
            messages = self.ask('getmessages', n)
            if messages is None:
                return
            self.put_client('Printing %s previous messages.' %
                            (n if n != '*' else 'all'))
            for msg in messages:
                self.put_message(msg)
            self.put_client('End of messages.')
        elif cmd in ('w', 'where'):
            self.print_where()
        elif cmd == 'wait':
            if arg:
                time.sleep(float(arg))
            else:
                # this command is mainly meant for testing and scripting purposes
                time.sleep(0.1)
                while self.status != 'idle':
                    time.sleep(0.1)
        elif cmd == 'trace':
            trace = self.ask('gettrace')
            if trace is None:
                return
            self.put_client('Current stacktrace of script execution:')
            for line in trace.splitlines():
                if line:
                    self.put('# ' + line)
            self.put_client('End of stacktrace.')
        elif cmd == 'debugclient':
            import pdb
            pdb.set_trace()
        elif cmd == 'debug':
            self.tell('debug', arg)
        elif cmd == 'eval':
            timefmt = colorize('lightgray', strftime('[%H:%M:%S]'))
            self.put('%s -> %s' % (timefmt, self.eval(arg, None, stringify=True)))
        elif cmd == 'spy':
            if not self.spy_mode:
                self.put_client('Spy mode on: normal input is evaluated as '
                                'an expression, use /exec to execute as script.')
            else:
                self.put_client('Spy mode off.')
            self.spy_mode = not self.spy_mode
            self.set_status(self.status)
        elif cmd == 'plot':
            self.plot_data(xterm_mode=(arg == 'x'))
        elif cmd == 'subsec':
            self.subsec_ts = not self.subsec_ts
        else:
            self.put_error('Unknown command %r.' % cmd)

    # -- command-line completion support

    def simulate(self, fpath, code):
        self.simuuid = str(uuid1())
        self.tell('simulate', fpath, code, self.simuuid)

    def complete_filename(self, fn, word):
        """Try to complete a script filename."""
        # script filenames are relative to the current scriptpath; nevertheless
        # the user can override this by giving an absolute path to the script
        initpath = path.join(self.scriptpath, fn)
        candidates = []
        # omit the part already on the line, but not what readline considers the
        # current "word"
        omit = len(initpath) - len(word)
        # complete directories and .py/.txt script files
        for f in glob.glob(initpath + '*'):
            if path.isdir(f):
                candidates.append(f[omit:] + '/')
            elif path.isfile(f) and f.endswith(('.py', '.txt')):
                candidates.append(f[omit:])
        return candidates

    commands = ['run', 'simulate', 'edit', 'update', 'break', 'continue',
                'stop', 'where', 'disconnect', 'connect', 'reconnect',
                'quit', 'help', 'log', 'pending', 'cancel', 'eval', 'spy',
                'debug', 'trace', 'subsec', 'plot']

    def completer(self, text, state):
        """Try to complete the command line.  Called by readline."""
        if state == 0:
            # we got a a new bit of text to complete...
            line = readline.get_line_buffer()
            # handle line without command
            if not line.startswith('/'):
                line = '/exec ' + line
            # split into command and arguments
            parts = line[1:].split(None, 1)
            if len(parts) < 2 and not line.endswith(' '):
                # complete client command names
                self.completions = [cmd for cmd in self.commands
                                    if cmd.startswith(text)]
            elif parts[0] in ('r', 'run', 'e', 'edit',
                              'update', 'sim', 'simulate'):
                # complete filenames
                try:
                    fn = parts[1]
                except IndexError:
                    fn = ''
                self.completions = self.complete_filename(fn, text)
            elif parts[0] in ('eval', 'exec'):
                # complete code -- ask the server to complete for us
                try:
                    self.completions = self.ask('complete', text,
                                                parts[1], default=[])
                except Exception:
                    self.completions = []
            else:
                # no completions for other commands
                self.completions = []
        try:
            return self.completions[state]
        except IndexError:
            return None

    # -- main loop

    def handle(self, cmd):
        """Handle a command line."""
        # dispatch either as a client command...
        if cmd.startswith('/'):
            args = cmd[1:].split(None, 1) + ['', '']
            return self.command(args[0], args[1])
        elif cmd:
            # or as "normal" Python code to execute
            return self.command('cmd', cmd)
        # an empty line is ignored

    def main(self):
        """Connect and then run the main read-send-print loop."""
        try:
            self.command('reconnect', '')
            while 1:
                if self.debug_mode:
                    self.debug_repl()
                try:
                    cmd = self.readline(self.prompt)
                except KeyboardInterrupt:
                    # offer the user a choice of ways of stopping
                    if self.status == 'running' and not self.spy_mode:
                        self.stop_query('Keyboard interrupt')
                    continue
                except EOFError:
                    self.command('quit', '')
                    return 0
                except StateChange:
                    continue
                ret = self.handle(cmd)
                if ret is not None:
                    return ret
        finally:
            try:
                readline.write_history_file(self.histfile)
            except IOError:
                pass

    def main_with_command(self, command):
        self.quiet_connect = True
        self.command('reconnect', '')
        self.handle(command)
        time.sleep(0.1)
        while self.status != 'idle':
            time.sleep(0.1)
        self.command('quit', '')
        return 0
Beispiel #60
0
class CallSequence(object):
    """@brief Call sequence manager.

    Contains an ordered sequence of tasks. Each task has a name and associated
    callable. The CallSequence class itself is callable, so instances can be nested
    as tasks within other CallSequences.

    When tasks within a sequence are called, they may optionally return a new CallSequence
    instance. If this happens, the new sequence is executed right away, before continuing
    with the next task in the original sequence.

    A CallSequence can be iterated over. It will return tuples of (task-name, callable).
    """
    def __init__(self, *args):
        """@brief Constructor.

        The constructor accepts an arbitrary number of parameters describing an ordered
        set of tasks. Each parameter must be a 2-tuple with the first element being the
        task's name and the second element a callable that implements the task. If you
        need to pass parameters to the callable, use a lambda.
        """
        self._validate_tasks(args)
        self._calls = OrderedDict(args)

    def _validate_tasks(self, tasks):
        for i in tasks:
            assert len(i) == 2
            assert type(i[0]) is str
            assert isinstance(i[1], Callable)

    @property
    def sequence(self):
        """@brief Returns an OrderedDict of the call sequence.

        Task names are keys.
        """
        return self._calls

    @sequence.setter
    def sequence(self, seq):
        """@brief Replace the entire call sequence.

        Accepts either an OrderedDict or a list of 2-tuples like the constructor.
        """
        if isinstance(seq, OrderedDict):
            self._calls = seq
        elif type(seq) is list and len(seq) and type(seq[0]) is tuple:
            self._calls = OrderedDict(seq)

    @property
    def count(self):
        """@brief Returns the number of tasks in the sequence."""
        return len(self._calls)

    def clear(self):
        """@brief Remove all tasks from the sequence."""
        self._calls = OrderedDict()

    def copy(self):
        """@brief Duplicate the sequence."""
        new_seq = CallSequence()
        new_seq._calls = self._calls.copy()
        return new_seq

    def remove_task(self, name):
        """@brief Remove a task with the given name.
        @exception KeyError Raised if no task with the specified name exists.
        """
        del self._calls[name]
        return self

    def has_task(self, name):
        """@brief Returns a boolean indicating presence of the named task in the sequence."""
        return name in self._calls

    def get_task(self, name):
        """@brief Return the callable for the named task.
        @exception KeyError Raised if no task with the specified name exists.
        """
        return self._calls[name]

    def replace_task(self, name, replacement):
        """@brief Change the callable associated with a task."""
        assert isinstance(replacement, Callable)
        if name not in self._calls:
            raise KeyError(name)
        else:
            # OrderedDict preserves the order when changing the value of a key
            # that is already in the dict.
            self._calls[name] = replacement
        return self

    def wrap_task(self, name, wrapper):
        """@brief Wrap an existing task with a new callable.

        The wrapper is expected to take a single parameter, the return value from the
        original task. This allows for easy filtering of a new call sequence returned by
        the original task.
        """
        if name not in self._calls:
            raise KeyError(name)

        # Get original callable.
        orig = self._calls[name]

        # OrderedDict preserves the order when changing the value of a key
        # that is already in the dict.
        self._calls[name] = lambda: wrapper(orig())
        return self

    def append(self, *args):
        """@brief Append a new task or tasks to the sequence.

        Like the constructor, this method takes any number of arguments. Each must be a
        2-tuple task description.
        """
        self._validate_tasks(args)

        # Insert iterable.
        self._calls.update(args)
        return self

    def insert_before(self, beforeTaskName, *args):
        """@brief Insert a task or tasks before a named task.

        @param beforeTaskName The name of an existing task. The new tasks will be inserted
          prior to this task.

        After the task name parameter, any number of task description 2-tuples may be
        passed.

        @exception KeyError Raised if the named task does not exist in the sequence.
        """
        self._validate_tasks(args)

        if not self.has_task(beforeTaskName):
            raise KeyError(beforeTaskName)

        seq = list(self._calls.items())
        for i, v in enumerate(seq):
            if v[0] == beforeTaskName:
                for c in args:
                    # List insert() inserts before the given index.
                    seq.insert(i, c)
                    i += 1
                break
        self._calls = OrderedDict(seq)
        return self

    def insert_after(self, afterTaskName, *args):
        """@brief Insert a task or tasks after a named task.

        @param afterTaskName The name of an existing task. The new tasks will be inserted
          after this task.

        After the task name parameter, any number of task description 2-tuples may be
        passed.

        @exception KeyError Raised if the named task does not exist in the sequence.
        """
        self._validate_tasks(args)

        if not self.has_task(afterTaskName):
            raise KeyError(afterTaskName)

        seq = list(self._calls.items())
        for i, v in enumerate(seq):
            if v[0] == afterTaskName:
                for c in args:
                    # List insert() inserts before the given index.
                    seq.insert(i + 1, c)
                    i += 1
                break
        self._calls = OrderedDict(seq)
        return self

    def invoke(self):
        """@brief Execute each task in order.

        A task may return a CallSequence, in which case the new sequence is immediately
        executed.
        """
        for name, call in self._calls.items():
            LOG.debug("Running task %s", name)
            resultSequence = call()

            # Invoke returned call sequence.
            if resultSequence is not None and isinstance(
                    resultSequence, CallSequence):
                #                 LOG.debug("Invoking returned call sequence: %s", resultSequence)
                resultSequence.invoke()

    def __call__(self, *args, **kwargs):
        """@brief Another way to execute the tasks.

        Supports nested CallSequences.
        """
        self.invoke()

    def __iter__(self):
        """@brief Iterate over the sequence."""
        return iter(self._calls.items())

    def __repr__(self):
        s = "<%s@%x: " % (self.__class__.__name__, id(self))
        for name, task in self._calls.items():
            s += "\n%s: %s" % (name, task)
        s += ">"
        return s