def createTrainingVectors(tokenized_texts_dict): """ Given the filenames and their contents, this methods creates the training vectors by creating a unique list of all words together in the training set """ print("Creating vectors for training data") unique_words = [] for filename, text in tokenized_texts_dict.iteritems(): # print("Reading {0} and adding to unique word list".format(filename)) unique_words.extend(word_tokenize(text)) unique_words = set(unique_words) # Creating the initial vector with counts 0 for all training sets zero_vector = OrderedDict(zip(unique_words, [0] * len(unique_words))) print("Creating the zero vector") # For each training file, create an OrderedDict containing its word counts (together with zero counts), # and store it in a dict, indexed by its corresponding filename vectors = {} for filename, token_list in tokenized_texts_dict.iteritems(): current_vector = zero_vector.copy() current_vector.update(Counter(word_tokenize(token_list))) vectors[filename] = current_vector return vectors, zero_vector
class ValidatorRegister(object): """Validators register.""" def __init__(self): """Initialize the register. Take no parameters.""" # Dictionary of (code, description) pairs self._errors = OrderedDict() # Dictionary of (code, callback) pairs self._validators = OrderedDict() @property def errors(self): """Return dictionary of (error_code, error_description) pairs.""" return self._errors.copy() @property def validators(self): """Return dictionary of (error_code, callback) pairs.""" return self._validators.copy() def register(self, callback, error_code, error_description): """Register validator. @param callback: Function which performs validation. @type callback: function @param error_code: Error code which will be reported if validation fails. @type error_code: text @param error_description: Error description which will be reported if validation fails. @type error_description: text @raises ValueError: If `error_code` is already registered. """ if error_code in self._validators: raise ValueError('Validator for %s is already registered.' % error_code) self._errors[error_code] = error_description self._validators[error_code] = callback
def parse_input_volfile(orig_volfile): volfile = OrderedDict() with open(orig_volfile, "r") as f: for line in f: try: isinstance(opt_dict, dict) except UnboundLocalError: opt_dict = OrderedDict() try: isinstance(sub_dict, dict) except UnboundLocalError: sub_dict = OrderedDict() temp = line.split() if not temp: continue if line.startswith("volume "): xl_name = temp[1] elif temp[0] == "type": sub_dict[temp[0]] = temp[1] sub_dict["options"] = OrderedDict() elif temp[0] == "option": opt_dict[temp[1]] = temp[2] elif temp[0] == "subvolumes": sub_dict[temp[0]] = temp[1:] elif line.startswith("end-volume"): sub_dict["options"] = opt_dict.copy() volfile[xl_name] = sub_dict.copy() del(opt_dict) del(sub_dict) else: print "Oops!!! Bug, Parser needs an update..." sys.exit(1) return volfile
class DisorderManager(object): implements(IHandler, IDisorderObserver) def __init__(self, handlers): for handler in handlers: if IRetrieveCapable.providedBy(handler): self.retriever = handler break else: self.retriever = None self.seekers = {} self.disorders = OrderedDict() self.observers = [] def add(self, key, seeker): seeker.add_observer(self, key) self.seekers[key] = seeker self.disorders[key] = None if self.retriever and IReplayable.providedBy(seeker): seeker.replay(self.retriever) def there_is_disorder(self, key, disorder): self.set_disorder_value(key, MaybeDisorder(True, disorder)) def there_is_no_disorder(self, key, disorder): self.set_disorder_value(key, MaybeDisorder(False, disorder)) def seeker_is_not_functional(self, key): self.set_disorder_value(key, None) def set_disorder_value(self, key, value): if value != self.disorders[key]: self.disorders[key] = value self.notify_observers() def add_observer(self, observer): self.observers.append(observer) observer.update_disorders(self.disorders.copy()) def notify_observers(self): for observer in self.observers: observer.update_disorders(self.disorders.copy()) def initialize(self): pass def process(self, record): for seeker in self.seekers.values(): seeker.receive_record(record)
class YkmanContextObject(MutableMapping): def __init__(self): self._objects = OrderedDict() self._resolved = False def add_resolver(self, key, f): if self._resolved: f = f() self._objects[key] = f def resolve(self): if not self._resolved: self._resolved = True for k, f in self._objects.copy().items(): self._objects[k] = f() def __getitem__(self, key): self.resolve() return self._objects[key] def __setitem__(self, key, value): if not self._resolved: raise ValueError('BUG: Attempted to set item when unresolved.') self._objects[key] = value def __delitem__(self, key): del self._objects[key] def __len__(self): return len(self._objects) def __iter__(self): return iter(self._objects)
def _post(uri, data, ext_encode_data=None, add_security_key=True): """ :type uri: str :type data: OrderedDict :rtype: Response """ request_data = OrderedDict(MerchantId=MERCHANT_ID) request_data.update(data) if add_security_key: encode_data = request_data.copy() encode_data.update(ext_encode_data or {}) request_data['SecurityKey'] = _gen_security_key(encode_data) try: r = requests.post(base_url + uri, data=request_data) if r.text: print r.text _create_log(uri, request_data, r.text) else: print r.headers return Response(body=r.text) except: print 'Error Payonline' print traceback.format_exc() return ErrorResponse()
def naive_model(game, df): state_rep = ['time', 'home_score', 'away_score'] new_game = game.copy() new_game['win'] = pd.Series() new_game['tie'] = pd.Series() new_game['win_or_tie'] = pd.Series() for row in new_game.index: params = OrderedDict([(k,game.iloc[row][k]) for k in state_rep]) params_away = params.copy() params_away['home_score'], params_away['away_score'] = params['away_score'], params['home_score'] home = probability_of_consolid_state(params, state_rep, df) away = probability_of_consolid_state(params_away, state_rep, df) if home is None or away is None: continue win, tie = home_win(home, away), home_tie(home, away) new_game['win'].ix[row] = win new_game['tie'].ix[row] = tie new_game['win_or_tie'].ix[row] = win+tie return new_game
def goal_diff_model(game, df): new_game = game.copy() new_game['win'] = pd.Series() new_game['tie'] = pd.Series() new_game['win_or_tie'] = pd.Series() state_match = ['time', 'diff'] for row in new_game.index: paramHome = OrderedDict([(k,game.iloc[row][k]) for k in state_match]) paramAway = paramHome.copy() paramAway['diff'] = -1 * paramHome['diff'] home = probability_of_consolid_state(paramHome, state_match, df) away = probability_of_consolid_state(paramAway, state_match, df) if home is None or away is None: continue win, tie = home_win(home, away), home_tie(home, away) new_game['win'].ix[row] = win new_game['tie'].ix[row] = tie new_game['win_or_tie'].ix[row] = win+tie return new_game
class FrozenOrderedDict(Mapping): """ Frozen OrderedDict. """ def __init__(self, *args, **kwargs): self.__dict = OrderedDict(*args, **kwargs) self.__hash = None def __getitem__(self, item): return self.__dict[item] def __iter__(self): return iter(self.__dict) def __len__(self): return len(self.__dict) def __hash__(self): if self.__hash is None: self.__hash = reduce(operator.xor, map(hash, self.iteritems()), 0) return self.__hash def __repr__(self): return '{}({!r})'.format(self.__class__.__name__, self.items()) def copy(self, *args, **kwargs): new_dict = self.__dict.copy() if args or kwargs: new_dict.update(OrderedDict(*args, **kwargs)) return self.__class__(new_dict)
def test_copying(self): # Check that ordered dicts are copyable, deepcopyable, picklable, # and have a repr/eval round-trip pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) update_test = OrderedDict() update_test.update(od) for i, dup in enumerate([ od.copy(), copy.copy(od), copy.deepcopy(od), pickle.loads(pickle.dumps(od, 0)), pickle.loads(pickle.dumps(od, 1)), pickle.loads(pickle.dumps(od, 2)), pickle.loads(pickle.dumps(od, 3)), pickle.loads(pickle.dumps(od, -1)), eval(repr(od)), update_test, OrderedDict(od), ]): self.assert_(dup is not od) self.assertEquals(dup, od) self.assertEquals(list(dup.items()), list(od.items())) self.assertEquals(len(dup), len(od)) self.assertEquals(type(dup), type(od))
def _fill_related_objects_cache(self): cache = OrderedDict() parent_list = self.get_parent_list() for parent in self.parents: for obj, model in parent._meta.get_all_related_objects_with_model(include_hidden=True): if (obj.field.creation_counter < 0 or obj.field.rel.parent_link) and obj.model not in parent_list: continue if not model: cache[obj] = parent else: cache[obj] = model # Collect also objects which are in relation to some proxy child/parent of self. proxy_cache = cache.copy() for klass in self.apps.get_models(include_auto_created=True): if not klass._meta.swapped: for f in klass._meta.local_fields + klass._meta.virtual_fields: if (hasattr(f, 'rel') and f.rel and not isinstance(f.rel.to, six.string_types) and f.generate_reverse_relation): if self == f.rel.to._meta: cache[f.related] = None proxy_cache[f.related] = None elif self.concrete_model == f.rel.to._meta.concrete_model: proxy_cache[f.related] = None self._related_objects_cache = cache self._related_objects_proxy_cache = proxy_cache
class Buckets(object): """Proxy for OrderedDict""" def __init__(self, *args, **kwargs): self._od = OrderedDict(*args, **kwargs) def __getattr__(self, a): return getattr(self._od, a) def __setitem__(self, *args, **kwargs): return self._od.__setitem__(*args, **kwargs) def __getitem__(self, *args, **kwargs): return self._od.__getitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): return self._od.__delitem__(*args, **kwargs) def __eq__(self, other): if isinstance(other, Buckets): return self._od.__eq__(other._od) else: return self._od.__eq__(other) def copy(self, *args, **kwargs): new = Buckets() new._od = self._od.copy() return new
def post_process(self, paths, dry_run=False, **options): if dry_run: return packager = Packager(storage=self) for _abs_path, rel_path in paths: files_to_process = OrderedDict() files_to_process[rel_path] = (self, rel_path) for package_name in packager.packages['css']: package = packager.package_for('css', package_name) output_file = package.output_filename if rel_path in package.paths: if self.packing: packager.pack_stylesheets(package) files_to_process[output_file] = (self, output_file) yield output_file, output_file, True for package_name in packager.packages['js']: package = packager.package_for('js', package_name) output_file = package.output_filename if rel_path in package.paths: if self.packing: packager.pack_javascripts(package) files_to_process[output_file] = (self, output_file) yield output_file, output_file, True super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): for name, hashed_name, processed in super_class.post_process( files_to_process.copy(), dry_run, **options): yield name, hashed_name, processed
def merge_data_for_people(people, models): """ Collect data for a certain set of people from a list of model objects. Merge results from models that have the same name. """ # All headers from the models all_headers = list(chain.from_iterable([m.analytics_headers() for m in models])) # Initialize a dict containing all people each with all headers, # and default values of '' for each header data = OrderedDict() headers = [] for h in all_headers: if h['title'] not in data.keys(): data[h['title']] = '' headers.append(h) persondata = dict((p, data.copy()) for p in people) persondata['_headers'] = headers for m in models: mdata = m.analytics_data_by_person() for p in people: if p in mdata: persondata[p].update(mdata[p]) return persondata
class TestSerialization(unittest.TestCase): def setUp(self): self.serialized = OrderedDict() self.serialized["typeid"] = "malcolm:core/MethodMeta:1.0" self.takes = MapMeta() self.takes.set_elements(ElementMap({"in_attr": StringMeta("desc")})) self.serialized["takes"] = self.takes.to_dict() self.serialized["defaults"] = OrderedDict({"in_attr": "default"}) self.serialized["description"] = "test_description" self.serialized["tags"] = [] self.serialized["writeable"] = True self.serialized["label"] = "" self.serialized["returns"] = MapMeta().to_dict() def test_to_dict(self): m = MethodMeta("test_description") m.set_takes(self.takes) m.set_defaults(self.serialized["defaults"]) self.assertEqual(m.to_dict(), self.serialized) def test_from_dict(self): m = MethodMeta.from_dict(self.serialized.copy()) self.assertEqual(m.takes.to_dict(), self.takes.to_dict()) self.assertEqual(m.defaults, self.serialized["defaults"]) self.assertEqual(m.tags, []) self.assertEqual(m.writeable, True) self.assertEqual(m.label, "") self.assertEqual(m.returns.to_dict(), MapMeta().to_dict())
def test_include_exclude_versions(self): source_versions = OrderedDict([('egg', '0.1'), ('Egg', '0.2')]) self.assertEquals(self.checker.include_exclude_versions( source_versions), source_versions) results = source_versions.copy() results['Django'] = '0.0.0' self.assertEquals( self.checker.include_exclude_versions( source_versions, includes=['Django', 'egg']), results) source_versions['Django'] = '1.5.1' source_versions['pytz'] = '2013b' results = OrderedDict([('pytz', '2013b')]) self.assertEquals( self.checker.include_exclude_versions( source_versions, excludes=['Django', 'egg']), results) self.assertEquals( self.checker.include_exclude_versions( source_versions, includes=['Django', 'egg'], excludes=['Django', 'egg']), results) results['zc.buildout'] = '0.0.0' self.assertEquals( self.checker.include_exclude_versions( source_versions, includes=['zc.buildout'], excludes=['Django', 'egg']), results)
def _classes_in_config_sample(self): """ Yields only classes with own traits, and their subclasses. Thus, produced sample config-file will contain all classes on which a trait-value may be overridden: - either on the class owning the trait, - or on its subclasses, even if those subclasses do not define any traits themselves. """ cls_to_config = OrderedDict( (cls, bool(cls.class_own_traits(config=True))) for cls in self._classes_inc_parents()) def is_any_parent_included(cls): return any(b in cls_to_config and cls_to_config[b] for b in cls.__bases__) ## Mark "empty" classes for inclusion if their parents own-traits, # and loop until no more classes gets marked. # while True: to_incl_orig = cls_to_config.copy() cls_to_config = OrderedDict( (cls, inc_yes or is_any_parent_included(cls)) for cls, inc_yes in cls_to_config.items()) if cls_to_config == to_incl_orig: break for cl, inc_yes in cls_to_config.items(): if inc_yes: yield cl
class APIError(exceptions.APIException): """ Base class for JSON-API errors. Reference: http://jsonapi.org/format/#error-objects """ def __init__(self, *args, **kwargs): self._data = OrderedDict([ ('id', kwargs.pop('id', None)), ('links', kwargs.pop('links', None)), ('status', kwargs.pop('status', self.status_code)), ('code', kwargs.pop('code', None)), ('title', kwargs.pop('title', None)), ('detail', None), ('source', kwargs.pop('source', None)), ('meta', kwargs.pop('meta', None)), ]) super(APIError, self).__init__(*args, **kwargs) @property def data(self): error = self._data.copy() for key, value in error.items(): error[key] = getattr(self, key, None) if value is None else value # filter out empty values return OrderedDict((k, v) for k, v in error.items() if v)
def get_top_scorers(): current_year = season() goals = Goal.objects.filter(match__competition_year__year=current_year).select_related('scorer').prefetch_related( 'match__competition_year__competition') competition_years = CompetitionYear.objects.filter(year=current_year).select_related() competition_dct = OrderedDict() competitions = OrderedDict() for competition_year in competition_years: # if competition_year.year == current_year: competition_dct[competition_year.competition_id] = 0 competitions[competition_year.competition_id] = competition_year.competition.name players = OrderedDict() for goal in goals: if goal.own_goal: continue competition_id = goal.match.competition_year.competition_id if goal.scorer not in players: players[goal.scorer] = {} players[goal.scorer]['competitions'] = competition_dct.copy() players[goal.scorer]['total'] = 0 players[goal.scorer]['competitions'][competition_id] += 1 players[goal.scorer]['total'] += 1 players = OrderedDict(sorted(players.items(), key=lambda item: item[1]['total'], reverse=True)) context = { 'players': players, 'competitions': competitions } return context
def _compute_root_packages(self, pool, install_map, update_map): """ Look at the root packages in the given maps. Root packages are packages which are not dependencies of other packages. """ packages = OrderedDict(install_map) packages.update(update_map) roots = packages.copy() for package_id, operation in packages.items(): package = operation.package if package_id not in roots: continue for dependency in package.dependencies: candidates = pool.what_provides( Requirement.from_legacy_requirement_string(dependency) ) for candidate in candidates: candidate_id = pool.package_id(candidate) roots.pop(candidate_id, None) return roots
def test_mqtt_subscribes_topics_on_connect(hass): """Test subscription to topic on connect.""" mqtt_client = yield from mock_mqtt_client(hass) subscribed_topics = OrderedDict() subscribed_topics['topic/test'] = 1 subscribed_topics['home/sensor'] = 2 wanted_topics = subscribed_topics.copy() wanted_topics['still/pending'] = 0 hass.data['mqtt'].wanted_topics = wanted_topics hass.data['mqtt'].subscribed_topics = subscribed_topics hass.data['mqtt'].progress = {1: 'still/pending'} # Return values for subscribe calls (rc, mid) mqtt_client.subscribe.side_effect = ((0, 2), (0, 3)) hass.add_job = mock.MagicMock() hass.data['mqtt']._mqtt_on_connect(None, None, 0, 0) yield from hass.async_block_till_done() assert not mqtt_client.disconnect.called expected = [(topic, qos) for topic, qos in wanted_topics.items()] assert [call[1][1:] for call in hass.add_job.mock_calls] == expected assert hass.data['mqtt'].progress == {}
def gaylord(self, out, session): fields = [ 'First Name', 'Last Name', 'Guest Email Address for confirmation purposes', 'Special Requests', 'Arrival', 'Departure', 'City', 'State', 'Zip', 'GUEST COUNTRY', 'Telephone', 'Payment Type', 'Card #', 'Exp.', 'BILLING ADDRESS', 'BILLING CITY', 'BILLING STATE', 'BILLING ZIP CODE', 'BILLING COUNTRY', 'Additional Guest First Name-2', 'Additional Guest Last Name-2', 'Additional Guest First Name-3', 'Additional Guest Last Name3', # No, this is not a typo 'Additional Guest First Name-4', 'Additional Guest Last Name-4', 'Notes', 'Emails', ] blank = OrderedDict([(field, '') for field in fields]) out.writerow(fields) for room in session.query(Room).order_by(Room.created).all(): if room.assignments: row = blank.copy() row.update({ 'Notes': room.notes, 'Arrival': room.check_in_date.strftime('%m/%d/%Y'), 'Departure': room.check_out_date.strftime('%m/%d/%Y'), 'Emails': ','.join(room.email), }) for i, attendee in enumerate([ra.attendee for ra in room.assignments[0:4]]): if i == 0: prefix, suffix = '', '' row.update({'Guest Email Address for confirmation purposes': attendee.email}) else: prefix = 'Additional Guest' suffix = '-{}'.format(i+1) if i != 2 else str(i+1) row.update({ prefix + 'First Name' + suffix: attendee.legal_first_name, prefix + 'Last Name' + suffix: attendee.legal_last_name }) out.writerow(list(row.values()))
def hotel_email_info(self, out, session): fields = [ 'CheckIn Date', 'CheckOut Date', 'Number of Guests', 'Room Notes', 'Guest1 First Name', 'Guest1 Last Name', 'Guest1 Legal Name', 'Guest2 First Name', 'Guest2 Last Name', 'Guest2 Legal Name', 'Guest3 First Name', 'Guest3 Last Name', 'Guest3 Legal Name', 'Guest4 First Name', 'Guest4 Last Name', 'Guest4 Legal Name', 'Guest5 First Name', 'Guest5 Last Name', 'Guest5 Legal Name', 'Emails', ] blank = OrderedDict([(field, '') for field in fields]) out.writerow(fields) for room in session.query(Room).order_by(Room.created).all(): if room.assignments: row = blank.copy() row.update({ 'Room Notes': room.notes, 'Number of Guests': min(4, len(room.assignments)), 'CheckIn Date': room.check_in_date.strftime('%m/%d/%Y'), 'CheckOut Date': room.check_out_date.strftime('%m/%d/%Y'), 'Emails': ','.join(room.email), }) for i, attendee in enumerate([ra.attendee for ra in room.assignments[:4]]): prefix = 'Guest{}'.format(i + 1) row.update({ prefix + ' First Name': attendee.first_name, prefix + ' Last Name': attendee.last_name, prefix + ' Legal Name': attendee.legal_name, }) out.writerow(list(row.values()))
def findDensestsSubgraphParallell(edgeWeights, nodeDegrees, epsilon): # Create a sorted dictionary with ascending node degrees. NB! Does not sort itself later! nodeDegrees = OrderedDict(sorted(nodeDegrees.iteritems(), key=lambda (k,v): (v,k))) avgGraphDensity = computeAverageDensity(edgeWeights, nodeDegrees) avgGraphDensityBest = avgGraphDensity count = 0 while count < 50: print('Iteration #{}'.format(count)) nodeDegreesUnchanged = nodeDegrees.copy() for node in nodeDegrees.keys(): degree = nodeDegreesUnchanged[node] if degree <= 2*(1+epsilon)*avgGraphDensity: del nodeDegrees[node] # Delete all edges connected to this node for edge in edgeWeights.keys(): node1, node2 = edge.split('-') """If edge is a self edge, delete it and don't reduce the degree of "the other" node, since it is yourself""" if node1 == node2: del edgeWeights[edge] else: if node == node1: nodeDegrees[node2] -= edgeWeights[edge] del edgeWeights[edge] elif node == node2: nodeDegrees[node1] -= edgeWeights[edge] del edgeWeights[edge] # If no more edges left, we are finished if len(edgeWeights) == 0: break avgGraphDensity = computeAverageDensity(edgeWeights, nodeDegrees) if avgGraphDensity > avgGraphDensityBest: newEdgeWeights = edgeWeights.copy() newNodeDegrees = nodeDegrees.copy() avgGraphDensityBest = avgGraphDensity count += 1 return newEdgeWeights, newNodeDegrees
class Basic(object): """ Basic object deals with Basic HTTP Authorization configuration file. It is passed the path to userdb file. """ def __init__(self, userdb): self.userdb = userdb self.initial_users = OrderedDict() self.new_users = OrderedDict() def __enter__(self): with open(self.userdb, "r") as users: for i in users: user, password = i.split(":", 1) self.initial_users[user] = password self.new_users = self.initial_users.copy() return self def __exit__(self, type, value, traceback): if self.new_users == self.initial_users: return with open(self.userdb, "w") as userdb: for user in self.new_users: userdb.write("%s:%s" % (user, self.new_users[user])) def __contains__(self, user): return user in self.users @property def users(self): """ Returns users in a tuple """ return self.new_users.keys() def add(self, user, password): """ Adds a user with password """ if self.__contains__(user): raise UserExists self.new_users[user] = self._crypt_password(password) + "\n" def pop(self, user): """ Deletes a user """ if not self.__contains__(user): raise UserNotExists self.new_users.pop(user) def change_password(self, user, password): """ Changes user password """ if not self.__contains__(user): raise UserNotExists self.new_users[user] = self._crypt_password(password) + "\n" def _crypt_password(self, password): """ Crypts password """ def salt(): """ Generates some salt """ symbols = ascii_letters + digits return choice(symbols) + choice(symbols) return crypt(password, salt())
def get_sched(): SCHED = OrderedDict() SCHED['R'] = ['Semester Courses'] SCHED['A'] = ['Adult Undergraduate Studies 7-Week Courses'] SCHED['G'] = ['Graduate Education'] #SCHED['T'] = ['Accelerated Certification for Teachers (ACT)'] SCHED['P'] = ['Paralegal Program'] return SCHED.copy()
def prepare_dht2_xlator (volfile, volname, mds_count, ds_count, brick_list, server, n): opt_dict = OrderedDict() sub_dict = OrderedDict() mds_bricks = "" ds_bricks = "" i = 0 while i < ds_count: if i == ds_count -1: ds_bricks += volname + "-client-" + str(i) else: ds_bricks += volname + "-client-" + str(i) + ":" i += 1 i = ds_count while i < mds_count + ds_count: if i == mds_count + ds_count - 1: mds_bricks += volname + "-client-" + str(i) else: mds_bricks += volname + "-client-" + str(i) + ":" i += 1 subvols = [volname + "-client-" + str(n)] i = 0 while i < mds_count + ds_count: if i == n: i += 1 continue subvols.append(volname + "-client-" + str(i)) i += 1 if server: sub_dict["type"] = "experimental/dht2s" else: sub_dict["type"] = "experimental/dht2c" opt_dict["lock-migration"] = "off" opt_dict["dht2-data-subvolumes"] = ds_bricks opt_dict["dht2-metadata-subvolumes"] = mds_bricks if server: opt_dict["dht2-server-local-subvol"] = volname + "-client-" + str(n) sub_dict["option"] = opt_dict.copy() sub_dict["subvolumes"] = subvols volfile[volname + "-dht"] = sub_dict.copy() del(sub_dict) del(opt_dict)
class TestConfigurationVisitor(ConfigurationVisitor): """ A means to creates configurations for single test """ def __init__(self, test, design_unit, enable_configuration, default_config): ConfigurationVisitor.__init__(self) self._test = test assert test.is_explicit self.design_unit = design_unit self._enable_configuration = enable_configuration self._configs = OrderedDict({default_config.name: default_config}) @property def name(self): return self._test.name @property def test(self): return self._test def get_default_config(self): """ Get the default configuration of this test case """ self._check_enabled() return self._configs[DEFAULT_NAME] def _check_enabled(self): if not self._enable_configuration: raise RuntimeError("Individual test configuration is not possible with run_all_in_same_sim") def get_configuration_dicts(self): """ Get all configurations of this test """ return [self._configs] def _get_configurations_to_run(self): """ Get all simulation runs for this test bench """ configs = self._configs.copy() if len(configs) > 1: # Remove default configurations when there are more than one del configs[DEFAULT_NAME] return configs.values() def create_tests(self, simulator_if, elaborate_only, test_list=None): """ Create all tests from this test case which may be several depending on the number of configurations """ for config in self._get_configurations_to_run(): test_list.add_test( IndependentSimTestCase( test=self._test, config=config, simulator_if=simulator_if, elaborate_only=elaborate_only))
def _derive(cls, name, info, otherbases, recurse=False): # collect the 3 set of infos # info = OrderedDict(info) baseinfo = cls._getpairs().copy() obasesinfo = OrderedDict() for obase in otherbases: if isinstance(obase, (tuple, dict)): obasesinfo.update(obase) else: obasesinfo.update(obase._getpairs()) # update the info of this class (base) with that from the other bases baseinfo.update(obasesinfo) # The info of the new class is a copy of the full base info # plus and update from parameter clsinfo = baseinfo.copy() clsinfo.update(info) # The new items to update/set are those from the otherbase plus the new info2add = obasesinfo.copy() info2add.update(info) clsmodule = sys.modules[cls.__module__] newclsname = str(cls.__name__ + '_' + name) # str - Python 2/3 compat # This loop makes sure that if the name has already been defined, a new # unique name is found. A collision example is in the plotlines names # definitions of bt.indicators.MACD and bt.talib.MACD. Both end up # definining a MACD_pl_macd and this makes it impossible for the pickle # module to send results over a multiprocessing channel namecounter = 1 while hasattr(clsmodule, newclsname): newclsname += str(namecounter) namecounter += 1 newcls = type(newclsname, (cls,), {}) setattr(clsmodule, newclsname, newcls) setattr(newcls, '_getpairsbase', classmethod(lambda cls: baseinfo.copy())) setattr(newcls, '_getpairs', classmethod(lambda cls: clsinfo.copy())) setattr(newcls, '_getrecurse', classmethod(lambda cls: recurse)) for infoname, infoval in info2add.items(): if recurse: recursecls = getattr(newcls, infoname, AutoInfoClass) infoval = recursecls._derive(name + '_' + infoname, infoval, []) setattr(newcls, infoname, infoval) return newcls
def fetch_metadata(metadata_model): """ Fetch the metadata table, and store it in metadata_dict. """ metadata = OrderedDict() metadata_query = metadata_model.objects.all() for table in metadata_query: for field in METADATA_FIELDS: metadata[field] = table.__dict__[field] # Add the metadata to metadata_dict metadata_dict[table.table_name] = metadata.copy()
class ProteinComplex(object): def __init__(self, T=300): self.T = T self.normalized_constant_energy = 0.0 self.residue_variables = OrderedDict() self.interaction_energies = OrderedDict() self.normalized_interaction_energies = None def add_residue(self, residue_type, chain, location): """Adds a residue to the protein if needed. Automatically creates all instances needed before simplification.""" residue_tuple = (residue_type, chain, location) if residue_tuple not in self.residue_variables: residue_data = titratable_residue_data.get(residue_type) base_name = '_'.join(residue_tuple) + '_' res_var = ResidueVariable('_'.join(residue_tuple)) tautomers = residue_data["tautomers"] model_pka = residue_data["model_pka"] for tautomer in tautomers["deprotonated"]: instance = ResidueInstance(False, base_name + tautomer) instance.energy = 0 res_var.instances[tautomer] = instance for tautomer in tautomers["protonated"]: instance = ResidueInstance(True, base_name + tautomer) instance.energy = -1.0 * math.log(10) * model_pka res_var.instances[tautomer] = instance if residue_type != "HIS": #Placeholder instances for later consolidation #HIS is split later so we don't add these as they are not needed. instance = ResidueInstance(True, base_name + "PROTONATED") res_var.instances["PROTONATED"] = instance instance = ResidueInstance(False, base_name + "DEPROTONATED") res_var.instances["DEPROTONATED"] = instance self.residue_variables[residue_tuple] = res_var def get_residue(self, name, chain, location): "Returns the residue for this specific name, chain, and location." return self.residue_variables.get((name, chain, location)) def get_instance(self, residue_type, chain, location, state): "Returns the instance for this specific name, chain, and location and state name." res_var = self.residue_variables.get((residue_type, chain, location)) if res_var is None: return None return res_var.get_instance(state) def drop_interaction_pairs(self, pair_list): "Drop each pair of interaction energies in the pair_list" for instance1_name, instance2_name in pair_list: del self.interaction_energies[instance1_name, instance2_name] del self.interaction_energies[instance2_name, instance1_name] def get_interaction_combinations(self, pair1, pair2): "Get a list of all pairwise combinations from lists pair1 and pair2." product_list = [(x, y) for x, y in product(pair1, pair2)] return product_list def add_interaction_energy_pair(self, instance1, instance2, energy, normalized=False): "Insert interaction energy pair." ie = self.normalized_interaction_energies if normalized else self.interaction_energies ie[instance1, instance2] = energy ie[instance2, instance1] = energy def simplify(self): """Simplify the instances into protonated and deprotonated. Also divides HIS into HID and HIE.""" self.update_special_instance_energies() self.consolidate() self.divide_his() def update_special_instance_energies(self): """After we have loaded the backgroind and desolvation files we need to update the consolidated instances to use the minimum energy of the instances they are meant to replace.""" for key, residue in self.residue_variables.items(): name = key[0] if name == 'HIS': continue prot_consolidated = residue.instances["PROTONATED"] deprot_consolidated = residue.instances["DEPROTONATED"] protonated, deprotonated = residue.get_prot_and_deprot_instances() #Update consolidated instance energy prot_consolidated.energy = min(x.energy for x in protonated) deprot_consolidated.energy = min(x.energy for x in deprotonated) def consolidate(self): """Each residue has multiple protonated and deprotonated states. Here we consolidate those into two states for each residue, PROT and DEPROT. We take minimums of energies between states in each class. For example, assume we have two amino acids, A and B, where A has protonated states 1, 2, 3 and deprotonated state 4, and B has protonated states 1, 2, and deprotonated state 3. Then E(A_PROT, B_PROT) = min{E(A1,B1), E(A1,B2), E(A2,B1), E(A2,B2), E(A3,B1), E(A3,B2)}, E(A_PROT, B_DEPROT) = min{E(A1,B3), E(A2,B3), E(A3,B3)}, E(A_DEPROT, B_PROT) = min{E(A4,B1), E(A4,B2)}, and E(A_DEPROT, B_DEPROT) = E(A4,B3). We do not deal with HIS here, it is kept in its 3 states for now. After this is finished all unused interaction energies will be removed from self.interaction_energies.""" handled_interaction_pairs = set() #See https://docs.python.org/2/library/itertools.html#itertools.combinations for v, w in combinations(iter(self.residue_variables.items()), 2): v_key, v_residue = v w_key, w_residue = w v_name = v_key[0] w_name = w_key[0] #Skip HIS for now if v_name == 'HIS' or w_name == 'HIS': continue v_protinated, v_unprotonated = v_residue.get_prot_and_deprot_instances( ) v_prot_consolidated = v_residue.instances["PROTONATED"] v_deprot_consolidated = v_residue.instances["DEPROTONATED"] w_protinated, w_unprotonated = w_residue.get_prot_and_deprot_instances( ) w_prot_consolidated = w_residue.instances["PROTONATED"] w_deprot_consolidated = w_residue.instances["DEPROTONATED"] #For every pairing of v and w (protonated and unprotonated) find the # minimum interaction energy and update the interaction map accordingly. v_stuff = ((v_protinated, v_prot_consolidated), (v_unprotonated, v_deprot_consolidated)) w_stuff = ((w_protinated, w_prot_consolidated), (w_unprotonated, w_deprot_consolidated)) for v_product, w_product in product(v_stuff, w_stuff): v_instances, v_consolidated = v_product w_instances, w_consolidated = w_product energies = [] #Find all of the pairs for v_instance, w_instance in product(v_instances, w_instances): energy = self.interaction_energies[v_instance, w_instance] energies.append(energy) #Mark for deletion. handled_interaction_pairs.add((v_instance, w_instance)) min_energy = min(energies) self.add_interaction_energy_pair(v_consolidated, w_consolidated, min_energy) #Now handle HIS. #See https://docs.python.org/2/library/itertools.html#itertools.permutations for v, w in permutations(iter(self.residue_variables.items()), 2): his_key, his_residue = v other_key, other_residue = w his_name = his_key[0] other_name = other_key[0] #We only care about his this pass if his_name != 'HIS': continue #HIS - HIS is already correct for what this pass does. if other_name == 'HIS': continue other_protinated, other_unprotonated = other_residue.get_prot_and_deprot_instances( ) other_prot_consolidated = other_residue.instances["PROTONATED"] other_deprot_consolidated = other_residue.instances["DEPROTONATED"] his_protinated, his_unprotonated = his_residue.get_prot_and_deprot_instances( ) his_stuff = his_protinated + his_unprotonated other_stuff = ((other_protinated, other_prot_consolidated), (other_unprotonated, other_deprot_consolidated)) #For every pairing of a HIS instance and another non HIS residue find the # minimum interaction energy and update the interaction map accordingly. #See https://docs.python.org/2/library/itertools.html#itertools.product for his_instance, other_product in product(his_stuff, other_stuff): other_instances, other_consolidated = other_product energies = [] for other_instance in other_instances: energy = self.interaction_energies[his_instance, other_instance] energies.append(energy) handled_interaction_pairs.add( (his_instance, other_instance)) min_energy = min(energies) self.add_interaction_energy_pair(his_instance, other_consolidated, min_energy) #Clean out unused interaction energies self.drop_interaction_pairs(handled_interaction_pairs) for key, residue in self.residue_variables.items(): name = key[0] if name == 'HIS': continue residue.instances = OrderedDict( (k, v) for k, v in residue.instances.items() if "PROTONATED" in k) def divide_his(self): """Here we divide HIS into two residues - HID, HIE - each with half the pKa value. We have to set interaction energies between HIS and other residues and for HIS-HIS interactions. Do this based on the values given in the paper""" to_drop_his = set() new_to_old_his = {} handled_interaction_pairs = set() items = list(self.residue_variables.items()) #Find all HIS and split them. for key, residue in items: name = key[0] if name != 'HIS': continue old_instance_1 = residue.get_instance('1') old_instance_2 = residue.get_instance('2') old_instance_12 = residue.get_instance('1+2') base_name = '_' + '_'.join(key[-2:]) + '_' to_drop_his.add(key) hid = ResidueVariable("HId" + base_name) name = "HId" + base_name + "PROTONATED" hid_prot = ResidueInstance(True, name, energy=old_instance_1.energy) name = "HId" + base_name + "DEPROTONATED" hid_deprot = ResidueInstance(False, name) hid.instances["PROTONATED"] = hid_prot hid.instances["DEPROTONATED"] = hid_deprot res_tuple = ("HId", ) + key[-2:] self.residue_variables[res_tuple] = hid new_to_old_his[hid] = residue hie = ResidueVariable("HIe" + base_name) name = "HIe" + base_name + "PROTONATED" hie_prot = ResidueInstance(True, name, energy=old_instance_2.energy) name = "HIe" + base_name + "DEPROTONATED" hie_deprot = ResidueInstance(False, name) hie.instances["PROTONATED"] = hie_prot hie.instances["DEPROTONATED"] = hie_deprot res_tuple = ("HIe", ) + key[-2:] self.residue_variables[res_tuple] = hie #Keep a mapping to the original residue object so we can look up # interaction energies in the HIS/HIS interactions loop below. new_to_old_his[hie] = residue #Create interaction energies between newly created residues. energy = old_instance_12.energy - old_instance_1.energy - old_instance_2.energy self.add_interaction_energy_pair(hid_prot, hie_prot, energy) self.add_interaction_energy_pair(hid_prot, hie_deprot, 0.0) self.add_interaction_energy_pair(hid_deprot, hie_prot, 0.0) self.add_interaction_energy_pair(hid_deprot, hie_deprot, sys.float_info.max) #Delete residue variables from main map. for key in to_drop_his: del self.residue_variables[key] #This loop is order dependent for HIS/HIS interactions. That is why some of the code paths that #appear as though they should be duplicate or symmetrical are not #and why we compare chain locations. # (In case you were wondering why HIe <-> HId is not the same as # HId <-> HIe.) #See https://docs.python.org/2/library/itertools.html#itertools.product for v, w in product(iter(self.residue_variables.items()), repeat=2): his_key, his_residue = v other_key, other_residue = w his_name, his_chain, his_location = his_key other_name, other_chain, other_location = other_key if his_name not in ('HId', 'HIe'): continue his_location = int(his_location) other_location = int(other_location) his_prot = his_residue.instances["PROTONATED"] his_deprot = his_residue.instances["DEPROTONATED"] old_his = new_to_old_his[his_residue] old_his_instance_1 = old_his.get_instance('1') old_his_instance_2 = old_his.get_instance('2') old_his_instance_12 = old_his.get_instance('1+2') other_prot = other_residue.instances["PROTONATED"] other_deprot = other_residue.instances["DEPROTONATED"] #Handle create interaction with HId if other_name == 'HId': # HIS/HIS is order dependent. if his_chain > other_chain or his_location >= other_location: continue old_other = new_to_old_his[other_residue] old_other_instance_1 = old_other.get_instance('1') old_other_instance_2 = old_other.get_instance('2') old_other_instance_12 = old_other.get_instance('1+2') if his_name == 'HIe': energy = self.interaction_energies[old_his_instance_12, old_other_instance_12] self.add_interaction_energy_pair(his_prot, other_prot, energy) self.add_interaction_energy_pair(his_prot, other_deprot, 0.0) self.add_interaction_energy_pair(his_deprot, other_prot, 0.0) energy = ( self.interaction_energies[old_his_instance_1, old_other_instance_2] - self.interaction_energies[old_his_instance_1, old_other_instance_12] - self.interaction_energies[old_his_instance_12, old_other_instance_2]) self.add_interaction_energy_pair(his_deprot, other_deprot, energy) elif his_name == 'HId': self.add_interaction_energy_pair(his_prot, other_prot, 0.0) energy = self.interaction_energies[old_his_instance_12, old_other_instance_2] self.add_interaction_energy_pair(his_prot, other_deprot, energy) energy = ( self.interaction_energies[old_his_instance_2, old_other_instance_12] - self.interaction_energies[old_his_instance_12, old_other_instance_12]) self.add_interaction_energy_pair(his_deprot, other_prot, energy) energy = self.interaction_energies[old_his_instance_2, old_other_instance_2] self.add_interaction_energy_pair(his_deprot, other_deprot, energy) combinations = self.get_interaction_combinations( (old_his_instance_1, old_his_instance_2, old_his_instance_12), (old_other_instance_1, old_other_instance_2, old_other_instance_12)) handled_interaction_pairs.update(combinations) #Handle create interaction with HIe elif other_name == 'HIe': # HIS/HIS is order dependent. if his_chain > other_chain or his_location >= other_location: continue old_other = new_to_old_his[other_residue] old_other_instance_1 = old_other.get_instance('1') old_other_instance_2 = old_other.get_instance('2') old_other_instance_12 = old_other.get_instance('1+2') if his_name == 'HIe': self.add_interaction_energy_pair(his_prot, other_prot, 0.0) energy = (self.interaction_energies[old_his_instance_12, old_other_instance_1] - self.interaction_energies[old_his_instance_12, old_other_instance_12]) self.add_interaction_energy_pair(his_prot, other_deprot, energy) energy = self.interaction_energies[old_his_instance_1, old_other_instance_12] self.add_interaction_energy_pair(his_deprot, other_prot, energy) energy = self.interaction_energies[old_his_instance_1, old_other_instance_1] self.add_interaction_energy_pair(his_deprot, other_deprot, energy) elif his_name == 'HId': self.add_interaction_energy_pair(his_prot, other_prot, 0.0) self.add_interaction_energy_pair(his_prot, other_deprot, 0.0) self.add_interaction_energy_pair(his_deprot, other_prot, 0.0) energy = ( self.interaction_energies[old_his_instance_2, old_other_instance_1] + self.interaction_energies[old_his_instance_12, old_other_instance_12] - self.interaction_energies[old_his_instance_2, old_other_instance_12] - self.interaction_energies[old_his_instance_12, old_other_instance_1]) self.add_interaction_energy_pair(his_deprot, other_deprot, energy) combinations = self.get_interaction_combinations( (old_his_instance_1, old_his_instance_2, old_his_instance_12), (old_other_instance_1, old_other_instance_2, old_other_instance_12)) handled_interaction_pairs.update(combinations) #Handle create interaction with non-HIS else: if his_name == 'HIe': energy = (self.interaction_energies[old_his_instance_12, other_prot] - self.interaction_energies[old_his_instance_1, other_prot]) self.add_interaction_energy_pair(his_prot, other_prot, energy) energy = self.interaction_energies[old_his_instance_2, other_deprot] self.add_interaction_energy_pair(his_prot, other_deprot, energy) self.add_interaction_energy_pair(his_deprot, other_prot, 0.0) energy = (self.interaction_energies[old_his_instance_1, other_deprot] + self.interaction_energies[old_his_instance_2, other_deprot] - self.interaction_energies[old_his_instance_12, other_deprot]) self.add_interaction_energy_pair(his_deprot, other_deprot, energy) elif his_name == 'HId': energy = self.interaction_energies[old_his_instance_1, other_prot] self.add_interaction_energy_pair(his_prot, other_prot, energy) energy = (self.interaction_energies[old_his_instance_12, other_deprot] - self.interaction_energies[old_his_instance_2, other_deprot]) self.add_interaction_energy_pair(his_prot, other_deprot, energy) energy = (self.interaction_energies[old_his_instance_1, other_prot] + self.interaction_energies[old_his_instance_2, other_prot] - self.interaction_energies[old_his_instance_12, other_prot]) self.add_interaction_energy_pair(his_deprot, other_prot, energy) self.add_interaction_energy_pair(his_deprot, other_deprot, 0.0) residue_combinations = self.get_interaction_combinations( (old_his_instance_1, old_his_instance_2, old_his_instance_12), (other_prot, other_deprot)) name_combinations = [(tup[0], tup[1]) for tup in residue_combinations] handled_interaction_pairs.update(name_combinations) #Clean out unused interaction energies self.drop_interaction_pairs(handled_interaction_pairs) def evaluate_energy(self, labeling, normal_form=True, pH=0.0): """Get the total energy for the residue in the state specified by labeling. normal_form and pH are used for testing.""" energy = 0.0 if not normal_form: ph_multiplier = 0.0 for instance in list(labeling.values()): if instance.protonated: ph_multiplier += 1.0 #See https://docs.python.org/2/library/itertools.html#itertools.combinations for v, w in combinations(iter(self.residue_variables.items()), 2): v_key, v_residue = v w_key, w_residue = w is_same_his = (v_key[1:] == w_key[1:] and v_key[0] in ("HId", "HIe") and w_key[0] in ("HId", "HIe")) if is_same_his: if labeling[v_residue].protonated and labeling[ w_residue].protonated: ph_multiplier -= 2.0 for v in self.residue_variables.values(): v_instance = labeling[v] energy += v_instance.energyNF if normal_form else v_instance.energy ie = self.normalized_interaction_energies if normal_form else self.interaction_energies for v, w in combinations(iter(self.residue_variables.values()), 2): v_instance = labeling[v] w_instance = labeling[w] energy += ie[v_instance, w_instance] if normal_form: energy += self.normalized_constant_energy else: energy += pH * ph_multiplier return energy def instance_interaction_energy(self, instance, labeling, interaction_energy): return sum( interaction_energy.get((instance, labeling[x]), 0) for x in self.residue_variables.values()) def evaluate_energy_diff(self, residue, labeling, normal_form=False): """Returns the total energy difference between the deprontated and protonated states for the supplied residue. (protonated total energy - deprotonated total energy)""" ie = self.normalized_interaction_energies if normal_form else self.interaction_energies prot_instance = residue.instances["PROTONATED"] prot_energy = 0 for x in self.residue_variables.values(): prot_energy += ie.get((prot_instance, labeling[x]), 0.0) prot_energy += prot_instance.energyNF if normal_form else prot_instance.energy deprot_instance = residue.instances["DEPROTONATED"] deprot_energy = sum( ie.get((deprot_instance, labeling[x]), 0.0) for x in self.residue_variables.values()) deprot_energy += deprot_instance.energyNF if normal_form else deprot_instance.energy return prot_energy - deprot_energy def evaluate_energy_diff_his(self, hie_residue, hid_residue, labeling, normal_form=False): """Returns the total energy differences between 1. HSE total energy and HSP total energy (HSP total energy - HSE total energy) 2. HSD total energy and HSP total energy (HSP total energy - HSD total energy) in a tuple.""" # With both HIE and HID in protonated states: # HSP energy adds: # unary energy of all not-this-HIS residues in current state # unary energy of HSE_protonated # unary energy of HSD_protonated # binary energy of all not-this-HIS residues in current state # binary energy of all not-this-HIS (current state) with HSE_protonated # binary energy of all not-this-HIS (current state) with HSD_protonated # # HSE energy adds: # unary energy of all not-this-HIS residues in current state # unary energy of HSE_protonated # unary energy of HSD_deprotonated # binary energy of all not-this-HIS residues in current state # binary energy of all not-this-HIS (current state) with HSE_protonated # binary energy of all not-this-HIS (current state) with HSD_deprotonated # # HSD energy adds: # unary energy of all not-this-HIS residues in current state # unary energy of HSE_deprotonated # unary energy of HSD_protonated # binary energy of all not-this-HIS residues in current state # binary energy of all not-this-HIS (current state) with HSE_deprotonated # binary energy of all not-this-HIS (current state) with HSD_protonated # # HSP-HSE has # unary energy of HSD_protonated (EHd1) minus # unary energy of HSD_deprotonated (EHd0) # binary energy of all not-this-HIS (current state) with HSD_protonated (sum_ErHd_a1) minus # binary energy of all not-this-HIS (current state) with HSD_deprotonated (sum_ErHd_a0) # # HSP-HSD has # unary energy of HSE_protonated (EHe1) minus # unary energy of HSE_deprotonated (EHe0) # binary energy of all not-this-HIS (current state) with HSE_protonated (sum_ErHe_a1) minus # binary energy of all not-this-HIS (current state) with HSE_deprotonated (sum_ErHe_a0) labeling_copy = labeling.copy() ie = self.normalized_interaction_energies if normal_form else self.interaction_energies hie_prot_instance = hie_residue.instances["PROTONATED"] hie_deprot_instance = hie_residue.instances["DEPROTONATED"] hid_prot_instance = hid_residue.instances["PROTONATED"] hid_deprot_instance = hid_residue.instances["DEPROTONATED"] labeling_copy[hid_residue] = hid_prot_instance labeling_copy[hie_residue] = hie_prot_instance if normal_form: EHd1 = hid_prot_instance.energyNF EHd0 = hid_deprot_instance.energyNF else: EHd1 = hid_prot_instance.energy EHd0 = hid_deprot_instance.energy sum_ErHd_a1 = sum( ie.get((labeling_copy[x], hid_prot_instance), 0) for x in self.residue_variables.values()) sum_ErHd_a0 = sum( ie.get((labeling_copy[x], hid_deprot_instance), 0) for x in self.residue_variables.values()) if normal_form: EHe1 = hie_prot_instance.energyNF EHe0 = hie_deprot_instance.energyNF else: EHe1 = hie_prot_instance.energy EHe0 = hie_deprot_instance.energy sum_ErHe_a1 = sum( ie.get((labeling_copy[x], hie_prot_instance), 0) for x in self.residue_variables.values()) sum_ErHe_a0 = sum( ie.get((labeling_copy[x], hie_deprot_instance), 0) for x in self.residue_variables.values()) hsp_hse = EHd1 - EHd0 + sum_ErHd_a1 - sum_ErHd_a0 hsp_hsd = EHe1 - EHe0 + sum_ErHe_a1 - sum_ErHe_a0 return hsp_hse, hsp_hsd def normalize(self, pH): """Finds and stores the normal form of all instance and interaction energies at the supplied pH value. Instance energies are saved in instance.energyNF and interaction energies are stored in self.interaction_energies""" self.normalized_interaction_energies = self.interaction_energies.copy() self.normalized_constant_energy = 0.0 #Add the pH to the instances. for residue in self.residue_variables.values(): residue.instances["DEPROTONATED"].energyNF = residue.instances[ "DEPROTONATED"].energy # TODO - I'm not sure why this is +pH.... residue.instances["PROTONATED"].energyNF = residue.instances[ "PROTONATED"].energy + math.log(10) * pH rv = self.residue_variables keys = list(rv.keys()) #Add the pH values to the HIS self interactions. for v_key, w_key in combinations(keys, 2): v_residue = rv[v_key] w_residue = rv[w_key] is_same_his = (v_key[1:] == w_key[1:] and v_key[0] in ("HId", "HIe") and w_key[0] in ("HId", "HIe")) if is_same_his: v_prot = v_residue.instances["PROTONATED"] w_prot = w_residue.instances["PROTONATED"] self.normalized_interaction_energies[v_prot, w_prot] -= (2.0 * math.log(10) * pH) self.normalized_interaction_energies[w_prot, v_prot] -= (2.0 * math.log(10) * pH) #Normalize the interaction energies. for v_key, w_key in permutations(keys, 2): v_residue = rv[v_key] w_residue = rv[w_key] for v_instance in v_residue.instances.values(): w_instances = list(w_residue.instances.values()) min_energy = min( self.normalized_interaction_energies[v_instance, w_instance] for w_instance in w_instances) v_instance.energyNF += min_energy if min_energy != sys.float_info.max: for w_instance in w_instances: self.normalized_interaction_energies[ v_instance, w_instance] -= min_energy self.normalized_interaction_energies[ w_instance, v_instance] -= min_energy #Normalize the instance energies. for residue in self.residue_variables.values(): min_energy = min(instance.energyNF for instance in residue.instances.values()) self.normalized_constant_energy += min_energy if min_energy != sys.float_info.max: for instance in residue.instances.values(): instance.energyNF -= min_energy def energy_at_pH(self, pH): """Create a representation of the protein energy at the specified pH Used primarily for testing.""" self.interaction_energies_for_ph = self.interaction_energies.copy() for residue in self.residue_variables.values(): residue.instances[ "DEPROTONATED"].energy_with_ph = residue.instances[ "DEPROTONATED"].energy residue.instances["PROTONATED"].energy_with_ph = residue.instances[ "PROTONATED"].energy + math.log(10) * pH for v, w in combinations(iter(self.residue_variables.items()), 2): v_key, v_residue = v w_key, w_residue = w is_same_his = (v_key[1:] == w_key[1:] and v_key[0] in ("HId", "HIe") and w_key[0] in ("HId", "HIe")) if is_same_his: v_prot = v_residue.instances["PROTONATED"] w_prot = w_residue.instances["PROTONATED"] self.interaction_energies_for_ph[v_prot, w_prot] -= (2.0 * math.log(10) * pH) self.interaction_energies_for_ph[w_prot, v_prot] -= (2.0 * math.log(10) * pH)
class IBMQProvider(BaseProvider): """Provider for remote IBMQ backends with admin features. This class is the entry point for handling backends from IBMQ, allowing using different accounts. """ def __init__(self): super().__init__() # dict[credentials_unique_id: IBMQSingleProvider] # This attribute stores a reference to the different accounts. The # keys are tuples (hub, group, project), as the convention is that # that tuple uniquely identifies a set of credentials. self._accounts = OrderedDict() def backends(self, name=None, filters=None, **kwargs): """Return all backends accessible via IBMQ provider, subject to optional filtering. Args: name (str): backend name to filter by filters (callable): more complex filters, such as lambda functions e.g. IBMQ.backends(filters=lambda b: b.configuration['n_qubits'] > 5) kwargs: simple filters specifying a true/false criteria in the backend configuration or backend status or provider credentials e.g. IBMQ.backends(n_qubits=5, operational=True, hub='internal') Returns: list[IBMQBackend]: list of backends available that match the filter Raises: IBMQAccountError: if no account matched the filter. """ # pylint: disable=arguments-differ # Special handling of the credentials filters: match and prune from kwargs credentials_filter = {} for key in [ 'token', 'url', 'hub', 'group', 'project', 'proxies', 'verify' ]: if key in kwargs: credentials_filter[key] = kwargs.pop(key) providers = [ provider for provider in self._accounts.values() if self._credentials_match_filter(provider.credentials, credentials_filter) ] # Special handling of the `name` parameter, to support alias resolution. if name: aliases = self.aliased_backend_names() aliases.update(self.deprecated_backend_names()) name = aliases.get(name, name) # Aggregate the list of filtered backends. backends = [] for provider in providers: backends = backends + provider.backends( name=name, filters=filters, **kwargs) return backends @staticmethod def deprecated_backend_names(): """Returns deprecated backend names.""" return { 'ibmqx_qasm_simulator': 'ibmq_qasm_simulator', 'ibmqx_hpc_qasm_simulator': 'ibmq_qasm_simulator', 'real': 'ibmqx1' } @staticmethod def aliased_backend_names(): """Returns aliased backend names.""" return { 'ibmq_5_yorktown': 'ibmqx2', 'ibmq_5_tenerife': 'ibmqx4', 'ibmq_16_rueschlikon': 'ibmqx5', 'ibmq_20_austin': 'QS1_1' } def enable_account(self, token, url=QE_URL, **kwargs): """Authenticate a new IBMQ account and add for use during this session. Login into Quantum Experience or IBMQ using the provided credentials, adding the account to the current session. The account is not stored in disk. Args: token (str): Quantum Experience or IBM Q API token. url (str): URL for Quantum Experience or IBM Q (for IBM Q, including the hub, group and project in the URL). **kwargs (dict): * proxies (dict): Proxy configuration for the API. * verify (bool): If False, ignores SSL certificates errors """ credentials = Credentials(token, url, **kwargs) self._append_account(credentials) def save_account(self, token, url=QE_URL, overwrite=False, **kwargs): """Save the account to disk for future use. Login into Quantum Experience or IBMQ using the provided credentials, adding the account to the current session. The account is stored in disk for future use. Args: token (str): Quantum Experience or IBM Q API token. url (str): URL for Quantum Experience or IBM Q (for IBM Q, including the hub, group and project in the URL). overwrite (bool): overwrite existing credentials. **kwargs (dict): * proxies (dict): Proxy configuration for the API. * verify (bool): If False, ignores SSL certificates errors """ credentials = Credentials(token, url, **kwargs) store_credentials(credentials, overwrite=overwrite) def active_accounts(self): """List all accounts currently in the session. Returns: list[dict]: a list with information about the accounts currently in the session. """ information = [] for provider in self._accounts.values(): information.append({ 'token': provider.credentials.token, 'url': provider.credentials.url, }) return information def stored_accounts(self): """List all accounts stored to disk. Returns: list[dict]: a list with information about the accounts stored on disk. """ information = [] stored_creds = read_credentials_from_qiskitrc() for creds in stored_creds: information.append({ 'token': stored_creds[creds].token, 'url': stored_creds[creds].url }) return information def load_accounts(self, **kwargs): """Load IBMQ accounts found in the system into current session, subject to optional filtering. Automatically load the accounts found in the system. This method looks for credentials in the following locations, in order, and returns as soon as credentials are found: 1. in the `Qconfig.py` file in the current working directory. 2. in the environment variables. 3. in the `qiskitrc` configuration file Raises: IBMQAccountError: if no credentials are found. """ for credentials in discover_credentials().values(): if self._credentials_match_filter(credentials, kwargs): self._append_account(credentials) if not self._accounts: raise IBMQAccountError('No IBMQ credentials found on disk.') def disable_accounts(self, **kwargs): """Disable accounts in the current session, subject to optional filtering. The filter kwargs can be `token`, `url`, `hub`, `group`, `project`. If no filter is passed, all accounts in the current session will be disabled. Raises: IBMQAccountError: if no account matched the filter. """ disabled = False # Try to remove from session. current_creds = self._accounts.copy() for creds in current_creds: credentials = Credentials(current_creds[creds].credentials.token, current_creds[creds].credentials.url) if self._credentials_match_filter(credentials, kwargs): del self._accounts[credentials.unique_id()] disabled = True if not disabled: raise IBMQAccountError( 'No matching account to disable in current session.') def delete_accounts(self, **kwargs): """Delete saved accounts from disk, subject to optional filtering. The filter kwargs can be `token`, `url`, `hub`, `group`, `project`. If no filter is passed, all accounts will be deleted from disk. Raises: IBMQAccountError: if no account matched the filter. """ deleted = False # Try to delete from disk. stored_creds = read_credentials_from_qiskitrc() for creds in stored_creds: credentials = Credentials(stored_creds[creds].token, stored_creds[creds].url) if self._credentials_match_filter(credentials, kwargs): remove_credentials(credentials) deleted = True if not deleted: raise IBMQAccountError('No matching account to delete from disk.') def _append_account(self, credentials): """Append an account with the specified credentials to the session. Args: credentials (Credentials): set of credentials. Returns: IBMQSingleProvider: new single-account provider. """ # Check if duplicated credentials are already in use. By convention, # we assume (hub, group, project) is always unique. if credentials.unique_id() in self._accounts.keys(): warnings.warn('Credentials are already in use.') single_provider = IBMQSingleProvider(credentials, self) self._accounts[credentials.unique_id()] = single_provider return single_provider def _credentials_match_filter(self, credentials, filter_dict): """Return True if the credentials match a filter. These filters apply on properties of a Credentials object: token, url, hub, group, project, proxies, verify Any other filter has no effect. Args: credentials (Credentials): IBMQ credentials object filter_dict (dict): dictionary of filter conditions Returns: bool: True if the credentials meet all the filter conditions """ return all( getattr(credentials, key_, None) == value_ for key_, value_ in filter_dict.items())
class ModuleStruct: """ Define a module struct which stores all info. to generate statement. Args: nd_struct_list (list): A list of node structs. init_as_parent (bool): Control init method if the ModuleStruct be init as a parent module struct. parent_base (ModuleStruct): The base ModuleStruct the current ModuleStruct to be init as. """ def __init__(self, nd_struct_list, init_as_parent=False, parent_base=None): """Init. a module by NodeStructs.""" self.pattern_id = -1 # pattern num, -1 as Main module self.pattern_uid = -1 # unique module id for this pattern self.parent_id = None # parent's pattern num self.parent_uid = None # parent's pattern module unique id self.initialized = False self.identifier = None self.module_name = None self.scope_depth = None self.head_nd_struct = None self.head_nd_struct_index = None self.tail_nd_struct = None self.tail_nd_struct_index = None self._node_structs = list() self._module_structs = list() self._fragment = None self._args_translator = None self._parent_module_struct = None # only store original formal args name, not global self._nodes_structs_formal_args_list = list() # define other settings here self._node_args_translation_list = list() self._var_name_mgr = LocalVarNameMgr() self.construct_header_x = OrderedDict( ) # key is header x, value is precursors onnx name self.inputs_in_construct_header = OrderedDict( ) # key is precursors onnx name, value is x in parent construct # key is node's onnx name(output provider), value is (provider_succ_name, opt_var_name) self.outputs_collection = dict() self.matched_inputs = list( ) # Matched inputs will can be directly used by code line generation # key is ext. succ node onnx name, value is local opt_var self.external_successor_local_returns_map = OrderedDict() # Define outputs manager, note this will be assigned later by Generator. self.outputs_manager = None self._global_context = GlobalContext() # Define a dict to store the reference for quick searching self.rapid_reference = dict() # new vars for matcher self.inputs_register = OrderedDict() # reg by sub self.outputs_register = OrderedDict() # reg by sub self.internal_outputs_collection = dict() # reg by sub # new vars for shared weights self.shared_weights_collection = dict() # reg by sub self.shared_weights_counter = 0 # updated by sub if init_as_parent and (parent_base is not None): self.reset_as_parent_passed_in(parent_base) else: # start initialization if not self.initialized: self._init_module(nd_struct_list) else: self._update_module(nd_struct_list) # assign this module reference to node for (_, nd_struct) in nd_struct_list: nd_struct.parent_module_struct = self def reset_as_parent_passed_in(self, parent_base): """ Reset all attributes and filled as a parent module of the module passed in. Args: parent_base(ModuleStruct): The base ModuleStruct to be passed in for ModuleStruct init. Note: This function must be called only if the new ModuleStruct is a parent of parent_base. """ self.identifier = copy.deepcopy(parent_base.identifier)[:-1] self.scope_depth = copy.deepcopy(parent_base.scope_depth) - 1 self.module_name = Scope.scope_to_module_name(self.identifier) self.head_nd_struct = parent_base.head_nd_struct self.head_nd_struct_index = parent_base.head_nd_struct_index self.tail_nd_struct = parent_base.tail_nd_struct self.tail_nd_struct_index = parent_base.tail_nd_struct_index self._node_structs = list() self._module_structs = list() self._fragment = None self._args_translator = None self.initialized = True self._set_pattern_id() self._find_parent_module() self.init_args_translator() self._parent_module_struct = None self._nodes_structs_formal_args_list = list() self._node_args_translation_list = list() def _set_pattern_id(self): """Set pattern id which matches the module fragment pattern.""" if not self.initialized: return if self.scope_depth < 1: self.pattern_id = -1 self.pattern_uid = -1 return self.pattern_id = self.identifier[-1][0] self.pattern_uid = self.identifier[-1][1] def _init_module(self, nd_struct_list): """Init this ModuleStruct by a list of Nodes.""" (nd_topo_idx, nd_struct) = nd_struct_list[0] self.identifier = nd_struct.scope.path self.module_name = nd_struct.scope.to_str self.scope_depth = nd_struct.scope.depth self.head_nd_struct = nd_struct self.head_nd_struct_index = nd_topo_idx self.tail_nd_struct = nd_struct_list[-1][1] self.tail_nd_struct_index = nd_struct_list[-1][0] self._node_structs = nd_struct_list self.initialized = True self._set_pattern_id() self._find_parent_module() self.init_args_translator() def _update_module(self, nd_struct_list): """Update the ModuleStruct attributes from a list of Nodes.""" (nd_topo_idx_head, nd_struct_head) = nd_struct_list[0] (nd_topo_idx_tail, nd_struct_tail) = nd_struct_list[-1] if self.identifier != nd_struct_head.scope.path: raise ValueError( "Unable to update this module struct {} due to different identifier {}" .format(self.identifier, nd_struct_head.scope.path)) if nd_topo_idx_head < self.head_nd_struct_index: self.head_nd_struct_index = nd_topo_idx_head self.head_nd_struct = nd_struct_head if nd_topo_idx_tail > self.tail_nd_struct_index: self.tail_nd_struct_index = nd_topo_idx_tail self.tail_nd_struct = nd_struct_tail self._node_structs += nd_struct_list def _find_parent_module(self): """Set the parent's module pattern and uid.""" if not self.initialized: return if self.scope_depth == 0: # is Main Module pass elif self.scope_depth == 1: # parent pattern is Main module self.parent_id = -1 self.parent_uid = -1 else: # this is a submodule in a module (self.parent_id, self.parent_uid) = Scope.get_parent_module_num_and_uid( self.identifier) def __repr__(self): return str({ "address": hex(id(self)), "identifier": self.identifier, "parent": (self.parent_id, self.parent_uid), "name": self.module_name, "pattern": self.pattern_id, "scope_depth": self.scope_depth, "nd_idx_range": "{} -> {}".format(self.head_nd_struct_index, self.tail_nd_struct_index), "initialized": self.initialized }) def init_args_translator(self): """Initialize the Args Translator for the module.""" var_name = self.ms_var_name self._args_translator = ArgsTranslation(None, var_name, None) def update_module_fragment(self): """Update this module's fragment.""" if self._fragment is None: return # update input output shape self._fragment.input_shape = self.head_nd_struct.fragment.input_shape self._fragment.output_shape = self.tail_nd_struct.fragment.output_shape # update formal args self._fragment.formal_args.update(self._args_translator.formal_args) self._fragment.formal_args_value.update( self._args_translator.formal_args_values) # update actual args self._fragment.actual_args.update(self._args_translator.actual_args) # update others.. def add_submodule(self, md_structs): """ Add another module struct(s) to this ModuleStruct. Args: md_structs ([ModuleStruct, list]): a (list) ModuleStruct to be added in this ModuleStruct. """ tail_md = md_structs if isinstance(md_structs, ModuleStruct): md_structs.args_translator.take_formal_args_from_nodes_and_submodules( md_structs.get_all_sub_translators()) self._module_structs.append(md_structs) md_structs.parent_module_struct = self elif isinstance(md_structs, list): for md_s in md_structs: md_s.args_translator.take_formal_args_from_nodes_and_submodules( md_s.get_all_sub_translators()) md_s.parent_module_struct = self self._module_structs += md_structs tail_md = md_structs[-1] else: raise TypeError( "ModuleStruct cannot add an unsupported Type {} to module_structs list." .format(type(md_structs))) # update tail node and index if self.tail_nd_struct_index < tail_md.tail_nd_struct_index: self.tail_nd_struct = tail_md.tail_nd_struct self.tail_nd_struct_index = tail_md.tail_nd_struct_index def _update_formal_args_for_all_nd_structs(self): """ Init nodes' args translator and find formal args. And collect nodes' formal args. """ if len(self._node_args_translation_list) != len(self._node_structs): raise ValueError( "ModuleStruct cannot update nodes' formal args due to length inconsistent." ) for idx, (_, nd_struct) in enumerate(self._node_structs): formal_arg_of_this_node = self._node_args_translation_list[idx] # update var_name to ensure all node names' are unique in a module. nd_struct.update_var_name(idx) nd_struct.init_args_translator(formal_arg_of_this_node) if nd_struct.args_translator is not None: self._nodes_structs_formal_args_list.append( nd_struct.args_translator.formal_args_values) else: self._nodes_structs_formal_args_list.append(None) def update_args_translation_list(self, formal_args): """ Receive a list of args name to be changed to formal args, and change them. Args: formal_args (list[str]): a list of args name to be changed to formal args. """ self._node_args_translation_list = formal_args self._update_formal_args_for_all_nd_structs() def get_all_sub_translators(self): """ Return a list of args_translators of submodules / nodes affiliated to this module. Note: The order of returned list is followed by the actual topological order. Returns: list, a list of args_translators. """ ret = [] for (_, struct) in self.get_generate_order(): if struct.args_translator is not None: ret.append(struct.args_translator) return ret def get_generate_order(self): """ Return the order of generated code by index. Return: list, a list of reference of node_struct or module_struct. """ ret = list() if not self._module_structs: return self._node_structs # Generate a list of tuple (idx, module_structs) for md_struct in self._module_structs: ret.append((md_struct.head_nd_struct_index, md_struct)) if self.node_structs: ret += self.node_structs ret.sort(key=lambda x: x[0]) return ret def _code_line_init_statement_shared_weights_args(self): """Generate the args for shared weights where calling this module.""" args_list = list() for passthrough_w_onnx_name, passthrough_w_var_name in self.shared_weights_collection.items( ): passthrough_w_var_name_in_parent = \ self.parent_module_struct.shared_weights_collection.get(passthrough_w_onnx_name) if self.parent_module_struct.identifier == []: # now only consider declaration in main model args_list.append( f"{passthrough_w_var_name}=self.{passthrough_w_var_name_in_parent}" ) else: args_list.append( f"{passthrough_w_var_name}={passthrough_w_var_name_in_parent}" ) return args_list def _code_line_init_generate_shared_w_declaration_for_repeated(self): """Force to repeat sub nodes init code line for fulfillment of shared weight declaration in main model.""" for _, nd_struct in self._node_structs: nd_struct.code_line_in_init() def code_line_in_init(self): """Initialization line of code in module init block.""" self._code_line_init_generate_shared_w_declaration_for_repeated() left = "self.{}".format(self.ms_var_name) args_list = list() # Load args in init statement. if self._args_translator is not None: # from args_translator if self._args_translator.actual_args: # load actual args args_list += self._args_translator.actual_args_to_str_list elif self._args_translator.actual_args_backup and self.parent_id == -1: # For modules repeated in multiple levels, the module under main model should # not use formal args as it is unnecessary -> load from actual args backup args_list += self._args_translator.actual_args_backup_to_str_list args_list += self._args_translator.formal_args_to_str_list # load from formal args else: args_list += self._fragment.actual_args args_list += self._code_line_init_statement_shared_weights_args() right = f"{self.class_name}({', '.join(args_list)})" return left, right def code_line_in_construct(self, inputs=None): """Construct line of code in module construct block.""" outputs_edges = list(self.outputs_register.keys()) num_output = len(outputs_edges) # Allocate opt_var_name if num_output == 1: # single output left = [f"{self.ms_opt_var_name}"] else: left = [ f"{self.ms_opt_var_name}_{num}" for num in range(num_output) ] inputs = [] # Update self's outputs mgr for idx, edge in enumerate(outputs_edges): base_out = self.outputs_manager.get_base_out(edge) if base_out.opt_var_name is None: print( f"ModuleStruct {self.identifier} has an output {base_out.onnx_edge_name} not has opt_var_name" ) base_out.opt_var_name = left[idx] self.parent_module_struct.internal_outputs_collection[ base_out.onnx_edge_name] = base_out.opt_var_name # Take inputs from parent & previous for input_edge in self.inputs_register: if input_edge in self.parent_module_struct.inputs_register: inputs.append( self.parent_module_struct.inputs_register.get(input_edge)) elif input_edge in self.parent_module_struct.internal_outputs_collection: inputs.append( self.parent_module_struct.internal_outputs_collection.get( input_edge)) right = f"self.{self.ms_var_name}({', '.join(inputs)})" left = ", ".join(left) return (left, right) @property def node_structs(self): """Return all node structs in this module.""" return self._node_structs @property def module_structs(self): """Return all module structs in this module.""" return self._module_structs @property def parent_module_struct(self): """Return this module's parent module struct.""" return self._parent_module_struct @parent_module_struct.setter def parent_module_struct(self, ref): """Set this modu;e's parent module struct.""" self._parent_module_struct = ref @property def args_translator(self): """Return the args translator.""" return self._args_translator @property def head_nd_struct_precursor_nodes_names(self) -> list: """Return head node's precursor nodes names.""" return self.head_nd_struct.precursor_nodes_names @property def head_nd_struct_precursor_nodes_structs(self) -> list: """Return head node's precursor nodes structs.""" return self.head_nd_struct.precursor_nodes_structs @property def tail_nd_struct_successor_nodes_names(self) -> list: """Return tail node's successor nodes names.""" return self.tail_nd_struct.successor_nodes_names @property def tail_nd_struct_successor_nodes_structs(self) -> list: """Return tail node's successor nodes structs.""" return self.tail_nd_struct.successor_nodes_structs @property def onnx_names_from_nodes(self) -> list: """Return all nodes onnx names in this module.""" if self._global_context.build_struct_finished and "_onnx_names_from_nodes" in self.rapid_reference: return self.rapid_reference["_onnx_names_from_nodes"] ret = [node.onnx_name for (_, node) in self.node_structs] if self._global_context.build_struct_finished: self.rapid_reference["_onnx_names_from_nodes"] = ret return ret @property def onnx_names_from_submodules(self) -> list: """Return all nodes onnx names in submodules of this module.""" if self._global_context.build_struct_finished and "_onnx_names_from_submodules" in self.rapid_reference: return self.rapid_reference["_onnx_names_from_submodules"] ret = [] for md_struct in self.module_structs: ret += md_struct.onnx_names if self._global_context.build_struct_finished: self.rapid_reference["_onnx_names_from_submodules"] = ret return ret @property def onnx_names(self) -> list: """Return all nodes' onnx names which contained by this module.""" if self._global_context.build_struct_finished and "_onnx_names" in self.rapid_reference: return self.rapid_reference["_onnx_names"] ret = self.onnx_names_from_nodes + self.onnx_names_from_submodules if self._global_context.build_struct_finished: self.rapid_reference["_onnx_names"] = ret return ret @property def external_precursor_nodes_names(self) -> list: """Return all precursors nodes names not in this module.""" ret = [] for _, struct in self.get_generate_order(): if isinstance(struct, NodeStruct): precursor_nodes_names = struct.precursor_nodes_names if isinstance(struct, ModuleStruct): precursor_nodes_names = struct.external_precursor_nodes_names for p_name in precursor_nodes_names: if p_name in self.onnx_names: continue ret.append(p_name) return ret @property def external_successor_nodes_names(self) -> list: """Return all successor nodes names not in this module.""" ret = [] for _, struct in self.get_generate_order(): if isinstance(struct, NodeStruct): successor_nodes_names = struct.successor_nodes_names if isinstance(struct, ModuleStruct): successor_nodes_names = struct.external_successor_nodes_names for s_name in successor_nodes_names: if s_name in self.onnx_names: continue ret.append(s_name) return ret @property def class_name(self) -> str: """Return the class name for generating code of this module.""" if self.pattern_id == -1: return "Model" if self._global_context.known_module_name.get("Module{}".format( self.pattern_id)) is not None: class_name = self._global_context.known_module_name.get( "Module{}".format(self.pattern_id)) else: class_name = "Module{}".format(self.pattern_id) return class_name @property def ms_var_name(self) -> str: """Return the variable name for generated code statement of this module.""" if self.pattern_id == -1: return "Model" return f"{self.class_name}_{self.pattern_uid}".lower() @property def ms_opt_var_name(self) -> str: """Return the variable name for generated code statement of the output of this module.""" return "{}_opt".format(self.ms_var_name).lower() # The following part will be resetting nodes' external inputs for supporting multi-in/out # and should be called after generator.recursive_form_modules() def set_inputs_in_construct_header(self, header_x, onnx_precursor_node_name): """ Mark the registered external inputs for code generation. Note: This function to be called by its parent (ModuleStruct). Args: header_x (str): The `x` in module construct header. onnx_precursor_node_name (str): The original onnx node name. """ if self.inputs_in_construct_header.get( onnx_precursor_node_name) is not None: raise ValueError( "The input from {} has already registered. Check this Module \ {} has duplicate inputs or not.".format( onnx_precursor_node_name, self.identifier)) self.inputs_in_construct_header[onnx_precursor_node_name] = header_x def allocate_construct_header_x(self, force_x=None): """ Allocate the x in construct header for each external input. Args: force_x (str): Force the arg name to customized. """ local_x_name = 'x' if force_x: # name of x indicated by external local_x_name = force_x # set construct_header_x for current module allocated = set() for prec_name in self.external_precursor_nodes_names: if prec_name in allocated: continue x_name_in_construct_header = self._var_name_mgr.get_name( local_x_name) self.construct_header_x[x_name_in_construct_header] = prec_name allocated.add(prec_name) # Assign these inputs to nodes and submodules for _, struct in self.get_generate_order(): if isinstance(struct, NodeStruct): # register node's ext input self.reset_node_external_input_to_local(struct) self.register_node_output_to_module(struct) if isinstance(struct, ModuleStruct): # reg module's ext input if not struct.construct_header_x: struct.allocate_construct_header_x() self.reset_submodule_external_input_to_local(struct) self.register_submodule_output_to_module(struct) # remove parent module's ext. map if ext nodes in this module (no need return) for user_name in self.external_successor_local_returns_map.copy().keys( ): if user_name in self.onnx_names: self.external_successor_local_returns_map.pop(user_name) def _match_node_inputs(self, struct): """Match node's inputs with its precursor nodes.""" for output_provider in struct.precursor_nodes_names: output_list = self.outputs_collection.get(output_provider) if output_list is None: # not in this module, check construct header for (self_x_name, self_output_provider) in self.construct_header_x.items(): if self_output_provider == output_provider: struct.matched_inputs.append(self_x_name) continue for output in output_list: (provider_succ, provider_closet_opt_var) = output if provider_closet_opt_var in struct.matched_inputs: continue # skip repeat if provider_succ == struct.onnx_name: struct.matched_inputs.append(provider_closet_opt_var) def _match_sub_modules_inputs(self): """ Match current module's submodules' inputs with corresponding outputs registered in current module. Description: The function matches these inputs by the following steps: 1. For each submodule in the current module, take submodule's construct header 2. Check submodule's construct header element requires an input from current module's construct header or outputs from other submodules. 3. If from current module's construct header, assign corresponding x to the submodule. If from other submodules, assign required submodule output name to the submodule. """ if not self.outputs_collection: return # skip first node for (_, struct) in self.get_generate_order(): if isinstance(struct, NodeStruct): self._match_node_inputs(struct) continue # skip node sub_construct_header = struct.construct_header_x for (_, output_provider) in sub_construct_header.items(): # check from outputs collection output_list = self.outputs_collection.get(output_provider) if output_list is None: # not in this module, need from current module construct header for (self_x_name, self_output_provider ) in self.construct_header_x.items(): if self_output_provider == output_provider: struct.matched_inputs.append(self_x_name) continue for output in output_list: (provider_succ, provider_closet_opt_var) = output if provider_closet_opt_var in struct.matched_inputs: continue # skip repeat if provider_succ in struct.onnx_names: struct.matched_inputs.append(provider_closet_opt_var) def _append_to_outputs_collection(self, provider_name, val): """ Helper function to add a nodes or submodules outputs to current module return statement. Args: provider_name (str): The onnx name of the output provider. val (list[tuple]): A list of tuple which contains the output provider's successor name and its opt_var_name. """ exist_output = self.outputs_collection.get(provider_name) if isinstance(val, tuple): val = [val] if exist_output is None: # add new entry exist_output = list() exist_output += (val) self.outputs_collection[provider_name] = exist_output def collect_returns(self): """ Collect all nodes and submodules' returns in the module. Note: The logic is to collect the return from nodes and submodules by the order of topological index. For returns from a node, it will check if the return will be used externally. If external (external means the successor a.k.a the return user has different scope with the node), add this return to current module's outputs_collection, where key is this node's original onnx_name, and value is a list of tuple(successor_name, this node's opt_var_name) For returns from a submodule, it will check if the submodule has already collected returns, If not, do it and then continue the following procedures. Now we will check each element in submodule's outputs_collection. Note that we DO NOT check submodule's returns should be continued returning, but just return them. All these returns from submodules will be changes their original nodes output (a.k.a outputs provider) `opt_var_name` to submodules' `opt_var_name`. Finally, we match the outputs and inputs in the current module level. """ for (_, struct) in self.get_generate_order(): if isinstance(struct, NodeStruct): outputs_list = [] # add these successor nodes name to collection for future use for succ in struct.successor_nodes_names: outputs_list.append((succ, struct.ms_opt_var_name)) if outputs_list: self._append_to_outputs_collection(struct.onnx_name, outputs_list) if isinstance(struct, ModuleStruct): # Remove unnecessary returns, succ are all inside current if not struct.outputs_collection: struct.collect_returns() sub_outputs_collection = struct.outputs_collection # check each returns in sub for provider_name, outputs_list in sub_outputs_collection.items( ): for output in outputs_list: (succ, _) = output # (succ, provider_opt_var_name) in output new_output = (succ, struct.ms_opt_var_name) self._append_to_outputs_collection( provider_name, new_output) self._match_sub_modules_inputs() def get_returned_opt_var_name(self) -> list: """Return a list of returned output var of this module.""" idx = 0 added_to_return = set() ret = [] for ext_successor_requested, opt_var_name_in_this_module in self.external_successor_local_returns_map.items( ): if ext_successor_requested in added_to_return: continue ret.append( (ext_successor_requested, opt_var_name_in_this_module, idx)) added_to_return.add(ext_successor_requested) return ret def reset_node_external_input_to_local(self, nd_struct): """ Reset node's input to module's construct args """ for prec_node_name in nd_struct.precursor_nodes_names_external: if prec_node_name in self.onnx_names: # prec node in current module's. continue if prec_node_name in self.construct_header_x.values(): # prec node assigned to construct header to passed in. local_x = get_dict_key_by_value(prec_node_name, self.construct_header_x) nd_struct.set_inputs_in_construct_header( local_x, prec_node_name) else: # Extra precursor nodes, raise error raise ValueError( "Found external inputs of the Node but the module does not have it." ) def reset_submodule_external_input_to_local(self, md_struct): """ Reset submodule's external input to current module. Args: md_struct (ModuleStruct): The submodule in the current module. """ # check submodule's input for _, submodule_precursor in md_struct.construct_header_x.items(): if submodule_precursor in self.onnx_names: # if internal, match with local nodes/submodules return # but do nothing here continue # if external, match with current module construct header x if submodule_precursor in self.construct_header_x.values(): local_x = get_dict_key_by_value(submodule_precursor, self.construct_header_x) md_struct.set_inputs_in_construct_header( local_x, submodule_precursor) else: # Extra precursor nodes, raise error raise ValueError( "Found external inputs of the submodule but the module does not have it." ) def register_node_output_to_module(self, nd_struct): """Register nodes outputs to this module's return.""" for succ_node_name in nd_struct.successor_nodes_names_external: self.external_successor_local_returns_map[ succ_node_name] = nd_struct.ms_opt_var_name def register_submodule_output_to_module(self, md_struct): """Register submodule outputs to this module's return.""" submodule_returns = md_struct.get_returned_opt_var_name() submodule_opt_var_name = md_struct.ms_opt_var_name for (submodule_ext_succ, _, ith_output) in submodule_returns: self.external_successor_local_returns_map[submodule_ext_succ] = ( submodule_opt_var_name, ith_output) # The following funcs are designated to be invoked by matcher. def add_inputs_edge(self, edge_name: str): construct_header_length = len(self.inputs_register.values()) default_x_str = "x" if not edge_name in self.inputs_register: self.inputs_register[edge_name] = "".join([default_x_str, str(construct_header_length-1)]) \ if construct_header_length > 0 else default_x_str def add_outputs_edge(self, edge_name: str): if edge_name in self.outputs_register: return # to be filled during code generation, should from sub's opt_var_name self.outputs_register[edge_name] = "<placeholder>" def fill_outputs_edge(self, edge_name: str, opt_var_name: str): # FILL the outputs edge once you got a opt_var_name of corresponding node!!! if not edge_name in self.outputs_register: raise ValueError(f"ModuleStruct {self.identifier} does not have edge "\ f"{edge_name} and unable to fill its output var name.") if self.outputs_register[edge_name] != "<placeholder>": raise ValueError(f"The edge has been already filled as {self.outputs_register[edge_name]}" \ f" instead of your {opt_var_name}") self.outputs_register[edge_name] = opt_var_name
class RunInfo: """A little plaintext file containing very basic info about a pants run. Can only be appended to, never edited. """ def __init__(self, info_file): self._info_file = info_file safe_mkdir_for(self._info_file) self._info = OrderedDict() if os.path.exists(self._info_file): with open(self._info_file, 'r') as infile: info = infile.read() for m in re.finditer("""^([^:]+):(.*)$""", info, re.MULTILINE): self._info[m.group(1).strip()] = m.group(2).strip() def path(self): return self._info_file def get_info(self, key): return self._info.get(key, None) def __getitem__(self, key): ret = self.get_info(key) if ret is None: raise KeyError(key) return ret def get_as_dict(self): return self._info.copy() def add_info(self, key, val, ignore_errors=False): """Adds the given info and returns a dict composed of just this added info.""" self.add_infos((key, val), ignore_errors=ignore_errors) def add_infos(self, *keyvals, **kwargs): """Adds the given info and returns a dict composed of just this added info.""" kv_pairs = [] for key, val in keyvals: key = key.strip() val = str(val).strip() if ':' in key: raise ValueError(f'info key "{key}" must not contain a colon.') kv_pairs.append((key, val)) for k, v in kv_pairs: if k in self._info: raise ValueError( f'info key "{k}" already exists with value {self._info[k]}. ' 'Cannot add it again with value {v}.') self._info[k] = v try: with open(self._info_file, 'a') as outfile: for k, v in kv_pairs: outfile.write('{}: {}\n'.format(k, v)) except IOError: if not kwargs.get('ignore_errors', False): raise def add_basic_info(self, run_id, timestamp): """Adds basic build info.""" datetime = time.strftime('%A %b %d, %Y %H:%M:%S', time.localtime(timestamp)) user = getpass.getuser() machine = socket.gethostname() buildroot = get_buildroot() # TODO: Get rid of the redundant 'path' key once everyone is off it. self.add_infos(('id', run_id), ('timestamp', timestamp), ('datetime', datetime), ('user', user), ('machine', machine), ('path', buildroot), ('buildroot', buildroot), ('version', VERSION)) def add_scm_info(self): """Adds SCM-related info.""" scm = get_scm() if not scm: return revision = scm.commit_id branch = scm.branch_name or revision self.add_infos(('revision', revision), ('branch', branch))
def main(run_dct): """ data_sources: list, i.e. [1,2] 0: dxcluster 1: WSPRNet 2: RBN loc_sources: list, i.e. ['P','Q'] P: user Provided Q: QRZ.com or HAMCALL E: Estimated using prefix """ # Get Variables from run_dct sDate = run_dct['sDate'] eDate = run_dct['eDate'] params = run_dct['params'] xkeys = run_dct['xkeys'] rgc_lim = run_dct['rgc_lim'] filter_region = run_dct['filter_region'] filter_region_kind = run_dct['filter_region_kind'] band_obj = run_dct['band_obj'] xb_size_min = run_dct['xb_size_min'] yb_size_km = run_dct['yb_size_km'] base_dir = run_dct.get('base_dir') output_dir = run_dct.get('output_dir') reprocess = run_dct.get('reprocess', False) loc_sources = run_dct.get('loc_sources') data_sources = run_dct.get('data_sources', [1, 2]) # Define path for saving NetCDF Files if output_dir is None: tmp = [] if filter_region is not None: tmp.append('{!s}'.format(filter_region)) tmp.append('{!s}'.format(filter_region_kind)) tmp.append('{:.0f}-{:.0f}km'.format(rgc_lim[0], rgc_lim[1])) tmp.append('dx{:.0f}min'.format(xb_size_min)) tmp.append('dy{:.0f}km'.format(yb_size_km)) ncs_path = os.path.join(base_dir, '_'.join(tmp)) else: ncs_path = output_dir gl.prep_output({0: ncs_path}, clear=reprocess) # Loop through dates dates = list(daterange(sDate, eDate)) if strip_time(sDate) != strip_time(eDate): dates = dates[:-1] for dt in tqdm.tqdm(dates): nc_name = dt.strftime('%Y%m%d') + '.data.nc' nc_path = os.path.join(ncs_path, nc_name) # Skip processing if file already exists if os.path.exists(nc_path + '.bz2'): continue # Load spots from CSVs load_dates = [dt, dt + pd.Timedelta('1D')] df = pd.DataFrame() for ld_inx, load_date in enumerate(load_dates): ld_str = load_date.strftime("%Y-%m-%d") dft = gl.load_spots_csv(ld_str, data_sources=data_sources, rgc_lim=rgc_lim, loc_sources=loc_sources, filter_region=filter_region, filter_region_kind=filter_region_kind) if dft is None: print('No data for {!s}'.format(ld_str)) continue for xkey in xkeys: dft[xkey] = ld_inx * 24 + dft[xkey] df = df.append(dft, ignore_index=True) # Set Up Data Storage Containers data_das = {} for xkey in xkeys: data_das[xkey] = {} for param in params: data_das[xkey][param] = [] map_das = {} for xkey in xkeys: map_das[xkey] = [] src_cnts = {} for xkey in xkeys: if xkey not in src_cnts: src_cnts[xkey] = {} for band_inx, (band_key, band) in enumerate(band_obj.band_dict.items()): if band_key not in src_cnts[xkey]: src_cnts[xkey][band_key] = {} for band_inx, (band_key, band) in enumerate(band_obj.band_dict.items()): if len(df) == 0: continue frame = df.loc[df["band"] == band.get('meters')].copy() # Create attrs diction to save with xarray DataArray attrs = OrderedDict() attrs['sTime'] = dt attrs['param'] = param attrs['filter_region'] = filter_region attrs['filter_region_kind'] = filter_region_kind attrs['band'] = band_key attrs['band_name'] = band['name'] attrs['band_fname'] = band['freq_name'] for xkey in xkeys: src_cnts[xkey][band_key] = gl.count_sources(frame) for param in params: # Compute General Data attrs['xkey'] = xkey attrs['param'] = param attrs['dx'] = xb_size_min / 60. attrs['xlim'] = (0, 24) attrs['ykey'] = 'dist_Km' attrs['ylim'] = rgc_lim attrs['dy'] = yb_size_km result = calc_histogram(frame, attrs) data_da_result = result data_das[xkey][param].append(result) # Compute Map time_bins = np.array(data_da_result.coords[xkey]) map_attrs = attrs.copy() del map_attrs['param'] map_attrs['xkey'] = 'md_long' map_attrs['xlim'] = (-180, 180) map_attrs['dx'] = 1 map_attrs['ykey'] = 'md_lat' map_attrs['ylim'] = (-90, 90) map_attrs['dy'] = 1 map_tmp = [] for tb_inx, tb_0 in enumerate(time_bins[:-1]): tb_1 = time_bins[tb_inx + 1] tf = np.logical_and(frame[xkey] >= tb_0, frame[xkey] < tb_1) tb_frame = frame[tf].copy() result = calc_histogram(tb_frame, map_attrs) result.coords[xkey] = tb_0 map_tmp.append(result) map_tmp_da = xr.concat(map_tmp, dim=xkey) map_das[xkey].append(map_tmp_da) # Continue if no data. if len(map_das[xkeys[0]]) == 0: continue # Maps - Concatenate all bands into single DataArray for xkey in xkeys: map_das[xkey] = xr.concat(map_das[xkey], dim='freq_MHz') map_dss = OrderedDict() for xkey in xkeys: map_ds = xr.Dataset() map_ds['spot_density'] = map_das[xkey] map_dss[xkey] = map_ds # Time Series - Concatenate all bands into single DataArray for xkey in xkeys: for param in params: data_da = data_das[xkey][param] = xr.concat( data_das[xkey][param], dim='freq_MHz') data_dss = OrderedDict() for xkey in xkeys: data_ds = xr.Dataset() for param in params: d_ds = data_das[xkey][param] src_cnt = src_cnts[xkey] d_ds.attrs['src_cnt'] = pd.DataFrame(src_cnt).to_json() data_ds[param] = d_ds data_dss[xkey] = data_ds # Save to data file. first = True for xkey, map_ds in map_dss.items(): if first: mode = 'w' first = False else: mode = 'a' group = 'map/{!s}'.format(xkey) map_ds.to_netcdf(nc_path, mode=mode, group=group) for xkey, data_ds in data_dss.items(): group = 'time_series/{!s}'.format(xkey) data_ds.to_netcdf(nc_path, mode='a', group=group) mbz2 = gl.MyBz2(nc_path) mbz2.compress() return ncs_path
class Graph(object): """ A graph of ``Components`` """ def __init__(self, name=None, default_capacity=10, description=None, metadata=None): self.name = name self.default_capacity = default_capacity self.metadata = metadata or {} self.description = None # type: OrderedDict[str, rill.engine.component.Component] self._components = OrderedDict() # exported inports and outports # type: OrderedDict[str, rill.engine.inputport.InputPort] self.inports = OrderedDict() self.inport_metadata = OrderedDict() # type: OrderedDict[str, rill.engine.inputport.OutputPort] self.outports = OrderedDict() self.outport_metadata = OrderedDict() def __repr__(self): return '{}(name={!r})'.format(self.__class__.__name__, self.name) def __getstate__(self): # avoid copying InputPorts and OutputPorts data = self.__dict__.copy() for k in ('inports', 'outports'): data.pop(k) inports = [] for name, port in self.inports.items(): inports.append((name, port.component.name, port._name, port.index)) data['inports'] = inports outports = [] for name, port in self.outports.items(): outports.append( (name, port.component.name, port._name, port.index)) data['outports'] = outports return data def __setstate__(self, data): # avoid copying InputPorts and OutputPorts inports = data.pop('inports') outports = data.pop('outports') self.__dict__.update(data) self.inports = OrderedDict() for name, comp_name, port_name, port_index in inports: self.inports[name] = self.get_component_port( (comp_name, port_name), index=port_index, kind='in') self.outports = OrderedDict() for name, comp_name, port_name, port_index in outports: self.outports[name] = self.get_component_port( (comp_name, port_name), index=port_index, kind='out') def copy(self, deep=True): """ Create a deep copy of the graph. Returns ------- ``Graph`` """ if deep: return copy.deepcopy(self) else: graph = Graph(self.name, self.default_capacity) graph.inports = self.inports.copy() graph.outports = self.outports.copy() graph._components = self._components.copy() return graph @supports_listeners def set_metadata(self, metadata): """ Set graph level metadata Parameters ---------- metadata: dict """ merge_metadata(metadata, self.metadata) self.set_metadata.event.emit(metadata) return self.metadata @supports_listeners def rename(self, new_name): """ Rename graph Parameters ---------- new_name: str """ old_name = self.name self.name = new_name self.rename.event.emit(old_name, new_name) return new_name # Components -- def _get_unique_name(self, basename): i = 0 name = basename while name in self._components: i += 1 name = basename + str(i) return name def add_component(self, *args, **initializations): """ Instantiate a component and add it to the network. Parameters ---------- name : str name of component comp_type : Type[``rill.engine.component.Component``] component class to instantiate Returns ------- ``rill.engine.component.Component`` """ if len(args) == 1: arg = args[0] if isinstance(arg, Component): comp = arg name = comp.name elif isclass(arg) and issubclass(arg, Component): comp_type = arg name = comp_type.type_name or comp_type.__name__ name = self._get_unique_name(name) comp = comp_type(name) else: raise ValueError() elif len(args) == 2: name, comp_type = args if not isclass(comp_type) or not issubclass(comp_type, Component): raise TypeError("comp_type must be a sub-class of Component") comp = comp_type(name) else: raise ValueError() if name in self._components: raise FlowError( "Component {} already exists in network".format(name)) self.put_component(name, comp) for name, value in initializations.items(): receiver = comp.port(name, kind='in') if value is None and not receiver.required: continue if isinstance(value, (OutputPort, OutputArray, InputPort, InputArray)): self.connect(value, receiver) else: self.initialize(value, receiver) return comp @supports_listeners def set_node_metadata(self, node, metadata): """ Set metadata on node Parameters ---------- node: ``rill.engine.component.Component`` metadata: {} """ merge_metadata(metadata, node.metadata) self.set_node_metadata.event.emit(node, metadata) return node.metadata def add_graph(self, name, graph, **initializations): """ Instantiate a component and add it to the network. Parameters ---------- name : str name of component within the graph graph : ``Graph`` graph to add Returns ------- ``rill.engine.component.Component`` """ from rill.engine.subnet import make_subgraph comp_type = make_subgraph(name, graph) return self.add_component(name, comp_type, **initializations) @supports_listeners def remove_component(self, name): """ Remove component from graph Parameters ---------- name: str """ # FIXME: this needs some love # assert not self.active component = self._components.pop(name) for inport in component.inports: if inport.is_connected() and not inport.is_initialized(): for outport in inport._connection.outports: self.disconnect(inport, outport) for outport in component.outports: if outport.is_connected(): self.disconnect(outport._connection.inport, outport) # FIXME: remove references in self.inports and self.outports self.remove_component.event.emit(component) @supports_listeners def rename_component(self, orig_name, new_name): """ Change name of component Parameters ---------- orig_name: str new_name: str """ # FIXME: this needs some love # assert not self.active assert new_name not in self._components component = self._components.pop(orig_name) self._components[new_name] = component self.rename_component.event.emit(orig_name, new_name, component) def component(self, name): """ Parameters ---------- name : str name of component Returns ------- ``rill.engine.component.Component`` Raises ------ ``rill.engine.exceptions.FlowError`` : if no component found """ comp = self.get_component(name) if comp is None: raise FlowError("Reference to unknown component " + name) return comp def get_components(self): """ Get a dictionary of components in this network Returns ------- Dict[str, ``rill.enginge.component.Component``] """ return self._components def get_component(self, name): """ Returns the requested component in this network if present or None if not present. Returns ------- ``rill.enginge.component.Component`` """ return self._components.get(name) # FIXME: remove this and run/move Component._init to __init__ ? @supports_listeners def put_component(self, name, comp): """ Adds a component and inits it. Parameters ---------- name : str comp : ``rill.enginge.component.Component`` """ # initialize the component's ports so they can be used for network # building comp._init() self._components[name] = comp self.put_component.event.emit(name, comp) # Ports -- @supports_listeners def export(self, internal_port, external_port_name, metadata=None): """ Exports component port for connecting to other networks as a sub network Parameters ---------- internal_port : ``rill.engine.outputport.OutputPort`` or ``rill.engine.inputport.InputPort`` or str external_port_name : str name of port that will be exposed """ metadata = metadata or {} internal_port = self.get_component_port(internal_port) if isinstance(internal_port, InputPort): self.inports[external_port_name] = internal_port self.inport_metadata[external_port_name] = metadata elif isinstance(internal_port, OutputPort): self.outports[external_port_name] = internal_port self.outport_metadata[external_port_name] = metadata self.export.event.emit(internal_port, external_port_name, metadata) @supports_listeners def remove_inport(self, external_port_name): """ Remove public inport from graph Parameters ---------- external_port_name: str """ del self.inports[external_port_name] del self.inport_metadata[external_port_name] self.remove_inport.event.emit(external_port_name) @supports_listeners def remove_outport(self, external_port_name): """ Remove public outport from graph Parameters ---------- external_port_name: str """ del self.outports[external_port_name] del self.outport_metadata[external_port_name] self.remove_outport.event.emit(external_port_name) @supports_listeners def set_inport_metadata(self, external_port_name, metadata): """ Set metadata on inport Parameters ---------- external_port_name: str metadata: dict """ port_metadata = self.inport_metadata[external_port_name] merge_metadata(metadata, port_metadata) self.set_inport_metadata.event.emit(external_port_name, metadata) return port_metadata @supports_listeners def set_outport_metadata(self, external_port_name, metadata): """ Set metadata on outport Parameters ---------- external_port_name: str metadata: dict """ port_metadata = self.outport_metadata[external_port_name] merge_metadata(metadata, port_metadata) self.set_outport_metadata.event.emit(external_port_name, metadata) return port_metadata # FIXME: might be better to split this into get_component_inport / get_component_outport # the main argument for the current design is if you don't know or care what # kind of port you want, but in practice, 'kind' is always provided def get_component_port(self, arg, index=None, kind=None): """ Get a port on a component. Parameters ---------- arg : Union[``rill.engine.outputport.OutputPort``, ``rill.engine.inputport.InputPort``, str] index : Optional[int] index of element, if port is an array. If None, the next available index is used kind : str {'in', 'out'} Returns ------- Union[``rill.engine.outputport.OutputPort``, ``rill.engine.inputport.InputPort``] """ if isinstance(arg, (OutputPort, OutputArray, InputPort, InputArray)): port = arg if kind is not None and port.kind != kind: raise FlowError("Expected {}port: got {}".format( kind, type(port))) else: if isinstance(arg, (tuple, list)): comp_name, port_name = arg elif isinstance(arg, basestring): split = arg.split('.') comp_name = '.'.join(split[:-1]) port_name = split[-1] else: raise TypeError(arg) comp = self.component(comp_name) port = comp.port(port_name, kind=kind) if port.is_array() and index is not False: port = port.get_element(index, create=True) return port @supports_listeners def connect(self, sender, receiver, connection_capacity=None, count_packets=False, metadata=None): """ Connect an output port of one component to an input port of another. Parameters ---------- sender : Union[``rill.engine.inputport.InputPort``, str] receiver : Union[``rill.engine.outputport.OutputPort``, str] Returns ------- ``rill.engine.inputport.InputPort`` """ outport = self.get_component_port(sender, kind='out') inport = self.get_component_port(receiver, kind='in') assert not outport.is_array() assert not inport.is_array() if connection_capacity is None: connection_capacity = self.default_capacity if inport._connection is None: inport._connection = Connection() inport._connection.connect(inport, outport, connection_capacity) metadata = metadata or {} edge_metadata = inport._connection.metadata.setdefault(outport, {}) edge_metadata.update(metadata) self.connect.event.emit(outport, inport, connection_capacity, metadata) return inport @supports_listeners def disconnect(self, sender, receiver): """ Disconnect an output port of one component from an input port of another. Parameters ---------- sender : Union[``rill.engine.inputport.InputPort``, str] receiver : Union[``rill.engine.outputport.OutputPort``, str] """ outport = self.get_component_port(sender, kind='out') inport = self.get_component_port(receiver, kind='in') outport._connection = None if outport._connections: outport._connections.remove(inport._connection) inport._connection.outports.remove(outport) if not inport._connection.outports: inport._connection = None else: inport._connection.metadata.pop(outport) self.disconnect.event.emit(outport, inport) @supports_listeners def set_edge_metadata(self, sender, receiver, metadata): """ Set metadata on edge Parameters ---------- sender : Union[``rill.engine.inputport.InputPort``, str] receiver : Union[``rill.engine.outputport.OutputPort``, str] metadata: dict """ outport = self.get_component_port(sender, kind='out') inport = self.get_component_port(receiver, kind='in') edge_metadata = inport._connection.metadata.setdefault(outport, {}) merge_metadata(metadata, edge_metadata) self.set_edge_metadata.event.emit(outport, inport, metadata) return edge_metadata # FIXME: reverse the order of these args @supports_listeners def initialize(self, content, receiver): """ Initialize an inport port with a value Parameters ---------- content : Any receiver : Union[``rill.engine.inputport.InputPort``, str] """ inport = self.get_component_port(receiver, kind='in') inport.initialize(content) self.initialize.event.emit(inport, content) @supports_listeners def uninitialize(self, receiver): """ Remove the initialized value of an inport port. Parameters ---------- receiver : Union[``rill.engine.inputport.InputPort``, str] Returns ------- ``rill.engine.inputport.InitializationConnection`` The removed initialization connection """ inport = self.get_component_port(receiver, kind='in') result = inport.uninitialize() self.uninitialize.event.emit(inport) return result def validate(self): """ Validate the graph. """ errors = [] for component in self._components.values(): for port in component.ports: try: port.validate() except FlowError as e: errors.append(str(e)) if errors: for error in errors: logger.error(error) raise FlowError("Errors opening ports") # Serialization -- def to_dict(self): """ Serialize graph to dictionary Returns ------- definition : dict json-serializable representation of graph according to fbp json standard """ from rill.events.listeners.memory import fbp_iip, fbp_edge definition = { 'processes': {}, 'connections': [], 'inports': {}, 'outports': {} } for (comp_name, component) in self.get_components().items(): if component.hidden: continue definition['processes'][comp_name] = { "component": component.get_type(), "metadata": component.metadata, } for inport in component.inports: if not inport.is_connected(): continue conn = inport._connection if isinstance(conn, InitializationConnection): definition['connections'].append( fbp_iip(inport, self.name, 'process')) else: for outport in conn.outports: if outport.component.hidden: continue definition['connections'].append( fbp_edge(outport, inport, self.name, self.default_capacity, 'process')) for (name, inport) in self.inports.items(): definition['inports'][name] = { 'process': inport.component.get_name(), 'port': inport.name, 'metadata': self.inport_metadata.get(name, {}) } for (name, outport) in self.outports.items(): definition['outports'][name] = { 'process': outport.component.get_name(), 'port': outport.name, 'metadata': self.outport_metadata.get(name, {}) } return definition @classmethod def from_dict(cls, definition, component_lookup=None): """ Create network from serialized definition Parameters ---------- definition : dict network structure according to fbp json standard component_lookup : Dict[str, Type[``rill.enginge.component.Component``]] maps of component name to component class Returns ------- ``rill.enginge.network.Graph`` """ from rill.utils import locate_class def _port(p, kind): return graph.get_component_port((p['process'], p['port']), index=p.get('index', None), kind=kind) graph = cls() for (name, spec) in definition['processes'].items(): if component_lookup: comp_class = component_lookup[spec['component']] else: comp_class = locate_class(spec['component'].replace('/', '.')) component = graph.add_component(name, comp_class) if spec.get('metadata'): component.metadata.update(spec['metadata']) for connection in definition['connections']: tgt = _port(connection['tgt'], 'in') if 'data' in connection['src']: # static initializer data = connection['src']['data'] content = deserialize(data) if not tgt.auto_receive: content = Stream(content) # content = tgt.type.to_native(content) graph.initialize(content, tgt) else: # connection src = _port(connection['src'], 'out') if 'cap' in connection.keys(): graph.connect(src, tgt, connection_capacity=connection['cap']) else: graph.connect(src, tgt) for (name, inport) in definition['inports'].items(): graph.export(_port(inport, 'in'), name) for (name, outport) in definition['outports'].items(): graph.export(_port(outport, 'out'), name) return graph
def build_update(self): """ Draw sample from proposal conditional on last sample. Then accept or reject the sample based on the ratio, .. math:: \\text{ratio} = \log p(x, z^{new}) - \log p(x, z^{old}) + \log g(z^{new} \mid z^{old}) - \log g(z^{old} \mid z^{new}) Notes ----- The updates assume each Empirical random variable is directly parameterized by tf.Variables(). """ old_sample = { z: tf.gather(qz.params, tf.maximum(self.t - 1, 0)) for z, qz in six.iteritems(self.latent_vars) } old_sample = OrderedDict(old_sample) # Form dictionary in order to replace conditioning on prior or # observed variable with conditioning on a specific value. dict_swap = {} for x, qx in six.iteritems(self.data): if isinstance(x, RandomVariable): if isinstance(qx, RandomVariable): qx_copy = copy(qx, scope='conditional') dict_swap[x] = qx_copy.value() else: dict_swap[x] = qx dict_swap_old = dict_swap.copy() dict_swap_old.update(old_sample) scope_old = 'inference_' + str(id(self)) + '/old' scope_new = 'inference_' + str(id(self)) + '/new' # Draw proposed sample and calculate acceptance ratio. new_sample = old_sample.copy() # copy to ensure same order ratio = 0.0 for z, proposal_z in six.iteritems(self.proposal_vars): # Build proposal g(znew | zold). proposal_znew = copy(proposal_z, dict_swap_old, scope=scope_old) # Sample znew ~ g(znew | zold). new_sample[z] = proposal_znew.value() # Increment ratio. ratio += tf.reduce_sum(proposal_znew.log_prob(new_sample[z])) dict_swap_new = dict_swap.copy() dict_swap_new.update(new_sample) for z, proposal_z in six.iteritems(self.proposal_vars): # Build proposal g(zold | znew). proposal_zold = copy(proposal_z, dict_swap_new, scope=scope_new) # Increment ratio. ratio -= tf.reduce_sum(proposal_zold.log_prob(dict_swap_old[z])) if self.model_wrapper is None: for z in six.iterkeys(self.latent_vars): # Build priors p(znew) and p(zold). znew = copy(z, dict_swap_new, scope=scope_new) zold = copy(z, dict_swap_old, scope=scope_old) # Increment ratio. ratio += tf.reduce_sum(znew.log_prob(dict_swap_new[z])) ratio -= tf.reduce_sum(zold.log_prob(dict_swap_old[z])) for x in six.iterkeys(self.data): if isinstance(x, RandomVariable): # Build likelihoods p(x | znew) and p(x | zold). x_znew = copy(x, dict_swap_new, scope=scope_new) x_zold = copy(x, dict_swap_old, scope=scope_old) # Increment ratio. ratio += tf.reduce_sum(x_znew.log_prob(dict_swap[x])) ratio -= tf.reduce_sum(x_zold.log_prob(dict_swap[x])) else: x = self.data ratio += self.model_wrapper.log_prob(x, new_sample) ratio -= self.model_wrapper.log_prob(x, old_sample) # Accept or reject sample. u = Uniform().sample() accept = tf.log(u) < ratio sample_values = tf.cond(accept, lambda: list(six.itervalues(new_sample)), lambda: list(six.itervalues(old_sample))) if not isinstance(sample_values, list): # ``tf.cond`` returns tf.Tensor if output is a list of size 1. sample_values = [sample_values] sample = { z: sample_value for z, sample_value in zip(six.iterkeys(new_sample), sample_values) } # Update Empirical random variables. assign_ops = [] for z, qz in six.iteritems(self.latent_vars): variable = qz.get_variables()[0] assign_ops.append(tf.scatter_update(variable, self.t, sample[z])) # Increment n_accept (if accepted). assign_ops.append(self.n_accept.assign_add(tf.where(accept, 1, 0))) return tf.group(*assign_ops)
class CommandInfo: """One command which is provided by one bundy module, it has zero or more parameters """ def __init__(self, name, desc=""): self.name = name self.desc = desc self.params = OrderedDict() # Set default parameter "help" self.add_param( ParamInfo("help", desc="Get help for command.", optional=True)) def __str__(self): return str("%s \t(%s)" % (self.name, self.desc)) def get_name(self): return self.name def get_desc(self): return self.desc def add_param(self, paraminfo): """Add a ParamInfo object to this CommandInfo""" self.params[paraminfo.name] = paraminfo def has_param_with_name(self, param_name): """Returns true if the parameter with param_name exists""" return param_name in self.params def get_param_with_name(self, param_name): """Returns the ParamInfo with the given name. Raises a KeyError if it doesn't exist""" return self.params[param_name] def get_params(self): """Returns a list of all ParamInfo objects for this CommandInfo""" return list(self.params.values()) def get_param_names(self): """Returns a list of the names of all parameters for this command""" return list(self.params.keys()) def get_mandatory_param_names(self): """Returns a list of the names of all mandatory parameters for this command""" all_names = self.params.keys() return [ name for name in all_names if not self.params[name].is_optional ] def get_param_name_by_position(self, pos, param_count): """ Find a proper parameter name for the position 'pos': If param_count is equal to the count of mandatory parameters of command, and there is some optional parameter, find the first mandatory parameter from the position 'pos' to the end. Else, return the name on position pos. (This function will be changed if bundyctl command line syntax is changed in the future. ) """ if type(pos) != int: raise KeyError(str(pos) + " is not an integer") else: params = self.params.copy() del params['help'] count = len(params) if (pos >= count): raise KeyError(str(pos) + " out of range") mandatory_count = len(self.get_mandatory_param_names()) param_names = list(params.keys()) if (param_count == mandatory_count) and (param_count < count): while pos < count: if not params[param_names[pos]].is_optional: return param_names[pos] pos += 1 raise KeyError(str(pos) + "parameters have error") else: return param_names[pos] def command_help(self): """Prints the help info for this command to stdout""" print("Command ", self) print("\t\thelp (Get help for command)") params = self.params.copy() del params["help"] if len(params) == 0: print("This command has no parameters") return print("Parameters:") for info in params.values(): print(" %s" % info.get_basic_info()) description = info.get_desc() if description != "": print( textwrap.fill(description, initial_indent=" ", subsequent_indent=" ", width=70))
class SGANNode(object): """ docstring """ def __init__(self): model_path = rospy.get_param("model_path") model_name = rospy.get_param("model_name") model = os.path.join(model_path, model_name + "_model.pt") checkpoint = torch.load(model) self.num_samples = rospy.get_param("num_samples", 20) self.seq_len = rospy.get_param("seq_len", 8) self.visualise = rospy.get_param("visualise", True) self.args_ = AttrDict(checkpoint['args']) self.generator = self.get_generator(checkpoint) self.tracked_persons_sub = rospy.Subscriber( "/spencer/perception/tracked_persons", TrackedPersons, self.tracked_persons_cb, queue_size=3) # self.predicted_tracks_pub = rospy.Publisher("/sgan/predictions", TBD, queue_size=1) self.predictions_marker_pub = rospy.Publisher("/sgan/predictions_marker", MarkerArray, queue_size=1) self.tracked_persons = {} def get_generator(self, checkpoint): generator = TrajectoryGenerator( obs_len=self.args_.obs_len, pred_len=self.args_.pred_len, embedding_dim=self.args_.embedding_dim, encoder_h_dim=self.args_.encoder_h_dim_g, decoder_h_dim=self.args_.decoder_h_dim_g, mlp_dim=self.args_.mlp_dim, num_layers=self.args_.num_layers, noise_dim=self.args_.noise_dim, noise_type=self.args_.noise_type, noise_mix_type=self.args_.noise_mix_type, pooling_type=self.args_.pooling_type, pool_every_timestep=self.args_.pool_every_timestep, dropout=self.args_.dropout, bottleneck_dim=self.args_.bottleneck_dim, neighborhood_size=self.args_.neighborhood_size, grid_size=self.args_.grid_size, batch_norm=self.args_.batch_norm, use_gpu=self.args_.use_gpu) generator.load_state_dict(checkpoint['g_state']) if self.args_.use_gpu: generator.cuda() generator.train() return generator def tracked_persons_cb(self, msg): """ docstring """ for track in msg.tracks: if track.twist.twist.linear.x > 0 or track.twist.twist.linear.y > 0: if track.track_id in self.tracked_persons: self.tracked_persons[track.track_id].append(track) else: self.tracked_persons[track.track_id] = deque([track], maxlen=8) current_ids = [track.track_id for track in msg.tracks] msg_arr = MarkerArray() for k in self.tracked_persons.copy().keys(): if k not in current_ids: if self.visualise: msg = Marker() msg.action = msg.DELETEALL msg.ns = "track_" + str(k) msg_arr.markers.append(copy(msg)) del self.tracked_persons[k] if self.visualise: self.predictions_marker_pub.publish(msg_arr) if type(self.tracked_persons) is dict: self.tracked_persons = OrderedDict( sorted(self.tracked_persons.items(), key=lambda t: t[0])) def predict_tracks(self): """ docstring """ curr_seq = np.zeros((self.seq_len, len(self.tracked_persons), 2)) curr_seq_rel = np.zeros(curr_seq.shape) valid_tracks = 0 with torch.no_grad(): # copy is needed, because the subscriber callback can mutate the dict for id, tracks in self.tracked_persons.copy().items(): if len(tracks) != self.seq_len: continue obs_traj = np.array([[track.pose.pose.position.x, track.pose.pose.position.y] for track in tracks]) obs_traj_rel = np.zeros(obs_traj.shape) obs_traj_rel[1:, :] = obs_traj[1:, :] - obs_traj[:-1, :] curr_seq[:, valid_tracks, :] = obs_traj curr_seq_rel[:, valid_tracks, :] = obs_traj_rel valid_tracks = valid_tracks + 1 if valid_tracks > 0: curr_seq = torch.from_numpy(curr_seq[:, :valid_tracks, :]).float() curr_seq_rel = torch.from_numpy(curr_seq_rel[:, :valid_tracks, :]).float() if self.args_.use_gpu: curr_seq = curr_seq.cuda() curr_seq_rel = curr_seq_rel.cuda() l = [valid_tracks] cum_start_idx = [0] + np.cumsum(l).tolist() seq_start_end = [ (start, end) for start, end in zip(cum_start_idx, cum_start_idx[1:]) ] seq_start_end = torch.Tensor(seq_start_end).type(torch.int) if self.args_.use_gpu: seq_start_end = seq_start_end.cuda() pred_samples = dict.fromkeys(self.tracked_persons.keys(), []) track_ids = list(self.tracked_persons.keys()) for _ in range(self.num_samples): pred_traj_fake_rel = self.generator(curr_seq, curr_seq_rel, seq_start_end) pred_traj_fake = relative_to_abs(pred_traj_fake_rel, curr_seq[-1]) for ped_id, pred in enumerate(pred_traj_fake.transpose(1, 0).tolist()): pred_samples[track_ids[ped_id]].extend( [Point(pred[idx][0], pred[idx][1], 0) for idx in range(0, self.seq_len)]) if self.visualise: self.visualise_predictions(pred_samples) def visualise_predictions(self, predictions_samples): msg = Marker() msg.header.frame_id = "odom" msg.ns = "tracks" msg.id = 0 msg.action = msg.ADD msg.pose.orientation.w = 1.0 msg.type = msg.LINE_LIST msg.points = [] msg.scale.x = 0.05 msg.color.a = 1. msg.color.r = 1. msg.color.g = 0. msg.color.b = 0. msg_arr = MarkerArray() for k, v in predictions_samples.items(): msg.points = [] msg.points.extend(v) msg.ns = "track_" + str(k) msg_arr.markers.append(copy(msg)) self.predictions_marker_pub.publish(msg_arr) def spin(self): """ docstring """ rate = rospy.Rate(5) while not rospy.is_shutdown(): if self.tracked_persons: self.predict_tracks() rate.sleep()
class Workspace: def __init__(self, workspace_load_target): from .cli_target import WorkspaceLoadTarget self._stack = ExitStack() self._workspace_load_target = check.opt_inst_param( workspace_load_target, "workspace_load_target", WorkspaceLoadTarget ) self._grpc_server_registry = self._stack.enter_context( ProcessGrpcServerRegistry(reload_interval=0, heartbeat_ttl=30) ) self._load_workspace() def _load_workspace(self): repository_location_origins = ( self._workspace_load_target.create_origins() if self._workspace_load_target else [] ) self._location_origin_dict = OrderedDict() check.list_param( repository_location_origins, "repository_location_origins", of_type=RepositoryLocationOrigin, ) self._location_dict = {} self._location_error_dict = {} for origin in repository_location_origins: check.invariant( self._location_origin_dict.get(origin.location_name) is None, 'Cannot have multiple locations with the same name, got multiple "{name}"'.format( name=origin.location_name, ), ) self._location_origin_dict[origin.location_name] = origin self._load_location(origin.location_name) # Can be overidden in subclasses that need different logic for loading repository # locations from origins def create_location_from_origin(self, origin): if not self._grpc_server_registry.supports_origin(origin): # pylint: disable=no-member return origin.create_location() else: endpoint = self._grpc_server_registry.reload_grpc_endpoint( # pylint: disable=no-member origin ) return GrpcServerRepositoryLocation( origin=origin, server_id=endpoint.server_id, port=endpoint.port, socket=endpoint.socket, host=endpoint.host, heartbeat=True, watch_server=False, grpc_server_registry=self._grpc_server_registry, ) def _load_location(self, location_name): if self._location_dict.get(location_name): del self._location_dict[location_name] if self._location_error_dict.get(location_name): del self._location_error_dict[location_name] origin = self._location_origin_dict[location_name] try: location = self.create_location_from_origin(origin) self._location_dict[location_name] = location except Exception: # pylint: disable=broad-except error_info = serializable_error_info_from_exc_info(sys.exc_info()) self._location_error_dict[location_name] = error_info warnings.warn( "Error loading repository location {location_name}:{error_string}".format( location_name=location_name, error_string=error_info.to_string() ) ) def create_snapshot(self): return WorkspaceSnapshot( self._location_origin_dict.copy(), self._location_error_dict.copy() ) @property def repository_locations(self): return list(self._location_dict.values()) def has_repository_location(self, location_name): check.str_param(location_name, "location_name") return location_name in self._location_dict def get_repository_location(self, location_name): check.str_param(location_name, "location_name") return self._location_dict[location_name] def has_repository_location_error(self, location_name): check.str_param(location_name, "location_name") return location_name in self._location_error_dict def reload_repository_location(self, location_name): self._load_location(location_name) def reload_workspace(self): for location in self.repository_locations: location.cleanup() self._load_workspace() def __enter__(self): return self def __exit__(self, exception_type, exception_value, traceback): for location in self.repository_locations: location.cleanup() self._stack.close()
def _eqcalculate(dbf, comps, phases, conditions, output, data=None, per_phase=False, callables=None, parameters=None, **kwargs): """ WARNING: API/calling convention not finalized. Compute the *equilibrium value* of a property. This function differs from `calculate` in that it computes thermodynamic equilibrium instead of randomly sampling the internal degrees of freedom of a phase. Because of that, it's slower than `calculate`. This plugs in the equilibrium phase and site fractions to compute a thermodynamic property defined in a Model. Parameters ---------- dbf : Database Thermodynamic database containing the relevant parameters. comps : list Names of components to consider in the calculation. phases : list or dict Names of phases to consider in the calculation. conditions : dict or (list of dict) StateVariables and their corresponding value. output : str Equilibrium model property (e.g., CPM, HM, etc.) to compute. This must be defined as an attribute in the Model class of each phase. data : Dataset, optional Previous result of call to `equilibrium`. Should contain the equilibrium configurations at the conditions of interest. If the databases are not the same as in the original calculation, the results may be meaningless. If None, `equilibrium` will be called. Specifying this keyword argument can save the user some time if several properties need to be calculated in succession. per_phase : bool, optional If True, compute and return the property for each phase present. If False, return the total system value, weighted by the phase fractions. parameters : dict, optional Maps SymPy Symbol to numbers, for overriding the values of parameters in the Database. callables : dict Callable functions to compute 'output' for each phase. kwargs Passed to `calculate`. Returns ------- Dataset of property as a function of equilibrium conditions """ if data is None: data = equilibrium(dbf, comps, phases, conditions) active_phases = unpack_phases(phases) or sorted(dbf.phases.keys()) conds = _adjust_conditions(conditions) indep_vars = ['N', 'P', 'T'] # TODO: Rewrite this to use the coord dict from 'data' str_conds = OrderedDict((str(key), value) for key, value in conds.items()) indep_vals = list([float(x) for x in np.atleast_1d(val)] for key, val in str_conds.items() if key in indep_vars) coord_dict = str_conds.copy() components = [x for x in sorted(comps)] desired_active_pure_elements = [ list(x.constituents.keys()) for x in components ] desired_active_pure_elements = [ el.upper() for constituents in desired_active_pure_elements for el in constituents ] pure_elements = sorted( set([x for x in desired_active_pure_elements if x != 'VA'])) coord_dict['vertex'] = np.arange( len(pure_elements) + 1 ) # +1 is to accommodate the degenerate degree of freedom at the invariant reactions grid_shape = np.meshgrid(*coord_dict.values(), indexing='ij', sparse=False)[0].shape prop_shape = grid_shape prop_dims = list(str_conds.keys()) + ['vertex'] result = Dataset({output: (prop_dims, np.full(prop_shape, np.nan))}, coords=coord_dict) # For each phase select all conditions where that phase exists # Perform the appropriate calculation and then write the result back for phase in active_phases: dof = sum([len(x) for x in dbf.phases[phase].constituents]) current_phase_indices = (data.Phase.values == phase) if ~np.any(current_phase_indices): continue points = data.Y.values[np.nonzero(current_phase_indices)][..., :dof] statevar_indices = np.nonzero(current_phase_indices)[:len(indep_vals)] statevars = { key: np.take(np.asarray(vals), idx) for key, vals, idx in zip(indep_vars, indep_vals, statevar_indices) } statevars.update(kwargs) if statevars.get('mode', None) is None: statevars['mode'] = 'numpy' calcres = calculate(dbf, comps, [phase], output=output, points=points, broadcast=False, callables=callables, parameters=parameters, **statevars) result[output].values[np.nonzero( current_phase_indices)] = calcres[output].values if not per_phase: result[output] = (result[output] * data['NP']).sum(dim='vertex', skipna=True) else: result['Phase'] = data['Phase'].copy() result['NP'] = data['NP'].copy() return result
def equilibrium(dbf, comps, phases, conditions, output=None, model=None, verbose=False, broadcast=True, calc_opts=None, scheduler='sync', parameters=None, solver=None, callables=None, **kwargs): """ Calculate the equilibrium state of a system containing the specified components and phases, under the specified conditions. Parameters ---------- dbf : Database Thermodynamic database containing the relevant parameters. comps : list Names of components to consider in the calculation. phases : list or dict Names of phases to consider in the calculation. conditions : dict or (list of dict) StateVariables and their corresponding value. output : str or list of str, optional Additional equilibrium model properties (e.g., CPM, HM, etc.) to compute. These must be defined as attributes in the Model class of each phase. model : Model, a dict of phase names to Model, or a seq of both, optional Model class to use for each phase. verbose : bool, optional Print details of calculations. Useful for debugging. broadcast : bool If True, broadcast conditions against each other. This will compute all combinations. If False, each condition should be an equal-length list (or single-valued). Disabling broadcasting is useful for calculating equilibrium at selected conditions, when those conditions don't comprise a grid. calc_opts : dict, optional Keyword arguments to pass to `calculate`, the energy/property calculation routine. scheduler : Dask scheduler, optional Job scheduler for performing the computation. If None, return a Dask graph of the computation instead of actually doing it. parameters : dict, optional Maps SymPy Symbol to numbers, for overriding the values of parameters in the Database. solver : pycalphad.core.solver.SolverBase Instance of a solver that is used to calculate local equilibria. Defaults to a pycalphad.core.solver.InteriorPointSolver. callables : dict, optional Pre-computed callable functions for equilibrium calculation. Returns ------- Structured equilibrium calculation, or Dask graph if scheduler=None. Examples -------- None yet. """ if not broadcast: raise NotImplementedError('Broadcasting cannot yet be disabled') comps = sorted(unpack_components(dbf, comps)) phases = unpack_phases(phases) or sorted(dbf.phases.keys()) # remove phases that cannot be active list_of_possible_phases = filter_phases(dbf, comps) active_phases = sorted( set(list_of_possible_phases).intersection(set(phases))) if len(list_of_possible_phases) == 0: raise ConditionError( 'There are no phases in the Database that can be active with components {0}' .format(comps)) if len(active_phases) == 0: raise ConditionError( 'None of the passed phases ({0}) are active. List of possible phases: {1}.' .format(phases, list_of_possible_phases)) if isinstance(comps, (str, v.Species)): comps = [comps] if len(set(comps) - set(dbf.species)) > 0: raise EquilibriumError('Components not found in database: {}'.format( ','.join([c.name for c in (set(comps) - set(dbf.species))]))) calc_opts = calc_opts if calc_opts is not None else dict() solver = solver if solver is not None else InteriorPointSolver( verbose=verbose) parameters = parameters if parameters is not None else dict() if isinstance(parameters, dict): parameters = OrderedDict(sorted(parameters.items(), key=str)) models = instantiate_models(dbf, comps, active_phases, model=model, parameters=parameters) # Temporary solution until constraint system improves if conditions.get(v.N) is None: conditions[v.N] = 1 if np.any(np.array(conditions[v.N]) != 1): raise ConditionError('N!=1 is not yet supported, got N={}'.format( conditions[v.N])) # Modify conditions values to be within numerical limits, e.g., X(AL)=0 # Also wrap single-valued conditions with lists conds = _adjust_conditions(conditions) for cond in conds.keys(): if isinstance(cond, (v.Composition, v.ChemicalPotential)) and cond.species not in comps: raise ConditionError( '{} refers to non-existent component'.format(cond)) state_variables = sorted(get_state_variables(models=models, conds=conds), key=str) str_conds = OrderedDict((str(key), value) for key, value in conds.items()) num_calcs = np.prod([len(i) for i in str_conds.values()]) components = [x for x in sorted(comps)] desired_active_pure_elements = [ list(x.constituents.keys()) for x in components ] desired_active_pure_elements = [ el.upper() for constituents in desired_active_pure_elements for el in constituents ] pure_elements = sorted( set([x for x in desired_active_pure_elements if x != 'VA'])) if verbose: print('Components:', ' '.join([str(x) for x in comps])) print('Phases:', end=' ') output = output if output is not None else 'GM' output = output if isinstance(output, (list, tuple, set)) else [output] output = set(output) output |= {'GM'} output = sorted(output) need_hessians = any( type(c) in v.CONDITIONS_REQUIRING_HESSIANS for c in conds.keys()) phase_records = build_phase_records(dbf, comps, active_phases, conds, models, output='GM', callables=callables, parameters=parameters, verbose=verbose, build_gradients=True, build_hessians=need_hessians) if verbose: print('[done]', end='\n') # 'calculate' accepts conditions through its keyword arguments grid_opts = calc_opts.copy() statevar_strings = [str(x) for x in state_variables] grid_opts.update({ key: value for key, value in str_conds.items() if key in statevar_strings }) if 'pdens' not in grid_opts: grid_opts['pdens'] = 500 grid = delayed(calculate, pure=False)(dbf, comps, active_phases, model=models, fake_points=True, callables=callables, output='GM', parameters=parameters, **grid_opts) coord_dict = str_conds.copy() coord_dict['vertex'] = np.arange( len(pure_elements) + 1 ) # +1 is to accommodate the degenerate degree of freedom at the invariant reactions coord_dict['component'] = pure_elements grid_shape = tuple(len(x) for x in conds.values()) + (len(pure_elements) + 1, ) properties = delayed(starting_point, pure=False)(conds, state_variables, phase_records, grid) conditions_per_chunk_per_axis = 2 if num_calcs > 1: # Generate slices of 'properties' slices = [] for val in grid_shape[:-1]: idx_arr = list(range(val)) num_chunks = int(np.floor(val / conditions_per_chunk_per_axis)) if num_chunks > 0: cond_slices = [ x for x in np.array_split(np.asarray(idx_arr), num_chunks) if len(x) > 0 ] else: cond_slices = [idx_arr] slices.append(cond_slices) chunk_dims = [len(slc) for slc in slices] chunk_grid = np.array( np.unravel_index(np.arange(np.prod(chunk_dims)), chunk_dims)).T res = [] for chunk in chunk_grid: prop_slice = properties[OrderedDict( list( zip(str_conds.keys(), [ np.atleast_1d(sl)[ch] for ch, sl in zip(chunk, slices) ])))] job = delayed(_solve_eq_at_conditions, pure=False)(comps, prop_slice, phase_records, grid, list(str_conds.keys()), state_variables, verbose, solver=solver) res.append(job) properties = delayed(_merge_property_slices, pure=False)(properties, chunk_grid, slices, list(str_conds.keys()), res) else: # Single-process job; don't create child processes properties = delayed(_solve_eq_at_conditions, pure=False)(comps, properties, phase_records, grid, list(str_conds.keys()), state_variables, verbose, solver=solver) # Compute equilibrium values of any additional user-specified properties # We already computed these properties so don't recompute them output = sorted(set(output) - {'GM', 'MU'}) for out in output: if (out is None) or (len(out) == 0): continue # TODO: How do we know if a specified property should be per_phase or not? # For now, we make a best guess if (out == 'degree_of_ordering') or (out == 'DOO'): per_phase = True else: per_phase = False eqcal = delayed(_eqcalculate, pure=False)(dbf, comps, active_phases, conditions, out, data=properties, per_phase=per_phase, callables=callables, parameters=parameters, model=models, **calc_opts) properties = delayed(properties.merge, pure=False)(eqcal, compat='equals') if scheduler is not None: properties = dask.compute(properties, scheduler=scheduler)[0] properties.attrs['created'] = datetime.utcnow().isoformat() if len(kwargs) > 0: warnings.warn( 'The following equilibrium keyword arguments were passed, but unused:\n{}' .format(kwargs)) return properties
class NamedObjectMap(object): def __init__(self, type): # @ReservedAssignment super(NamedObjectMap, self).__init__() self.type = type self._data = OrderedDict() if not issubclass(type, NamedObject): raise TypeError("type must be a NamedObject") def clear(self): self._data = OrderedDict() def copy(self): copy = NamedObjectMap(self.type) copy._data = self._data.copy() return copy def __len__(self): return len(self._data) def __getitem__(self, key): if isinstance(key, six.integer_types): for i, (_, v) in enumerate(six.iteritems(self._data)): if i == key: return v raise KeyError(key) else: return self._data[key] def __setitem__(self, key, value): if not isinstance(value, self.type): raise TypeError("can only add " + self.type.__name__ + " objects") if isinstance(key, six.integer_types): self._data = OrderedDict([(value.name, value) if i == key else (k, v) for i, (k, v) in enumerate(six.iteritems(self._data))]) else: if value.name != key: raise ValueError( "key does not match name of " + self.type.__name__) self._data[key] = value def __delitem__(self, key): del self._data[key] def __iter__(self): return iter(self._data.values()) def __contains__(self, item): return item in self._data def extend(self, value): if isinstance(value, NamedObject): self._data[value.name] = value elif hasattr(value, "__iter__"): for item in value: self._data[item.name] = item else: raise TypeError("can only add " + str(type) + " objects") def __add__(self, value): self.extend(value) return self def __iadd__(self, value): self.extend(value) return self def keys(self): return self._data.keys()
class Query: """ Helper to conform OData filters """ _mapping = { 'from': 'from/emailAddress/address', 'to': 'toRecipients/emailAddress/address', 'start': 'start/DateTime', 'end': 'end/DateTime', 'flag': 'flag/flagStatus' } def __init__(self, attribute=None, *, protocol): """ Build a query to apply OData filters https://docs.microsoft.com/en-us/graph/query-parameters :param str attribute: attribute to apply the query for :param Protocol protocol: protocol to use for connecting """ self.protocol = protocol() if isinstance(protocol, type) else protocol self._attribute = None self._chain = None self.new(attribute) self._negation = False self._filters = [] # store all the filters self._order_by = OrderedDict() self._selects = set() self._expands = set() self._search = None def __str__(self): return 'Filter: {}\nOrder: {}\nSelect: {}\nExpand: {}\nSearch: {}'.format( self.get_filters(), self.get_order(), self.get_selects(), self.get_expands(), self._search) def __repr__(self): return self.__str__() @fluent def select(self, *attributes): """ Adds the attribute to the $select parameter :param str attributes: the attributes tuple to select. If empty, the on_attribute previously set is added. :rtype: Query """ if attributes: for attribute in attributes: attribute = self.protocol.convert_case( attribute) if attribute and isinstance(attribute, str) else None if attribute: if '/' in attribute: # only parent attribute can be selected attribute = attribute.split('/')[0] self._selects.add(attribute) else: if self._attribute: self._selects.add(self._attribute) return self @fluent def expand(self, *relationships): """ Adds the relationships (e.g. "event" or "attachments") that should be expanded with the $expand parameter Important: The ApiComponent using this should know how to handle this relationships. eg: Message knows how to handle attachments, and event (if it's an EventMessage). Important: When using expand on multi-value relationships a max of 20 items will be returned. :param str relationships: the relationships tuple to expand. :rtype: Query """ for relationship in relationships: if relationship == 'event': relationship = '{}/event'.format( self.protocol.get_service_keyword('event_message_type')) self._expands.add(relationship) return self @fluent def search(self, text): """ Perform a search. Not from graph docs: You can currently search only message and person collections. A $search request returns up to 250 results. You cannot use $filter or $orderby in a search request. :param str text: the text to search :return: the Query instance """ if text is None: self._search = None else: # filters an order are not allowed self.clear_filters() self.clear_order() self._search = '"{}"'.format(text) return self def as_params(self): """ Returns the filters, orders, select, expands and search as query parameters :rtype: dict """ params = {} if self.has_filters: params['$filter'] = self.get_filters() if self.has_order: params['$orderby'] = self.get_order() if self.has_selects: params['$select'] = self.get_selects() if self.has_expands: params['$expand'] = self.get_expands() if self._search: params['$search'] = self._search params.pop('$filter', None) params.pop('$orderby', None) return params @property def has_filters(self): """ Whether the query has filters or not :rtype: bool """ return bool(self._filters) @property def has_order(self): """ Whether the query has order_by or not :rtype: bool """ return bool(self._order_by) @property def has_selects(self): """ Whether the query has select filters or not :rtype: bool """ return bool(self._selects) @property def has_expands(self): """ Whether the query has relationships that should be expanded or not :rtype: bool """ return bool(self._expands) def get_filters(self): """ Returns the result filters :rtype: str or None """ if self._filters: filters_list = self._filters if isinstance(filters_list[-1], Enum): filters_list = filters_list[:-1] return ' '.join([ fs.value if isinstance(fs, Enum) else fs[1] for fs in filters_list ]).strip() else: return None def get_order(self): """ Returns the result order by clauses :rtype: str or None """ # first get the filtered attributes in order as they must appear # in the order_by first if not self.has_order: return None filter_order_clauses = OrderedDict([(filter_attr[0], None) for filter_attr in self._filters if isinstance(filter_attr, tuple)]) # any order_by attribute that appears in the filters is ignored order_by_dict = self._order_by.copy() for filter_oc in filter_order_clauses.keys(): direction = order_by_dict.pop(filter_oc, None) filter_order_clauses[filter_oc] = direction filter_order_clauses.update( order_by_dict) # append any remaining order_by clause if filter_order_clauses: return ','.join([ '{} {}'.format(attribute, direction if direction else '').strip() for attribute, direction in filter_order_clauses.items() ]) else: return None def get_selects(self): """ Returns the result select clause :rtype: str or None """ if self._selects: return ','.join(self._selects) else: return None def get_expands(self): """ Returns the result expand clause :rtype: str or None """ if self._expands: return ','.join(self._expands) else: return None def _get_mapping(self, attribute): if attribute: mapping = self._mapping.get(attribute) if mapping: attribute = '/'.join([ self.protocol.convert_case(step) for step in mapping.split('/') ]) else: attribute = self.protocol.convert_case(attribute) return attribute return None @fluent def new(self, attribute, operation=ChainOperator.AND): """ Combine with a new query :param str attribute: attribute of new query :param ChainOperator operation: operation to combine to new query :rtype: Query """ if isinstance(operation, str): operation = ChainOperator(operation) self._chain = operation self._attribute = self._get_mapping(attribute) if attribute else None self._negation = False return self def clear_filters(self): """ Clear filters """ self._filters = [] def clear_order(self): """ Clears any order commands """ self._order_by = OrderedDict() @fluent def clear(self): """ Clear everything :rtype: Query """ self._filters = [] self._order_by = OrderedDict() self._selects = set() self._negation = False self._attribute = None self._chain = None self._search = None return self @fluent def negate(self): """ Apply a not operator :rtype: Query """ self._negation = not self._negation return self @fluent def chain(self, operation=ChainOperator.AND): """ Start a chain operation :param ChainOperator operation: how to combine with a new one :rtype: Query """ if isinstance(operation, str): operation = ChainOperator(operation) self._chain = operation return self @fluent def on_attribute(self, attribute): """ Apply query on attribute, to be used along with chain() :param str attribute: attribute name :rtype: Query """ self._attribute = self._get_mapping(attribute) return self def remove_filter(self, filter_attr): """ Removes a filter given the attribute name """ filter_attr = self._get_mapping(filter_attr) new_filters = [] remove_chain = False for flt in self._filters: if isinstance(flt, tuple): if flt[0] == filter_attr: remove_chain = True else: new_filters.append(flt) else: # this is a ChainOperator if remove_chain is False: new_filters.append(flt) else: remove_chain = False self._filters = new_filters def _add_filter(self, *filter_data): if self._attribute: if self._filters and not isinstance(self._filters[-1], ChainOperator): self._filters.append(self._chain) self._filters.append( (self._attribute, filter_data[0], filter_data[1])) else: raise ValueError( 'Attribute property needed. call on_attribute(attribute) ' 'or new(attribute)') def _parse_filter_word(self, word): """ Converts the word parameter into the correct format """ if isinstance(word, str): word = "'{}'".format(word) elif isinstance(word, dt.date): if isinstance(word, dt.datetime): if word.tzinfo is None: # if it's a naive datetime, localize the datetime. word = self.protocol.timezone.localize( word) # localize datetime into local tz if word.tzinfo != pytz.utc: word = word.astimezone( pytz.utc) # transform local datetime to utc if '/' in self._attribute: # TODO: this is a fix for the case when the parameter # filtered is a string instead a dateTimeOffset # but checking the '/' is not correct, but it will # differentiate for now the case on events: # start/dateTime (date is a string here) from # the case on other dates such as # receivedDateTime (date is a dateTimeOffset) word = "'{}'".format( word.isoformat()) # convert datetime to isoformat. else: word = "{}".format( word.isoformat()) # convert datetime to isoformat elif isinstance(word, bool): word = str(word).lower() return word @staticmethod def _prepare_sentence(attribute, operation, word, negation=False): negation = 'not' if negation else '' attrs = (negation, attribute, operation, word) return '{} {} {} {}'.format(negation, attribute, operation, word).strip(), attrs @fluent def logical_operator(self, operation, word): """ Apply a logical operator :param str operation: how to combine with a new one :param word: other parameter for the operation (a = b) would be like a.logical_operator('eq', 'b') :rtype: Query """ word = self._parse_filter_word(word) self._add_filter(*self._prepare_sentence(self._attribute, operation, word, self._negation)) return self @fluent def equals(self, word): """ Add a equals check :param str word: word to compare with :rtype: Query """ return self.logical_operator('eq', word) @fluent def unequal(self, word): """ Add a unequals check :param str word: word to compare with :rtype: Query """ return self.logical_operator('ne', word) @fluent def greater(self, word): """ Add a greater than check :param str word: word to compare with :rtype: Query """ return self.logical_operator('gt', word) @fluent def greater_equal(self, word): """ Add a greater than or equal to check :param str word: word to compare with :rtype: Query """ return self.logical_operator('ge', word) @fluent def less(self, word): """ Add a less than check :param str word: word to compare with :rtype: Query """ return self.logical_operator('lt', word) @fluent def less_equal(self, word): """ Add a less than or equal to check :param str word: word to compare with :rtype: Query """ return self.logical_operator('le', word) @staticmethod def _prepare_function(function_name, attribute, word, negation=False): negation = 'not' if negation else '' attrs = (negation, attribute, function_name, word) return "{} {}({}, {})".format(negation, function_name, attribute, word).strip(), attrs @fluent def function(self, function_name, word): """ Apply a function on given word :param str function_name: function to apply :param str word: word to apply function on :rtype: Query """ word = self._parse_filter_word(word) self._add_filter(*self._prepare_function( function_name, self._attribute, word, self._negation)) return self @fluent def contains(self, word): """ Adds a contains word check :param str word: word to check :rtype: Query """ return self.function('contains', word) @fluent def startswith(self, word): """ Adds a startswith word check :param str word: word to check :rtype: Query """ return self.function('startswith', word) @fluent def endswith(self, word): """ Adds a endswith word check :param str word: word to check :rtype: Query """ return self.function('endswith', word) @fluent def iterable(self, iterable_name, *, collection, attribute, word, func=None, operation=None): """ Performs a filter with the OData 'iterable_name' keyword on the collection For example: q.iterable('any', collection='email_addresses', attribute='address', operation='eq', word='*****@*****.**') will transform to a filter such as: emailAddresses/any(a:a/address eq '*****@*****.**') :param str iterable_name: the OData name of the iterable :param str collection: the collection to apply the any keyword on :param str attribute: the attribute of the collection to check :param str word: the word to check :param str func: the logical function to apply to the attribute inside the collection :param str operation: the logical operation to apply to the attribute inside the collection :rtype: Query """ if func is None and operation is None: raise ValueError('Provide a function or an operation to apply') elif func is not None and operation is not None: raise ValueError( 'Provide either a function or an operation but not both') current_att = self._attribute self._attribute = iterable_name word = self._parse_filter_word(word) collection = self._get_mapping(collection) attribute = self._get_mapping(attribute) if func is not None: sentence = self._prepare_function(func, attribute, word) else: sentence = self._prepare_sentence(attribute, operation, word) filter_str, attrs = sentence filter_data = '{}/{}(a:a/{})'.format(collection, iterable_name, filter_str), attrs self._add_filter(*filter_data) self._attribute = current_att return self @fluent def any(self, *, collection, attribute, word, func=None, operation=None): """ Performs a filter with the OData 'any' keyword on the collection For example: q.any(collection='email_addresses', attribute='address', operation='eq', word='*****@*****.**') will transform to a filter such as: emailAddresses/any(a:a/address eq '*****@*****.**') :param str collection: the collection to apply the any keyword on :param str attribute: the attribute of the collection to check :param str word: the word to check :param str func: the logical function to apply to the attribute inside the collection :param str operation: the logical operation to apply to the attribute inside the collection :rtype: Query """ return self.iterable('any', collection=collection, attribute=attribute, word=word, func=func, operation=operation) @fluent def all(self, *, collection, attribute, word, func=None, operation=None): """ Performs a filter with the OData 'all' keyword on the collection For example: q.any(collection='email_addresses', attribute='address', operation='eq', word='*****@*****.**') will transform to a filter such as: emailAddresses/all(a:a/address eq '*****@*****.**') :param str collection: the collection to apply the any keyword on :param str attribute: the attribute of the collection to check :param str word: the word to check :param str func: the logical function to apply to the attribute inside the collection :param str operation: the logical operation to apply to the attribute inside the collection :rtype: Query """ return self.iterable('all', collection=collection, attribute=attribute, word=word, func=func, operation=operation) @fluent def order_by(self, attribute=None, *, ascending=True): """ Applies a order_by clause :param str attribute: attribute to apply on :param bool ascending: should it apply ascending order or descending :rtype: Query """ attribute = self._get_mapping(attribute) or self._attribute if attribute: self._order_by[attribute] = None if ascending else 'desc' else: raise ValueError( 'Attribute property needed. call on_attribute(attribute) ' 'or new(attribute)') return self
class ConfiguredObject(object): """Superclass for classes that take configuration from YAML files""" # Roles of defined objects and the category they belong to # NB the double underscore is deliberate - attribute must be hidden from subclasses __content_roles = [] # Procedure names - placeholder. # Will be replaced by a set in any subclasses that can contain procedures # Note that _procedure_names may *not* be set if it is already set in a superclass _procedure_names = None def __init__(self, name): self.name = name self._objects = OrderedDict((role, None) for role in self.all_roles) def _init(self): """Object initialisation - executed *before* loading contents""" pass def init(self): """Object initialisation - executed *after* loading contents""" pass def replace_object(self, role, new_object): """Replace already defined Object with a new one - for runtime use Args: role (text_str): Role name of contained Object new_object (Optional[ConfiguredObject]): New contained Object """ if role in self._objects: self._objects[role] = new_object else: raise ValueError("Unknown contained Object role: %s" % role) # NB this function must be re-implemented in nested subclasses @property def all_roles(self): """Tuple of all content object roles, indefinition and loading order Returns: tuple[text_str, ...] """ return tuple(self.__content_roles) @property def all_objects_by_role(self): """All contained Objects mapped by role (in specification order). Includes objects defined in subclasses. Returns: OrderedDict[text_str, ConfiguredObject]: """ return self._objects.copy() @property def procedures(self): """Procedures attached to this object mapped by name (in specification order). Returns: OrderedDict[text_str, ConfiguredObject]: """ procedure_names = self.__class__._procedure_names result = OrderedDict() if procedure_names: for name in procedure_names: procedure = getattr(self, name) if procedure is not None: result[name] = procedure return result
class RecalculateSubsectionGradeTest(ModuleStoreTestCase): """ Ensures that the recalculate subsection grade task functions as expected when run. """ def setUp(self): super(RecalculateSubsectionGradeTest, self).setUp() self.user = UserFactory() PersistentGradesEnabledFlag.objects.create(enabled_for_all_courses=True, enabled=True) def set_up_course(self, enable_subsection_grades=True): """ Configures the course for this test. """ # pylint: disable=attribute-defined-outside-init,no-member self.course = CourseFactory.create( org='edx', name='course', run='run', ) if not enable_subsection_grades: PersistentGradesEnabledFlag.objects.create(enabled=False) self.chapter = ItemFactory.create(parent=self.course, category="chapter", display_name="Chapter") self.sequential = ItemFactory.create(parent=self.chapter, category='sequential', display_name="Sequential1") self.problem = ItemFactory.create(parent=self.sequential, category='problem', display_name='Problem') self.frozen_now_datetime = datetime.now().replace(tzinfo=pytz.UTC) self.frozen_now_timestamp = to_timestamp(self.frozen_now_datetime) self.problem_weighted_score_changed_kwargs = OrderedDict([ ('weighted_earned', 1.0), ('weighted_possible', 2.0), ('user_id', self.user.id), ('course_id', unicode(self.course.id)), ('usage_id', unicode(self.problem.location)), ('only_if_higher', None), ('modified', self.frozen_now_datetime), ]) create_new_event_transaction_id() self.recalculate_subsection_grade_kwargs = OrderedDict([ ('user_id', self.user.id), ('course_id', unicode(self.course.id)), ('usage_id', unicode(self.problem.location)), ('only_if_higher', None), ('expected_modified_time', self.frozen_now_timestamp), ('score_deleted', False), ('event_transaction_id', unicode(get_event_transaction_id())), ('event_transaction_type', u'edx.grades.problem.submitted'), ]) # this call caches the anonymous id on the user object, saving 4 queries in all happy path tests _ = anonymous_id_for_user(self.user, self.course.id) # pylint: enable=attribute-defined-outside-init,no-member @contextmanager def mock_get_score(self, score=MagicMock(grade=1.0, max_grade=2.0)): """ Mocks the scores needed by the SCORE_PUBLISHED signal handler. By default, sets the returned score to 1/2. """ with patch("lms.djangoapps.grades.tasks.get_score", return_value=score): yield def test_problem_weighted_score_changed_queues_task(self): """ Ensures that the PROBLEM_WEIGHTED_SCORE_CHANGED signal enqueues the correct task. """ self.set_up_course() send_args = self.problem_weighted_score_changed_kwargs local_task_args = self.recalculate_subsection_grade_kwargs.copy() local_task_args['event_transaction_type'] = u'edx.grades.problem.submitted' with self.mock_get_score() and patch( 'lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.apply_async', return_value=None ) as mock_task_apply: PROBLEM_WEIGHTED_SCORE_CHANGED.send(sender=None, **send_args) mock_task_apply.assert_called_once_with(kwargs=local_task_args) @patch('lms.djangoapps.grades.signals.signals.SUBSECTION_SCORE_CHANGED.send') def test_subsection_update_triggers_signal(self, mock_subsection_signal): """ Ensures that the subsection update operation triggers a signal. """ self.set_up_course() self._apply_recalculate_subsection_grade() self.assertTrue(mock_subsection_signal.called) @ddt.data( (ModuleStoreEnum.Type.mongo, 1, 23), (ModuleStoreEnum.Type.split, 3, 22), ) @ddt.unpack def test_subsection_grade_updated(self, default_store, num_mongo_calls, num_sql_calls): with self.store.default_store(default_store): self.set_up_course() self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id)) with check_mongo_calls(num_mongo_calls): with self.assertNumQueries(num_sql_calls): self._apply_recalculate_subsection_grade() @patch('lms.djangoapps.grades.signals.signals.SUBSECTION_SCORE_CHANGED.send') def test_other_inaccessible_subsection(self, mock_subsection_signal): self.set_up_course() accessible_seq = ItemFactory.create(parent=self.chapter, category='sequential') inaccessible_seq = ItemFactory.create(parent=self.chapter, category='sequential', visible_to_staff_only=True) # Update problem to have 2 additional sequential parents. # So in total, 3 sequential parents, with one inaccessible. for sequential in (accessible_seq, inaccessible_seq): sequential.children = [self.problem.location] modulestore().update_item(sequential, self.user.id) # pylint: disable=no-member # Make sure the signal is sent for only the 2 accessible sequentials. self._apply_recalculate_subsection_grade() self.assertEquals(mock_subsection_signal.call_count, 2) sequentials_signalled = { args[1]['subsection_grade'].location for args in mock_subsection_signal.call_args_list } self.assertSetEqual( sequentials_signalled, {self.sequential.location, accessible_seq.location}, ) def test_single_call_to_create_block_structure(self): self.set_up_course() self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id)) with patch( 'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_cache', return_value=None, ) as mock_block_structure_create: self._apply_recalculate_subsection_grade() self.assertEquals(mock_block_structure_create.call_count, 1) # TODO (TNL-6225) Fix the number of SQL queries so they # don't grow linearly with the number of sequentials. @ddt.data( (ModuleStoreEnum.Type.mongo, 1, 46), (ModuleStoreEnum.Type.split, 3, 45), ) @ddt.unpack def test_query_count_does_not_change_with_more_content(self, default_store, num_mongo_calls, num_sql_calls): with self.store.default_store(default_store): self.set_up_course() self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id)) num_problems = 10 for _ in range(num_problems): ItemFactory.create(parent=self.sequential, category='problem') num_sequentials = 10 for _ in range(num_sequentials): ItemFactory.create(parent=self.chapter, category='sequential') with check_mongo_calls(num_mongo_calls): with self.assertNumQueries(num_sql_calls): self._apply_recalculate_subsection_grade() @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) def test_subsection_grades_not_enabled_on_course(self, default_store): with self.store.default_store(default_store): self.set_up_course(enable_subsection_grades=False) self.assertFalse(PersistentGradesEnabledFlag.feature_enabled(self.course.id)) with check_mongo_calls(0): with self.assertNumQueries(0): self._apply_recalculate_subsection_grade() @skip("Pending completion of TNL-5089") @ddt.data( (ModuleStoreEnum.Type.mongo, True), (ModuleStoreEnum.Type.split, True), (ModuleStoreEnum.Type.mongo, False), (ModuleStoreEnum.Type.split, False), ) @ddt.unpack def test_query_counts_with_feature_flag(self, default_store, feature_flag): PersistentGradesEnabledFlag.objects.create(enabled=feature_flag) with self.store.default_store(default_store): self.set_up_course() with check_mongo_calls(0): with self.assertNumQueries(3 if feature_flag else 2): self._apply_recalculate_subsection_grade() @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry') @patch('lms.djangoapps.grades.new.subsection_grade.SubsectionGradeFactory.update') def test_retry_subsection_update_on_integrity_error(self, mock_update, mock_retry): """ Ensures that tasks will be retried if IntegrityErrors are encountered. """ self.set_up_course() mock_update.side_effect = IntegrityError("WHAMMY") self._apply_recalculate_subsection_grade() self._assert_retry_called(mock_retry) @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry') def test_retry_subsection_grade_on_update_not_complete(self, mock_retry): self.set_up_course() self._apply_recalculate_subsection_grade( mock_score=MagicMock(modified=datetime.utcnow().replace(tzinfo=pytz.UTC) - timedelta(days=1)) ) self._assert_retry_called(mock_retry) @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry') def test_retry_subsection_grade_on_no_score(self, mock_retry): self.set_up_course() self._apply_recalculate_subsection_grade(mock_score=None) self._assert_retry_called(mock_retry) @patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry') def test_retry_subsection_grade_on_no_sub_score(self, mock_retry): self.set_up_course() with patch('lms.djangoapps.grades.tasks.sub_api.get_score') as mock_sub_score: mock_sub_score.return_value = None self._apply_recalculate_subsection_grade( mock_score=MagicMock(module_type='openassessment') ) self._assert_retry_called(mock_retry) @patch('lms.djangoapps.grades.signals.signals.SUBSECTION_SCORE_CHANGED.send') @patch('lms.djangoapps.grades.new.subsection_grade.SubsectionGradeFactory.update') def test_retry_first_time_only(self, mock_update, mock_course_signal): """ Ensures that a task retry completes after a one-time failure. """ self.set_up_course() mock_update.side_effect = [IntegrityError("WHAMMY"), None] self._apply_recalculate_subsection_grade() self.assertEquals(mock_course_signal.call_count, 1) def _apply_recalculate_subsection_grade( self, mock_score=MagicMock(modified=datetime.utcnow().replace(tzinfo=pytz.UTC) + timedelta(days=1)) ): """ Calls the recalculate_subsection_grade task with necessary mocking in place. """ with self.mock_get_score(mock_score): recalculate_subsection_grade_v2.apply(kwargs=self.recalculate_subsection_grade_kwargs) def _assert_retry_called(self, mock_retry): """ Verifies the task was retried and with the correct number of arguments. """ self.assertTrue(mock_retry.called) self.assertEquals(len(mock_retry.call_args[1]['kwargs']), len(self.recalculate_subsection_grade_kwargs))
class MultiRemoteTest(unittest.TestCase): def setUp(self): self.servers = OrderedDict() self.users = {} for i in range(3): test_server = TestServer() self.servers["remote%d" % i] = test_server self.users["remote%d" % i] = [("lasote", "mypass")] self.client = TestClient(servers=self.servers, users=self.users) def predefine_remote_test(self): files = cpp_hello_conan_files("Hello0", "0.1", build=False) self.client.save(files) self.client.run("export . lasote/stable") self.client.run("upload Hello0/0.1@lasote/stable -r=remote0") self.client.run("upload Hello0/0.1@lasote/stable -r=remote1") self.client.run("upload Hello0/0.1@lasote/stable -r=remote2") self.client.run('remove "*" -f') self.client.run("remote add_ref Hello0/0.1@lasote/stable remote1") self.client.run("install Hello0/0.1@lasote/stable --build=missing") self.assertIn("Hello0/0.1@lasote/stable: Retrieving from predefined remote 'remote1'", self.client.user_io.out) self.client.run("remote list_ref") self.assertIn("Hello0/0.1@lasote/stable: remote1", self.client.user_io.out) def upload_test(self): conan_reference = ConanFileReference.loads("Hello0/0.1@lasote/stable") files = cpp_hello_conan_files("Hello0", "0.1", build=False) self.client.save(files) self.client.run("export . lasote/stable") self.client.run("upload %s" % str(conan_reference)) self.client.run("info %s" % str(conan_reference)) self.assertIn("remote0=http://", self.client.user_io.out) # The remote, once fixed does not change self.client.run("upload %s -r=remote1" % str(conan_reference)) self.client.run("info %s" % str(conan_reference)) self.assertIn("remote0=http://", self.client.user_io.out) # Now install it in other machine from remote 0 client2 = TestClient(servers=self.servers, users=self.users) client2.run("install %s --build=missing" % str(conan_reference)) client2.run("info %s" % str(conan_reference)) self.assertIn("remote0=http://", client2.user_io.out) # Now install it in other machine from remote 1 servers = self.servers.copy() servers.pop("remote0") client3 = TestClient(servers=servers, users=self.users) client3.run("install %s --build=missing" % str(conan_reference)) client3.run("info %s" % str(conan_reference)) self.assertIn("remote1=http://", client3.user_io.out) def fail_when_not_notfound_test(self): """ If a remote fails with a 404 it has to keep looking in the next remote, but if it fails by any other reason it has to stop """ servers = OrderedDict() servers["s0"] = TestServer() servers["s1"] = TestServer() servers["s2"] = TestServer() client = TestClient(servers=servers, users=self.users) files = cpp_hello_conan_files("MyLib", "0.1", build=False) client.save(files) client.run("create . lasote/testing") client.run("user lasote -p mypass -r s1") client.run("upload MyLib* -r s1 -c") servers["s1"].fake_url = "http://asdlhaljksdhlajkshdljakhsd" # Do not exist client2 = TestClient(servers=servers, users=self.users) err = client2.run("install MyLib/0.1@conan/testing --build=missing", ignore_error=True) self.assertTrue(err) self.assertIn("MyLib/0.1@conan/testing: Trying with 's0'...", client2.out) self.assertIn("MyLib/0.1@conan/testing: Trying with 's1'...", client2.out) self.assertIn("Unable to connect to s1=http://asdlhaljksdhlajkshdljakhsd", client2.out) # s2 is not even tried self.assertNotIn("MyLib/0.1@conan/testing: Trying with 's2'...", client2.out) def install_from_remotes_test(self): for i in range(3): conan_reference = ConanFileReference.loads("Hello%d/0.1@lasote/stable" % i) files = cpp_hello_conan_files("Hello%d" % i, "0.1", build=False) self.client.save(files) self.client.run("export . lasote/stable") self.client.run("upload %s -r=remote%d" % (str(conan_reference), i)) self.client.run("info %s" % str(conan_reference)) self.assertIn("remote%d=http://" % i, self.client.user_io.out) # Now install it in other machine from remote 0 client2 = TestClient(servers=self.servers, users=self.users) files = cpp_hello_conan_files("HelloX", "0.1", deps=["Hello0/0.1@lasote/stable", "Hello1/0.1@lasote/stable", "Hello2/0.1@lasote/stable"]) files["conanfile.py"] = files["conanfile.py"].replace("def build(", "def build2(") client2.save(files) client2.run("install . --build=missing") self.assertIn("Hello0/0.1@lasote/stable from remote0", client2.user_io.out) self.assertIn("Hello1/0.1@lasote/stable from remote1", client2.user_io.out) self.assertIn("Hello2/0.1@lasote/stable from remote2", client2.user_io.out) client2.run("info .") self.assertIn("Remote: remote0=http://", client2.user_io.out) self.assertIn("Remote: remote1=http://", client2.user_io.out) self.assertIn("Remote: remote2=http://", client2.user_io.out)
class DAGDependency: """Object to represent a quantum circuit as a directed acyclic graph via operation dependencies (i.e. lack of commutation). The nodes in the graph are operations represented by quantum gates. The edges correspond to non-commutation between two operations (i.e. a dependency). A directed edge from node A to node B means that operation A does not commute with operation B. The object's methods allow circuits to be constructed. The nodes in the graph have the following attributes: 'operation', 'successors', 'predecessors'. **Example:** Bell circuit with no measurement. .. parsed-literal:: ┌───┐ qr_0: ┤ H ├──■── └───┘┌─┴─┐ qr_1: ─────┤ X ├ └───┘ The dependency DAG for the above circuit is represented by two nodes. The first one corresponds to Hadamard gate, the second one to the CNOT gate as the gates do not commute there is an edge between the two nodes. **Reference:** [1] Iten, R., Moyard, R., Metger, T., Sutter, D. and Woerner, S., 2020. Exact and practical pattern matching for quantum circuit optimization. `arXiv:1909.05270 <https://arxiv.org/abs/1909.05270>`_ """ def __init__(self): """ Create an empty DAGDependency. """ # Circuit name self.name = None # Circuit metadata self.metadata = None # Directed multigraph whose nodes are operations(gates) and edges # represent non-commutativity between two gates. self._multi_graph = rx.PyDAG() # Map of qreg/creg name to Register object. self.qregs = OrderedDict() self.cregs = OrderedDict() # List of all Qubit/Clbit wires. self.qubits = [] self.clbits = [] self._global_phase = 0 self._calibrations = defaultdict(dict) self.duration = None self.unit = "dt" self.comm_checker = CommutationChecker() @property def global_phase(self): """Return the global phase of the circuit.""" return self._global_phase @global_phase.setter def global_phase(self, angle): """Set the global phase of the circuit. Args: angle (float, ParameterExpression) """ from qiskit.circuit.parameterexpression import ParameterExpression # needed? if isinstance(angle, ParameterExpression): self._global_phase = angle else: # Set the phase to the [0, 2π) interval angle = float(angle) if not angle: self._global_phase = 0 else: self._global_phase = angle % (2 * math.pi) @property def calibrations(self): """Return calibration dictionary. The custom pulse definition of a given gate is of the form {'gate_name': {(qubits, params): schedule}} """ return dict(self._calibrations) @calibrations.setter def calibrations(self, calibrations): """Set the circuit calibration data from a dictionary of calibration definition. Args: calibrations (dict): A dictionary of input in the format {'gate_name': {(qubits, gate_params): schedule}} """ self._calibrations = defaultdict(dict, calibrations) def to_networkx(self): """Returns a copy of the DAGDependency in networkx format.""" # For backwards compatibility, return networkx structure from terra 0.12 # where DAGNodes instances are used as indexes on the networkx graph. warnings.warn( "The to_networkx() method is deprecated and will be removed in a future release.", DeprecationWarning, stacklevel=2, ) try: import networkx as nx except ImportError as ex: raise MissingOptionalLibraryError( libname="Networkx", name="DAG dependency", pip_install="pip install networkx", ) from ex dag_networkx = nx.MultiDiGraph() for node in self.get_nodes(): dag_networkx.add_node(node) for node in self.topological_nodes(): for source_id, dest_id, edge in self.get_in_edges(node.node_id): dag_networkx.add_edge(self.get_node(source_id), self.get_node(dest_id), **edge) return dag_networkx def to_retworkx(self): """Returns the DAGDependency in retworkx format.""" return self._multi_graph def size(self): """Returns the number of gates in the circuit""" return len(self._multi_graph) def depth(self): """Return the circuit depth. Returns: int: the circuit depth """ depth = rx.dag_longest_path_length(self._multi_graph) return depth if depth >= 0 else 0 def add_qubits(self, qubits): """Add individual qubit wires.""" if any(not isinstance(qubit, Qubit) for qubit in qubits): raise DAGDependencyError("not a Qubit instance.") duplicate_qubits = set(self.qubits).intersection(qubits) if duplicate_qubits: raise DAGDependencyError("duplicate qubits %s" % duplicate_qubits) self.qubits.extend(qubits) def add_clbits(self, clbits): """Add individual clbit wires.""" if any(not isinstance(clbit, Clbit) for clbit in clbits): raise DAGDependencyError("not a Clbit instance.") duplicate_clbits = set(self.clbits).intersection(clbits) if duplicate_clbits: raise DAGDependencyError("duplicate clbits %s" % duplicate_clbits) self.clbits.extend(clbits) def add_qreg(self, qreg): """Add qubits in a quantum register.""" if not isinstance(qreg, QuantumRegister): raise DAGDependencyError("not a QuantumRegister instance.") if qreg.name in self.qregs: raise DAGDependencyError("duplicate register %s" % qreg.name) self.qregs[qreg.name] = qreg existing_qubits = set(self.qubits) for j in range(qreg.size): if qreg[j] not in existing_qubits: self.qubits.append(qreg[j]) def add_creg(self, creg): """Add clbits in a classical register.""" if not isinstance(creg, ClassicalRegister): raise DAGDependencyError("not a ClassicalRegister instance.") if creg.name in self.cregs: raise DAGDependencyError("duplicate register %s" % creg.name) self.cregs[creg.name] = creg existing_clbits = set(self.clbits) for j in range(creg.size): if creg[j] not in existing_clbits: self.clbits.append(creg[j]) def _add_multi_graph_node(self, node): """ Args: node (DAGDepNode): considered node. Returns: node_id(int): corresponding label to the added node. """ node_id = self._multi_graph.add_node(node) node.node_id = node_id return node_id def get_nodes(self): """ Returns: generator(dict): iterator over all the nodes. """ return iter(self._multi_graph.nodes()) def get_node(self, node_id): """ Args: node_id (int): label of considered node. Returns: node: corresponding to the label. """ return self._multi_graph.get_node_data(node_id) def _add_multi_graph_edge(self, src_id, dest_id, data): """ Function to add an edge from given data (dict) between two nodes. Args: src_id (int): label of the first node. dest_id (int): label of the second node. data (dict): data contained on the edge. """ self._multi_graph.add_edge(src_id, dest_id, data) def get_edges(self, src_id, dest_id): """ Edge enumeration between two nodes through method get_all_edge_data. Args: src_id (int): label of the first node. dest_id (int): label of the second node. Returns: List: corresponding to all edges between the two nodes. """ return self._multi_graph.get_all_edge_data(src_id, dest_id) def get_all_edges(self): """ Enumeration of all edges. Returns: List: corresponding to the label. """ return [ (src, dest, data) for src_node in self._multi_graph.nodes() for (src, dest, data) in self._multi_graph.out_edges(src_node.node_id) ] def get_in_edges(self, node_id): """ Enumeration of all incoming edges for a given node. Args: node_id (int): label of considered node. Returns: List: corresponding incoming edges data. """ return self._multi_graph.in_edges(node_id) def get_out_edges(self, node_id): """ Enumeration of all outgoing edges for a given node. Args: node_id (int): label of considered node. Returns: List: corresponding outgoing edges data. """ return self._multi_graph.out_edges(node_id) def direct_successors(self, node_id): """ Direct successors id of a given node as sorted list. Args: node_id (int): label of considered node. Returns: List: direct successors id as a sorted list """ return sorted(list(self._multi_graph.adj_direction(node_id, False).keys())) def direct_predecessors(self, node_id): """ Direct predecessors id of a given node as sorted list. Args: node_id (int): label of considered node. Returns: List: direct predecessors id as a sorted list """ return sorted(list(self._multi_graph.adj_direction(node_id, True).keys())) def successors(self, node_id): """ Successors id of a given node as sorted list. Args: node_id (int): label of considered node. Returns: List: all successors id as a sorted list """ return self._multi_graph.get_node_data(node_id).successors def predecessors(self, node_id): """ Predecessors id of a given node as sorted list. Args: node_id (int): label of considered node. Returns: List: all predecessors id as a sorted list """ return self._multi_graph.get_node_data(node_id).predecessors def topological_nodes(self): """ Yield nodes in topological order. Returns: generator(DAGNode): node in topological order. """ def _key(x): return x.sort_key return iter(rx.lexicographical_topological_sort(self._multi_graph, key=_key)) def add_op_node(self, operation, qargs, cargs): """Add a DAGDepNode to the graph and update the edges. Args: operation (qiskit.circuit.Instruction): operation as a quantum gate. qargs (list[Qubit]): list of qubits on which the operation acts cargs (list[Clbit]): list of classical wires to attach to. """ directives = ["measure"] if not getattr(operation, "_directive", False) and operation.name not in directives: qindices_list = [] for elem in qargs: qindices_list.append(self.qubits.index(elem)) if getattr(operation, "condition", None): for clbit in self.clbits: if clbit in operation.condition[0]: initial = self.clbits.index(clbit) final = self.clbits.index(clbit) + operation.condition[0].size cindices_list = range(initial, final) break else: cindices_list = [] else: qindices_list = [] cindices_list = [] new_node = DAGDepNode( type="op", op=operation, name=operation.name, qargs=qargs, cargs=cargs, successors=[], predecessors=[], qindices=qindices_list, cindices=cindices_list, ) self._add_multi_graph_node(new_node) self._update_edges() def _gather_pred(self, node_id, direct_pred): """Function set an attribute predecessors and gather multiple lists of direct predecessors into a single one. Args: node_id (int): label of the considered node in the DAG direct_pred (list): list of direct successors for the given node Returns: DAGDependency: A multigraph with update of the attribute ['predecessors'] the lists of direct successors are put into a single one """ gather = self._multi_graph gather.get_node_data(node_id).predecessors = [] for d_pred in direct_pred: gather.get_node_data(node_id).predecessors.append([d_pred]) pred = self._multi_graph.get_node_data(d_pred).predecessors gather.get_node_data(node_id).predecessors.append(pred) return gather def _gather_succ(self, node_id, direct_succ): """ Function set an attribute successors and gather multiple lists of direct successors into a single one. Args: node_id (int): label of the considered node in the DAG direct_succ (list): list of direct successors for the given node Returns: MultiDiGraph: with update of the attribute ['predecessors'] the lists of direct successors are put into a single one """ gather = self._multi_graph for d_succ in direct_succ: gather.get_node_data(node_id).successors.append([d_succ]) succ = gather.get_node_data(d_succ).successors gather.get_node_data(node_id).successors.append(succ) return gather def _list_pred(self, node_id): """ Use _gather_pred function and merge_no_duplicates to construct the list of predecessors for a given node. Args: node_id (int): label of the considered node """ direct_pred = self.direct_predecessors(node_id) self._multi_graph = self._gather_pred(node_id, direct_pred) self._multi_graph.get_node_data(node_id).predecessors = list( merge_no_duplicates(*(self._multi_graph.get_node_data(node_id).predecessors)) ) def _update_edges(self): """ Function to verify the commutation relation and reachability for predecessors, the nodes do not commute and if the predecessor is reachable. Update the DAGDependency by introducing edges and predecessors(attribute) """ max_node_id = len(self._multi_graph) - 1 max_node = self._multi_graph.get_node_data(max_node_id) for current_node_id in range(0, max_node_id): self._multi_graph.get_node_data(current_node_id).reachable = True # Check the commutation relation with reachable node, it adds edges if it does not commute for prev_node_id in range(max_node_id - 1, -1, -1): prev_node = self._multi_graph.get_node_data(prev_node_id) if prev_node.reachable and not self.comm_checker.commute( prev_node.op, prev_node.qargs, prev_node.cargs, max_node.op, max_node.qargs, max_node.cargs, ): self._multi_graph.add_edge(prev_node_id, max_node_id, {"commute": False}) self._list_pred(max_node_id) list_predecessors = self._multi_graph.get_node_data(max_node_id).predecessors for pred_id in list_predecessors: self._multi_graph.get_node_data(pred_id).reachable = False def _add_successors(self): """ Use _gather_succ and merge_no_duplicates to create the list of successors for each node. Update DAGDependency 'successors' attribute. It has to be used when the DAGDependency() object is complete (i.e. converters). """ for node_id in range(len(self._multi_graph) - 1, -1, -1): direct_successors = self.direct_successors(node_id) self._multi_graph = self._gather_succ(node_id, direct_successors) self._multi_graph.get_node_data(node_id).successors = list( merge_no_duplicates(*self._multi_graph.get_node_data(node_id).successors) ) def copy(self): """ Function to copy a DAGDependency object. Returns: DAGDependency: a copy of a DAGDependency object. """ dag = DAGDependency() dag.name = self.name dag.cregs = self.cregs.copy() dag.qregs = self.qregs.copy() for node in self.get_nodes(): dag._multi_graph.add_node(node.copy()) for edges in self.get_all_edges(): dag._multi_graph.add_edge(edges[0], edges[1], edges[2]) return dag def draw(self, scale=0.7, filename=None, style="color"): """ Draws the DAGDependency graph. This function needs `pydot <https://github.com/erocarrera/pydot>`, which in turn needs Graphviz <https://www.graphviz.org/>` to be installed. Args: scale (float): scaling factor filename (str): file path to save image to (format inferred from name) style (str): 'plain': B&W graph 'color' (default): color input/output/op nodes Returns: Ipython.display.Image: if in Jupyter notebook and not saving to file, otherwise None. """ from qiskit.visualization.dag_visualization import dag_drawer return dag_drawer(dag=self, scale=scale, filename=filename, style=style)
class GraphView(urwid.WidgetPlaceholder): """ A class responsible for providing the application's interface and graph display. """ def __init__(self, controller, args): self.controller = controller self.custom_temp = args.custom_temp self.custom_fan = args.custom_fan self.args = args self.hline = urwid.AttrWrap(urwid.SolidFill(u'_'), 'line') self.mode_buttons = [] self.refresh_rate_ctrl = urwid.Edit(('bold text', u'Refresh[s]:'), self.controller.refresh_rate) self.visible_graphs = {} self.graph_place_holder = urwid.WidgetPlaceholder(urwid.Pile([])) self.main_window_w = [] self.stress_menu = StressMenu(self.on_menu_close) self.help_menu = HelpMenu(self.on_menu_close) self.about_menu = AboutMenu(self.on_menu_close) self.temp_sensors_menu = TempSensorsMenu(self.on_sensors_menu_close) self.global_data = GlobalData(is_admin) self.stress_menu.sqrt_workers = str(self.global_data.num_cpus) self.left_margin = 0 self.top_margin = 0 self.v_relative = 50 self.h_relative = 50 urwid.WidgetPlaceholder.__init__(self, self.main_window()) urwid.connect_signal(self.refresh_rate_ctrl, 'change', self.update_refresh_rate) def update_refresh_rate(self, edit, new_refresh_rate): try: if float(new_refresh_rate) <= 0.001: pass else: self.controller.refresh_rate = new_refresh_rate except: self.controller.refresh_rate = '1.0' def update_displayed_information(self): """ Update all the graphs that are being displayed """ for key, val in self.summaries.items(): val.source.update() for g in self.visible_graphs.values(): g.update_displayed_graph_data() for s in self.available_summaries.values(): s.update() def on_reset_button(self, w): """Reset graph data and display empty graph""" for g in self.visible_graphs.values(): g.reset() for g in self.graphs.values(): try: g.source.reset() except (NotImplementedError): pass self.update_displayed_information() def on_menu_close(self): """Return to main screen""" self.original_widget = self.main_window_w def on_sensors_menu_close(self): """Return to main screen and update sensor""" if self.temp_sensors_menu.current_active_mode: logging.info("State is not None") self.args.custom_temp = self.temp_sensors_menu.current_active_mode self.__init__(self.controller, self.args) logging.info("Temp sensor updated to " + self.args.custom_temp) else: logging.info("Temp sensor is None") self.original_widget = self.main_window_w def on_stress_menu_open(self, w): """Open stress options""" self.original_widget = urwid.Overlay(self.stress_menu.main_window, self.original_widget, ('relative', self.left_margin), self.stress_menu.get_size()[1], ('relative', self.top_margin), self.stress_menu.get_size()[0]) def on_help_menu_open(self, w): """Open Help menu""" self.original_widget = urwid.Overlay(self.help_menu.main_window, self.original_widget, ('relative', self.left_margin), self.help_menu.get_size()[1], ('relative', self.top_margin), self.help_menu.get_size()[0]) def on_about_menu_open(self, w): """Open About menu""" self.original_widget = urwid.Overlay(self.about_menu.main_window, self.original_widget, ('relative', self.left_margin), self.about_menu.get_size()[1], ('relative', self.top_margin), self.about_menu.get_size()[0]) def on_temp_sensors_menu_open(self, w): """Open About menu""" self.original_widget = urwid.Overlay( self.temp_sensors_menu.main_window, self.original_widget, ('relative', self.left_margin), self.temp_sensors_menu.get_size()[1], ('relative', self.top_margin), self.temp_sensors_menu.get_size()[0]) def on_mode_button(self, button, state): """Notify the controller of a new mode setting.""" if state: # The new mode is the label of the button self.controller.set_mode(button.get_label()) self.controller.start_stress() def on_mode_change(self, m): """Handle external mode change by updating radio buttons.""" for rb in self.mode_buttons: if rb.get_label() == m: rb.set_state(True, do_callback=False) break def on_unicode_checkbox(self, w, state): """Enable smooth edges if utf-8 is supported""" logging.debug("unicode State is " + str(state)) if state: self.hline = urwid.AttrWrap( urwid.SolidFill(u'\N{LOWER ONE QUARTER BLOCK}'), 'line') else: self.hline = urwid.AttrWrap(urwid.SolidFill(u'_'), 'line') for g_name, g in self.graphs.items(): g.set_smooth_colors(state) self.show_graphs() def exit_program(self, w=None): """ Kill all stress operations upon exit""" try: kill_child_processes(self.controller.mode.get_stress_process()) except: logging.debug('Could not kill process') raise urwid.ExitMainLoop() def graph_controls(self): """ Dislplay sidebar controls. i.e. buttons, and controls""" modes = self.controller.get_modes() # setup mode radio buttons group = [] for m in modes: rb = radio_button(group, m, self.on_mode_button) self.mode_buttons.append(rb) # Create list of buttons control_options = [button("Reset", self.on_reset_button)] if stress_installed: control_options.append( button('Stress Options', self.on_stress_menu_open)) control_options.append( button('Temp Sensors', self.on_temp_sensors_menu_open)) control_options.append(button('Help', self.on_help_menu_open)) control_options.append(button('About', self.on_about_menu_open)) # Create the menu animate_controls = urwid.GridFlow(control_options, 18, 2, 0, 'center') if urwid.get_encoding_mode() == "utf8": unicode_checkbox = urwid.CheckBox( "Smooth Graph", state=False, on_state_change=self.on_unicode_checkbox) else: unicode_checkbox = urwid.Text("UTF-8 encoding not detected") install_stress_message = urwid.Text("") if not stress_installed: install_stress_message = urwid.Text( ('button normal', u"(N/A) install stress")) graph_checkboxes = [ urwid.CheckBox(x.get_graph_name(), state=True, on_state_change=lambda w, state, x=x: self. change_checkbox_state(x, state)) for x in self.available_graphs.values() ] unavalable_graphs = [ urwid.Text(("[N/A] " + x.get_graph_name())) for x in self.graphs.values() if x.source.get_is_available() == False ] graph_checkboxes += unavalable_graphs buttons = [ urwid.Text(('bold text', u"Modes"), align="center"), ] + self.mode_buttons + [ install_stress_message, urwid.Divider(), urwid.Text(('bold text', u"Control Options"), align="center"), animate_controls, urwid.Divider(), self.refresh_rate_ctrl, urwid.Divider(), urwid.LineBox(urwid.Pile(graph_checkboxes)), urwid.LineBox(unicode_checkbox), urwid.Divider(), button("Quit", self.exit_program), ] return buttons def change_checkbox_state(self, x, state): if state: self.visible_graphs[x.get_graph_name()] = x else: del self.visible_graphs[x.get_graph_name()] self.show_graphs() def show_graphs(self): """Show a pile of the graph selected for dislpay""" elements = itertools.chain.from_iterable( ([graph, ('fixed', 1, self.hline)] for graph in self.visible_graphs.values())) self.graph_place_holder.original_widget = urwid.Pile(elements) def cpu_stats(self): """Read and display processor name """ cpu_name = urwid.Text("CPU Name N/A", align="center") try: cpu_name = urwid.Text(get_processor_name().strip(), align="center") except: logging.info("CPU name not available") cpu_stats = [cpu_name, urwid.Divider()] return cpu_stats def graph_stats(self): fixed_stats = [] for key, val in self.available_summaries.items(): fixed_stats += val.get_text_item_list() return fixed_stats def main_window(self): user_config_path = None if not user_config_dir_exists(): user_config_path = make_user_config_dir() else: user_config_path = get_user_config_path() script_hooks_enabled = True if user_config_path is None: logging.warn( 'Failed to find or create scripts directory, proceeding without scripting support' ) script_hooks_enabled = False else: self.script_loader = ScriptHookLoader(user_config_path) # initiating the graphs self.graphs = OrderedDict() self.summaries = OrderedDict() # TODO: Update to find sensors automatically freq_source = FreqSource(is_admin) self.graphs[freq_source.get_source_name()] = StuiBarGraph( freq_source, 'freq light', 'freq dark', 'freq light smooth', 'freq dark smooth') self.summaries[freq_source.get_source_name()] = SummaryTextList( freq_source) util_source = UtilSource() self.graphs[util_source.get_source_name()] = StuiBarGraph( util_source, 'util light', 'util dark', 'util light smooth', 'util dark smooth') self.summaries[util_source.get_source_name()] = SummaryTextList( util_source) temp_source = TemperatureSource(self.custom_temp) if script_hooks_enabled: temp_source.add_edge_hook( self.script_loader.load_script(temp_source.__class__.__name__, 30000) ) # Invoke threshold script every 30s while threshold is exceeded. alert_colors = [ 'high temp light', 'high temp dark', 'high temp light smooth', 'high temp dark smooth' ] self.graphs[temp_source.get_source_name()] = StuiBarGraph( temp_source, 'temp light', 'temp dark', 'temp light smooth', 'temp dark smooth', alert_colors=alert_colors) self.summaries[temp_source.get_source_name()] = SummaryTextList( temp_source, 'high temp txt') rapl_power_source = RaplPowerSource() self.graphs[rapl_power_source.get_source_name()] = StuiBarGraph( rapl_power_source, 'power dark', 'power light', 'power dark smooth', 'power light smooth') self.summaries[rapl_power_source.get_source_name()] = SummaryTextList( rapl_power_source) fan_source = FanSource(self.custom_fan) self.summaries[fan_source.get_source_name()] = SummaryTextList( fan_source) # only interested in available graph self.available_graphs = OrderedDict( (key, val) for key, val in self.graphs.items() if val.get_is_available()) self.available_summaries = OrderedDict( (key, val) for key, val in self.summaries.items() if val.get_is_available()) self.visible_graphs = self.available_graphs.copy() self.show_graphs() cpu_stats = self.cpu_stats() graph_controls = self.graph_controls() graph_stats = self.graph_stats() text_col = ViListBox( urwid.SimpleListWalker(cpu_stats + graph_controls + [urwid.Divider()] + graph_stats)) vline = urwid.AttrWrap(urwid.SolidFill(u'\u2502'), 'line') w = urwid.Columns([ ('fixed', 20, text_col), ('fixed', 1, vline), ('weight', 2, self.graph_place_holder), ], dividechars=1, focus_column=0) w = urwid.Padding(w, ('fixed left', 1), ('fixed right', 0)) w = urwid.AttrWrap(w, 'body') w = urwid.LineBox(w) w = urwid.AttrWrap(w, 'line') self.main_window_w = w return self.main_window_w
class Serializer(BaseSerializer): """ Serializer class. """ def to_internal_value(self, data): """ Data transformation to python object. :param dict data: Data for transformation. :return: Transformed data. :rtype: dict :raise ValidationError: If not valid data. """ return self._field_validation(self.fields, data) def to_representation(self, instance): """ Transformation an object to a valid JSON object. :param object instance: The object to transformation. :return: Transformed data. :rtype: object """ res = OrderedDict() # Attributes storage. for field_name, field_val in six.iteritems(self.fields): # TODO: mini hack if not isinstance(field_val, SerializerMethodField): # We try to get the attribute. try: attribute = field_val.get_attribute(instance) except SkipError: # TODO: That thing, throw an error, if the attribute of the object is not found, or skip? continue else: attribute = instance # We try to turn it into a JSON valid format. res[field_name] = field_val.to_representation(attribute) # Return. return res def _manual_validate_method(self, field_name, validated_value): """ Manual validation of a specific field. :param str field_name: Field name. :param object validated_value: Field value. :return: Validation value. :rtype: object """ # We look, if there is a method of manual validation, we call it.. manual_validate_method = getattr(self, 'validate_' + field_name, None) if callable(manual_validate_method): validated_value = manual_validate_method(validated_value) return validated_value def _field_validation(self, fields_dict, data): """ Validation add fields :param dict fields_dict: Dictionary with initialized fields that we validate. :param dict data: Data that is validated. :return: Validated and transformed data. :raise ValidationError: If errors occurred during validation. """ validated_data, errors = OrderedDict(), OrderedDict() # Running through the fields. for field_name, field_obj in six.iteritems(fields_dict): try: # Transform to python type and validate each field. validated_val = field_obj.run_validation( data.get(field_name, None)) # Now manual validation. validated_val = self._manual_validate_method( field_name, validated_val) # And if there was a field in the incoming data, then we save it in the converted form. if field_name in data: validated_data[field_name] = validated_val elif field_obj.default: validated_data[field_name] = field_obj.default except ValidationError as e: # If not passed validation, save the error. errors[field_name] = e.detail except (AttributeError, TypeError, ValueError): # If data not valid format. errors[ field_name] = 'Could not parse data for field `{}`.'.format( field_name) if any(errors): raise ValidationError(errors) return validated_data def is_valid(self, raise_exception=False): """ Validates the data that came into the serializer. :param bool raise_exception: Whether to throw an exception if validation failed? :return: Validation result. :rtype: bool """ if not hasattr(self, 'initial_data'): raise AssertionError( 'Cannot call `.is_valid()` as no `data=` keyword argument ' 'was passed when instantiating the serializer instance.') # Preparing storage for results. self._errors, self._validated_data = OrderedDict(), OrderedDict() # Validated all fields. try: self._validated_data = self._field_validation( self.fields, self.initial_data) except ValidationError as e: self._errors = e.detail # Now run the full manual validation method. try: self._validated_data = self.validate( self._validated_data ) if not self._errors else self._validated_data except ValidationError as e: self._errors['errors'] = e.detail # If you need to throw an error, we throw. if self._errors and raise_exception: self._validated_data = OrderedDict() raise ValidationError(self._errors) # Return validation result. return not bool(self._errors) def run_validation(self, data): """ Runs validation on the current serializer.. :param object data: Data for validation. :return: Transformed and validated data. :rtype: dict """ # We first check to see if an empty field has arrived? is_empty, data = self.validate_empty_values(data) # Transformed to python type. value = self.to_internal_value(data) # Validating validators. try: self.run_validators(value) value = self.validate(value) assert value is not None, '`.validate ()` should return a valid value.' except ValidationError as e: raise ValidationError(detail=e.detail) # We return the validated and transformed value. return value @property def validated_data(self): """ Validated data. :return: Validated data. :rtype: dict """ if not hasattr(self, '_validated_data'): raise AssertionError( 'You must call `.is_valid()` before accessing `.validated_data`.' ) return self._validated_data.copy() @property def errors(self): """ Errors during validation. :return: Errors during validation. :rtype: dict """ if not hasattr(self, '_errors'): raise AssertionError( 'You must call `.is_valid()` before accessing `.errors`.') return self._errors.copy()
class Section: """ This class holds a set of settings. To add settings and sections to a dictionary of sections we can use ``append_to_sections``: >>> sections = {} >>> append_to_sections(sections, ... 'test1', ... 'val', ... 'origin', ... section_name='all') >>> 'all' in sections True >>> len(sections) 1 >>> str(sections) "{'all': <Section object(... contents=OrderedDict([('test1', ..." We can also add settings that can be appended to other settings. Basically it takes the default value of the setting which resides in the defaults of the section and appends the value of the setting in the second and returns the value of the setting: >>> append_to_sections(sections, ... 'test1', ... 'val2', ... 'origin', ... section_name='all.python', ... to_append=True) When the section has no defaults: >>> str(sections['all.python']['test1']) 'val2' After assigning defaults: >>> sections['all.python'].set_default_section(sections) >>> str(sections['all.python']['test1']) 'val, val2' """ @staticmethod def __prepare_key(key): return str(key).lower().strip() def __init__(self, name, defaults=None): if defaults is not None and not isinstance(defaults, Section): raise TypeError('defaults has to be a Section object or None.') if defaults is self: raise ValueError('defaults may not be self for non-recursivity.') self.name = str(name) self.defaults = defaults self.contents = OrderedDict() self.aspects = None self.language = None def bear_dirs(self): bear_dirs = path_list(self.get('bear_dirs', '')) for bear_dir in bear_dirs: sys.path.append(bear_dir) bear_dir_globs = [ os.path.join(glob_escape(bear_dir), '**') for bear_dir in bear_dirs] bear_dir_globs += [ os.path.join(glob_escape(bear_dir), '**') for bear_dir in collect_registered_bears_dirs('coalabears')] return bear_dir_globs def is_enabled(self, targets): """ Checks if this section is enabled or, if targets is not empty, if it is included in the targets list. :param targets: List of target section names, all lower case. :return: True or False """ if len(targets) == 0: return bool(self.get('enabled', 'true')) return self.name.lower() in targets def append(self, setting, custom_key=None): if not isinstance(setting, Setting): raise TypeError if custom_key is None: key = self.__prepare_key(setting.key) else: key = self.__prepare_key(custom_key) # Setting asserts key != "" for us self.contents[key] = setting def add_or_create_setting(self, setting, custom_key=None, allow_appending=True): """ Adds the value of the setting to an existing setting if there is already a setting with the key. Otherwise creates a new setting. """ if custom_key is None: key = setting.key else: key = custom_key if self.__contains__(key, ignore_defaults=True) and allow_appending: val = self[key] val.value = str(val._value) + '\n' + setting._value self.append(val, custom_key=key) else: self.append(setting, custom_key=key) @enforce_signature def __setitem__(self, key: str, value: (str, Setting)): """ Creates a Setting object from the given value if needed and assigns the setting to the key: >>> section = Section('section_name') >>> section['key'] = 'value' >>> section['key'].value 'value' :param key: Argument whose value is to be set :param value: The value of the given key :return: Returns nothing. """ if isinstance(value, Setting): self.append(value, custom_key=key) else: # It must be a string since signature is enforced self.append(Setting(key, value)) def __iter__(self, ignore_defaults=False): joined = self.contents.copy() if self.defaults is not None and not ignore_defaults: # Since we only return the iterator of joined (which doesnt contain # values) it's ok to override values here joined.update(self.defaults.contents) return iter(joined) def __contains__(self, item, ignore_defaults=False): try: self.__getitem__(item, ignore_defaults) return True except IndexError: return False def __getitem__(self, item, ignore_defaults=False): key = self.__prepare_key(item) if key == '': raise IndexError('Empty keys are invalid.') res = copy.deepcopy(self.contents.get(key, None)) if res is not None: if res.to_append and self.defaults and res.key in self.defaults: res.value = self.defaults[key]._value + ', ' + res._value res.to_append = False return res if self.defaults is None or ignore_defaults: raise IndexError('Required index is unavailable.') return self.defaults[key] def __str__(self): value_list = ', '.join(key + ' : ' + repr(str(self[key])) for key in self.contents) return self.name + ' {' + value_list + '}' def get(self, key, default='', ignore_defaults=False): """ Retrieves the item without raising an exception. If the item is not available an appropriate Setting will be generated from your provided default value. :param key: The key of the setting to return. :param default: The default value :param ignore_defaults: Whether or not to ignore the default section. :return: The setting. """ try: return self.__getitem__(key, ignore_defaults) except IndexError: return Setting(key, str(default)) def copy(self): """ :return: a deep copy of this object """ result = copy.copy(self) result.contents = copy.deepcopy(self.contents) if self.defaults is not None: result.defaults = self.defaults.copy() return result def update(self, other_section, ignore_defaults=False): """ Incorporates all keys and values from the other section into this one. Values from the other section override the ones from this one. Default values from the other section override the default values from this only. :param other_section: Another Section :param ignore_defaults: If set to true, do not take default values from other :return: self """ if not isinstance(other_section, Section): raise TypeError('other_section has to be a Section') self.contents.update(other_section.contents) if not ignore_defaults and other_section.defaults is not None: if self.defaults is None: self.defaults = other_section.defaults.copy() else: self.defaults.update(other_section.defaults) return self def update_setting(self, key, new_key=None, new_value=None): """ Updates a setting with new values. :param key: The old key string. :param new_key: The new key string. :param new_value: The new value for the setting """ if new_key is not None: self.contents[key].key = new_key self.contents = update_ordered_dict_key(self.contents, key, new_key) if new_value is not None: if new_key is not None: self.contents[new_key].value = new_value else: self.contents[key].value = new_value def delete_setting(self, key): """ Delete a setting :param key: The key of the setting to be deleted """ del self.contents[key] def set_default_section(self, sections, section_name=None): """ Find and set the defaults of a section from a dictionary of sections. The defaults are found on the basis of '.' in section names: >>> sections = {'all': Section('all')} >>> section = Section('all.python') >>> section.set_default_section(sections) >>> section.defaults.name 'all' >>> section = Section('all.python.syntax') >>> section.set_default_section(sections) >>> section.defaults.name 'all' This works case insensitive. The key of the sections dict is expected to be lowered though! >>> sections = {'c': Section('C'), 'cpp': Section('Cpp'), ... 'c.something': Section('C.something')} >>> section = Section('C.something') >>> section.set_default_section(sections) >>> section.defaults.name 'C' >>> section = Section('C.SOMETHING.else') >>> section.set_default_section(sections) >>> section.defaults.name 'C.something' >>> section = Section('Cpp.SOMETHING.else') >>> section.set_default_section(sections) >>> section.defaults.name 'Cpp' :param sections: A dictionary of sections. :param section_name: Optional section name argument to find the default section for. If not given then use member section name. """ default_section = '.'.join( (section_name or self.name).split('.')[:-1] ).lower() if default_section: if default_section in sections: self.defaults = sections[default_section] else: self.set_default_section(sections, default_section) elif 'cli' in sections and self.name.lower() != 'cli': # CLI section is now default self.defaults = sections['cli']
class PolygonPore(object): def __init__(self, poly, name="protein", **params): self.name = name self.protein = Polygon(poly) self.polygons = OrderedDict({name: self.protein}) self.balls = OrderedDict() self.params = Params(params) self.boundaries = OrderedDict() # cs ... protein crosssections for partition of boundary self.add_proteinpartition() self.add_molecule_ball() def add_molecule_ball(self): if "x0" in self.params and self.params.x0 is not None: self.molecule = True lc = self.params.lcMolecule if "lcMolecule" in self.params else 1. molecule = Ball(self.params.x0, self.params.rMolecule, lc) else: self.molecule = False molecule = EmptySet() self.balls.update(molecule=molecule) def add_balls(self, **balls): for pname, p in balls.items(): if not isinstance(p, Ball): x0, r = p balls[pname] = Ball(x0, r) self.balls.update(balls) def build_polygons(self): R = self.params.R if not "Hbot" in self.params: H = self.params.H self.params["Htop"] = H / 2. self.params["Hbot"] = H / 2. Hbot = self.params.Hbot Htop = self.params.Htop self.add_membrane() sections = self.add_poresections() sections = self.add_poreregions(sections) self.add_bulkfluids(R, Htop, Hbot, sections) for ball in self.balls.values(): self.add_molecule(ball) # for easier handling of alls domains self.domains = self.balls.copy() self.domains.update(self.polygons) return self.polygons def build_boundaries(self): boundaries = self.boundaries # membrane boundary mem = self.polygons["membrane"] if not isempty(mem): memb = mem.edgerange(mem.b, mem.c) | mem.edgerange(mem.d, mem.a) boundaries.update(memb=memb) # upper, lower, side btop = self.polygons["bulkfluid_top"] bbot = self.polygons["bulkfluid_bottom"] upperb = btop.edgerange(btop.b, btop.c) lowerb = bbot.edgerange(bbot.d, bbot.a) sideb = btop.edgerange(btop.c, btop.d) | bbot.edgerange(bbot.c, bbot.d) boundaries.update(upperb=upperb, lowerb=lowerb, sideb=sideb) boundaries.update(self.compute_proteinboundary()) # molecule for bname, ball in self.balls.items(): boundaries.update({bname + "b": ball.boundary(self.params.dim)}) return boundaries def molecule_intersects(self, z): eps = 0.5 x0 = self.params.x0 if "x0" in self.params else None if x0 is None: return False z0 = x0[-1] r = self.params.rMolecule return z0 - r - eps <= z <= z0 + r + eps def where_is_molecule(self, ball=None): "name of domain where ball lies (or None)" # TODO: currently this is only based on z position!! if ball is None: ball = self.balls["molecule"] if isempty(ball): return None domains = ["pore%d" % i for i in range(self.nsections)] if self.params.poreregion: domains = ["poreregion_bottom"] + domains + ["poreregion_top"] else: domains = ["bulkfluid_bottom"] + domains + ["bulkfluid_top"] i0 = bisect(self.cs, ball.x0[-1]) return domains[i0] def add_molecule(self, ball): # do nothing if empty if isempty(ball): return # check in which domain ball is contained domstr = self.where_is_molecule(ball) domain = self.polygons[domstr] # in 3D, add ball as hole in containing domain if "dim" in self.params and self.params.dim == 3: if not hasattr(domain, "holes"): domain.holes = [] domain.holes.append(ball) return # in 2D, modify containing domain x1, x2, x3 = tuple(ball.nodes) domain.left_intersection(x1[1]) domain.left_intersection(x3[1]) i = domain.edges.index((x1, x3)) domain.edges[i] = (x1, x2, x3) def add_membrane(self): R = self.params.R if not ("no_membrane" in self.params and self.params.no_membrane) and (self.protein.rmax()[0] < R): hmem = self.params.hmem zmem = self.params.zmem zbot = zmem - 0.5 * hmem ztop = zmem + 0.5 * hmem self.membrane = self.protein.clip_from_right(zbot, ztop, R) else: self.membrane = EmptySet() self.polygons["membrane"] = self.membrane return self.membrane def add_poresections(self, remove_intersections=True): # cs ... crosssections cs = self.params.cs if "cs" in self.params else None ztop = max(x[1] for x in self.protein.nodes) zbot = min(x[1] for x in self.protein.nodes) if cs is None: cs = [zbot + i / 3. * (ztop - zbot) for i in [1, 2]] else: cs = list(cs) assert all(zbot < z < ztop for z in cs) cs = [zbot] + sorted(cs) + [ztop] # do not add crosssections that intersect with molecule if remove_intersections: for z in list(cs): if self.molecule_intersects(z): cs.remove(z) pairs = zip(cs[:-1], cs[1:]) sections = tuple( [self.protein.clip_from_left(a, b, 0) for a, b in pairs]) names = ["pore%d" % i for i in range(len(sections))] self.nsections = len(sections) self.lpore = ztop - zbot self.cs = cs self.polygons.update({name: s for name, s in zip(names, sections)}) return sections def add_poreregions(self, sections): if not "poreregion" in self.params or not self.params.poreregion: self.params.poreregion = False self.polygons["poreregion_top"] = EmptySet() self.polygons["poreregion_bottom"] = EmptySet() return sections if not "H0" in self.params or self.params.H0 is None: self.params["H0"] = 0.5 * self.params.H + self.protein.zmax()[1] if not "R0" in self.params or self.params.R0 is None: self.params["R0"] = self.protein.rmax()[0] H0 = self.params.H0 R0 = self.params.R0 H0top = H0bot = H0 / 2. #print self.protein.zmax()[1], H0top, self.params.H*0.5 center = MultiPolygon(self.protein, self.membrane, *sections) prtop = center.clip_from("top", R0, 0, H0top) prbot = center.clip_from("bottom", 0, R0, -H0bot) # OLD VERSION: doesn't work bc pore region and memb can overlap # section = sections[-1] # a = section.b # d = self.protein.top_intersection(R0) # upper = self.protein.nrange(d, section.c, -1) + [a] # b, c = (0, H0top), (R0, H0top) # nodes = [b, c] + upper # prtop = Polygon(nodes) # prtop.set_corners(a, b, c, d) # # section = sections[0] # b = section.a # c = self.protein.bottom_intersection(R0) # lower = [b] + self.protein.nrange(section.d, c, -1) # a, d = (0, -H0bot), (R0, -H0bot) # nodes = lower + [d, a] # prbot = Polygon(nodes) # prbot.set_corners(a, b, c, d) self.polygons["poreregion_top"] = prtop self.polygons["poreregion_bottom"] = prbot sections = [prbot] + list(sections) + [prtop] return sections def add_bulkfluids(self, R, Htop, Hbot, sections): # sections ... pore sections that remain after adding molecule polygons = list(sections) + [self.protein, self.membrane] upper = compute_upper_boundary(*polygons) b, c = (0, Htop), (R, Htop) nodes = [b, c] + upper btop = Polygon(nodes) btop.a = upper[-1] btop.b = b btop.c = c btop.d = upper[0] lower = compute_lower_boundary(*polygons) d, a = (R, -Hbot), (0, -Hbot) nodes = lower + [d, a] bbot = Polygon(nodes) bbot.a = a bbot.b = lower[0] bbot.c = lower[-1] bbot.d = d self.polygons["bulkfluid_top"] = btop self.polygons["bulkfluid_bottom"] = bbot return btop, bbot def add_proteinpartition(self): # do at the beginning # insert intersection points with cs in polygon cs = self.params.proteincs if "proteincs" in self.params else None if cs is None or len(cs) == 0: self.proteinpartition = False return protein = self.protein ztop = max(x[1] for x in protein.nodes) zbot = min(x[1] for x in protein.nodes) assert all(zbot < z < ztop for z in cs) for z in cs: protein.all_intersections(z) self.proteincs = cs self.proteinpartition = True def compute_proteinboundary(self): # do after modifying edges!! protein = self.protein if not self.proteinpartition: dic = {"%sb" % self.name: set(self.protein.edges)} else: cs = self.proteincs ztop = max(x[1] for x in protein.nodes) zbot = min(x[1] for x in protein.nodes) cs = [zbot] + sorted(cs) + [ztop] npart = len(cs) - 1 sets = [set() for i in range(npart)] # partition edges for edge in protein.edges: x = min(edge, key=lambda t: t[1]) ix = bisect(cs, x[1]) - 1 if ix == npart: ix -= 1 sets[ix].add(edge) dic = {"%sb%d" % (self.name, i): sets[i] for i in range(npart)} # subtract edges on protein-membrane interface mem = self.polygons["membrane"] if not isempty(mem): pmemb = protein.edgerange(mem.b, mem.a) for s in dic.values(): s -= pmemb return dic def radius_at(self, z): "return z-dependent pore radius" I = self.protein.intersections(z) return min(x[0] for x in I.keys())
class MultiPolygonPore(PolygonPore): """For pores consisting of multiple subdomains, e.g. Wei-Rant pore, but without further pore bisection (proteinpartition). Assumes that polygons are disjoint and their union simply connected.""" def __init__(self, **params): self.polygons = OrderedDict() self.balls = OrderedDict() self.params = Params(params) self.boundaries = OrderedDict() self.names = [] self.add_molecule_ball() def add_polygons(self, **polygons): "enables adding polygons in defined order" for pname, p in polygons.items(): if not isinstance(p, Polygon): polygons[pname] = Polygon(p) self.polygons.update(polygons) self.names.extend(polygons.keys()) def build_polygons(self): # read global height, width params R = self.params.R if not "Hbot" in self.params: H = self.params.H self.params["Htop"] = H / 2. self.params["Hbot"] = H / 2. Hbot = self.params.Hbot Htop = self.params.Htop # first build encompassing polygon out of existing. self.protein = MultiPolygon(*self.polygons.values()) self.protein.cut_from_right(R) self.polygons = OrderedDict() bnames = ["%sb" % name for name in self.names] self.proteinb = OrderedDict(zip(bnames, self.protein.boundaries)) self.add_membrane() sections = self.add_poresections() sections = self.add_poreregions(sections) self.add_bulkfluids(R, Htop, Hbot, sections) self.polygons.update(zip(self.names, self.protein.polygons)) for ball in self.balls.values(): self.add_molecule(ball) # for easier handling of alls domains self.domains = self.balls.copy() self.domains.update(self.polygons) return self.polygons def build_boundaries(self): boundaries = self.boundaries proteinb = self.proteinb # upper, lower, side btop = self.polygons["bulkfluid_top"] bbot = self.polygons["bulkfluid_bottom"] upperb = btop.edgerange(btop.b, btop.c) lowerb = bbot.edgerange(bbot.d, bbot.a) sideb = btop.edgerange(btop.c, btop.d) | bbot.edgerange(bbot.c, bbot.d) boundaries.update(upperb=upperb, lowerb=lowerb, sideb=sideb) fluid_polys = [btop, bbot] + \ [self.polygons["poreregion_top"], self.polygons["poreregion_bottom"]] + \ [self.polygons["pore%d" % i] for i in range(self.nsections)] fluid_edges = set([e[::-1] for p in fluid_polys for e in p.edges]) # membrane-fluid boundary mem = self.polygons["membrane"] if not isempty(mem): memb = fluid_edges & (mem.edgerange(mem.b, mem.c) | mem.edgerange(mem.d, mem.a)) boundaries.update(memb=memb) else: boundaries.update(memb=set()) # protein-fluid boundary, divided between protein parts for bname in proteinb: proteinb[bname] &= fluid_edges boundaries.update(proteinb) # molecule for bname, ball in self.balls.items(): boundaries.update({bname + "b": ball.boundary(self.params.dim)}) return boundaries
class MoveScaleOverlay(Overlay): """Viewer overlays for GES.VideoSource transformations.""" def __init__(self, stack, action_log, source): Overlay.__init__(self, stack, source) self.__clicked_handle = None self.__click_diagonal_sign = None self.__box_hovered = False self.__action_log = action_log self.hovered_handle = None # Corner handles need to be ordered for drawing. self.corner_handles = OrderedDict([ ((Edge.top, Edge.left), CornerHandle(self)), ((Edge.bottom, Edge.left), CornerHandle(self)), ((Edge.bottom, Edge.right), CornerHandle(self)), ((Edge.top, Edge.right), CornerHandle(self)) ]) self.handles = self.corner_handles.copy() for edge in range(1, 5): self.handles[(edge, )] = EdgeHandle(self) for key in self.handles: self.handles[key].set_placement(key) self._source.connect("deep-notify", self.__source_property_changed_cb) self.update_from_source() def __get_source_property(self, prop): if self.__source_property_keyframed(prop): binding = self._source.get_control_binding(prop) res, position = self.__get_pipeline_position() if res: start = self._source.props.start in_point = self._source.props.in_point duration = self._source.props.duration # If the position is outside of the clip, take the property # value at the start/end (whichever is closer) of the clip. source_position = max(0, min(position - start, duration - 1)) + in_point value = binding.get_value(source_position) res = value is not None return res, value return self._source.get_child_property(prop) def __set_source_property(self, prop, value): if self.__source_property_keyframed(prop): control_source = self._source.get_control_binding( prop).props.control_source res, timestamp = self.__get_pipeline_position() if not res: return source_timestamp = timestamp - self._source.props.start + self._source.props.in_point control_source.set(source_timestamp, value) else: self._source.set_child_property(prop, value) def __source_property_keyframed(self, prop): binding = self._source.get_control_binding(prop) return binding is not None def __get_pipeline_position(self): project = self.stack.app.project_manager.current_project if project: pipeline = project.pipeline try: position = pipeline.getPosition() return True, position except PipelineError: pass return False, None def __get_source_position(self): res_x, x = self.__get_source_property("posx") res_y, y = self.__get_source_property("posy") assert res_x and res_y return numpy.array([x, y]) def __get_source_size(self): res_x, x = self.__get_source_property("width") res_y, y = self.__get_source_property("height") assert res_x and res_y return numpy.array([x, y]) def __get_normalized_source_position(self): return self.__get_source_position() / self.project_size def __set_source_position(self, position): self.__set_source_property("posx", int(position[0])) self.__set_source_property("posy", int(position[1])) def __set_source_size(self, size): self.__set_source_property("width", int(size[0])) self.__set_source_property("height", int(size[1])) def __get_size(self): return numpy.array([self.__get_width(), self.__get_height()]) def __get_size_stream(self): return self.__get_size() * self.project_size def __get_height(self): return self.handles[(Edge.bottom, Edge.left)].position[1] - self.handles[ (Edge.top, Edge.left)].position[1] def __get_width(self): return self.handles[(Edge.bottom, Edge.right)].position[0] - self.handles[ (Edge.bottom, Edge.left)].position[0] def __set_size(self, size): self.handles[(Edge.top, Edge.left)].position = numpy.array([0, 0]) self.handles[(Edge.bottom, Edge.left)].position = numpy.array([0, size[1]]) self.handles[(Edge.bottom, Edge.right)].position = numpy.array([size[0], size[1]]) self.handles[(Edge.top, Edge.right)].position = numpy.array([size[0], 0]) self.__update_edges_from_corners() def __set_position(self, position): for handle in self.handles.values(): handle.set_translation(position) def __reset_handle_sizes(self): for handle in self.handles.values(): handle.reset_size() self.__update_handle_sizes() def __update_handle_sizes(self): size = self.__get_size() * self.stack.window_size smaller_size = numpy.amin(size) for handle in self.handles.values(): handle.restrict_radius_to_size(smaller_size) def __update_edges_from_corners(self): half_w = numpy.array([self.__get_width() * 0.5, 0]) half_h = numpy.array([0, self.__get_height() * 0.5]) self.handles[(Edge.left, )].set_position(self.handles[ (Edge.top, Edge.left)].position + half_h) self.handles[(Edge.right, )].set_position(self.handles[ (Edge.top, Edge.right)].position + half_h) self.handles[(Edge.bottom, )].set_position(self.handles[ (Edge.bottom, Edge.left)].position + half_w) self.handles[(Edge.top, )].set_position(self.handles[ (Edge.top, Edge.right)].position - half_w) def __draw_rectangle(self, cr): for handle in self.corner_handles.values(): cr.line_to(*handle.get_window_position()) cr.line_to(*self.handles[(Edge.top, Edge.left)].get_window_position()) def get_center(self): diagonal = self.handles[(Edge.bottom, Edge.right)].position - self.handles[ (Edge.top, Edge.left)].position return self.handles[(Edge.top, Edge.left)].position + (diagonal / 2) def get_aspect_ratio(self): size = self.__get_size() return size[0] / size[1] def on_button_press(self): disconnectAllByFunc(self._source, self.__source_property_changed_cb) self.click_source_position = self.__get_source_position() self.__clicked_handle = None if self.hovered_handle or self.__box_hovered: self.__action_log.begin( "Video position change", finalizing_action=CommitTimelineFinalizingAction( self._source.get_timeline().get_parent()), toplevel=True) if self.hovered_handle: self.hovered_handle.on_click() self.__clicked_handle = self.hovered_handle elif self.__box_hovered: self._select() self.stack.set_cursor("grabbing") self.stack.selected_overlay = self elif self._is_selected(): self._deselect() self.hovered_handle = None def on_button_release(self, cursor_position): self.click_source_position = None self.update_from_source() self.on_hover(cursor_position) self.__action_log.commit("Video position change") if self.__clicked_handle: if not self.__clicked_handle.hovered: self.stack.reset_cursor() self.__clicked_handle.on_release() self.__clicked_handle = None elif self._is_hovered(): self.stack.set_cursor("grab") self._source.connect("deep-notify", self.__source_property_changed_cb) self.queue_draw() def __source_property_changed_cb(self, unused_source, unused_gstelement, unused_pspec): self.update_from_source() def on_motion_notify(self, cursor_pos): click_to_cursor = self.stack.get_normalized_drag_distance(cursor_pos) if self.__clicked_handle: # Resize Box / Use Handle self.__clicked_handle.on_drag(click_to_cursor) self.__update_edges_from_corners() # We only need to change translation coordinates in the source for resizing # when handle does not return NULL for get_source_position source_position = self.__clicked_handle.get_source_position() if isinstance(source_position, numpy.ndarray): self.__set_source_position(source_position) self.__set_source_size(self.__get_size_stream()) self.__update_handle_sizes() else: # Move Box stream_position = self.click_source_position + click_to_cursor * self.project_size self.__set_position(stream_position / self.project_size) self.__set_source_position(stream_position) self.queue_draw() self._commit() def on_hover(self, cursor_pos): if not self.is_visible(): return # handles hover check self.hovered_handle = None if self._is_selected(): for handle in self.handles.values(): handle.on_hover(cursor_pos) if handle.hovered: self.hovered_handle = handle if self.hovered_handle: self._hover() self.queue_draw() return True # box hover check source = self.__get_normalized_source_position() cursor = self.stack.get_normalized_cursor_position(cursor_pos) self.__box_hovered = False if (source < cursor).all() and (cursor < source + self.__get_size()).all(): self.__box_hovered = True self.stack.set_cursor("grab") self._hover() else: self.__box_hovered = False self.unhover() self.queue_draw() return self.__box_hovered def update_from_source(self): self.__set_size(self.__get_source_size() / self.project_size) self.__set_position(self.__get_source_position() / self.project_size) self.__reset_handle_sizes() self.queue_draw() def do_draw(self, cr): if not self._is_selected() and not self._is_hovered(): return cr.save() # clear background cr.set_operator(cairo.OPERATOR_OVER) cr.set_source_rgba(0.0, 0.0, 0.0, 0.0) cr.paint() if self.__box_hovered: brightness = 0.65 else: brightness = 0.3 # clip away outer mask self.__draw_rectangle(cr) cr.clip() cr.set_source_rgba(brightness, brightness, brightness, 0.6) self.__draw_rectangle(cr) cr.set_line_width(16) cr.stroke() cr.restore() if self._is_selected(): for handle in self.handles.values(): handle.draw(cr)
class DRSPath(object): """ Handler providing methods to deal with paths. """ # Default DRS tree version # Modified on the ProcessingContext instance TREE_VERSION = 'v{}'.format(datetime.now().strftime('%Y%m%d')) def __init__(self, parts): # Retrieve the dataset directory parts self.d_parts = OrderedDict( parts.items()[:parts.keys().index('version')]) # Retrieve the file directory parts self.f_parts = OrderedDict( parts.items()[parts.keys().index('version') + 1:]) # Retrieve the upgrade version self.v_upgrade = DRSPath.TREE_VERSION # If the dataset path is not equivalent to the file diretcory (e.g., CMIP5 like) # Get the physical files version self.v_latest = self.get_latest_version() if self.path(f_part=False) != self.path(): self.v_files = OrderedDict({ 'variable': '{}_{}'.format(self.get('variable'), self.v_upgrade[1:]) }) else: self.v_files = OrderedDict( {'data': 'd{}'.format(self.v_upgrade[1:])}) def get(self, key): """ Returns the attribute value corresponding to the key. The submitted key can refer to the DRS dataset parts of the DRS file parts. :param str key: The key :returns: The value :rtype: *str* or *list* or *dict* depending on the key :raises Error: If unknown key """ if key in self.d_parts: return self.d_parts[key] elif key in self.f_parts.keys(): return self.f_parts[key] else: raise KeyNotFound(key, self.d_parts.keys() + self.f_parts.keys()) def items(self, d_part=True, f_part=True, version=True, file_folder=False, latest=False, root=False): """ Itemizes the facet values along the DRS path. Flags can be combine to obtain different behaviors. :param boolean d_part: True to append the dataset facets :param boolean f_part: True to append the file facets :param boolean version: True to append the version facet :param boolean file_folder: True to append the folder for physical files :param boolean latest: True to switch from upgrade to latest version :param boolean root: True to prepend the DRS root directory :return: The corresponding facet values :rtype: *list* """ parts = OrderedDict() if d_part: parts = self.d_parts.copy() if version: if latest: parts.update(OrderedDict({'version': self.v_latest})) else: parts.update(OrderedDict({'version': self.v_upgrade})) if file_folder: parts.update(OrderedDict({'version': 'files'})) if f_part: parts.update(self.f_parts) if file_folder: parts.update(self.v_files) if not root and 'root' in parts.keys(): del parts['root'] return parts.values() def path(self, **kwargs): """ Convert a list of facet values into path. The arguments are the same as :func:`esgprep.drs.handler.DRSPath.items` :returns: The path :rtype: *str* """ return os.path.join(*self.items(**kwargs)) def get_latest_version(self): """ Get the current latest dataset version if exists. :returns: The latest dataset version properly formatted :rtype: *str* :raises Error: If latest version exists and is the same as upgrade version """ # Test if dataset path already exists dset_path = self.path(f_part=False, version=False, root=True) if os.path.isdir(dset_path): # Get and sort all existing dataset versions versions = sorted([ v for v in os.listdir(dset_path) if re.compile(r'v[\d]+').search(v) ]) # Upgrade version should not already exist if self.v_upgrade in versions: raise DuplicatedDataset(dset_path, self.v_upgrade) # Pickup respectively previous, next and latest version return versions[-1] else: return None
def replace(self, replacements): """Replace certain variables in the computation graph. Parameters ---------- replacements : dict The mapping from variables to be replaced to the corresponding substitutes. Examples -------- >>> import theano >>> from theano import tensor, function >>> x = tensor.scalar('x') >>> y = x + 2 >>> z = y + 3 >>> a = z + 5 Let's suppose we have dependent replacements like >>> replacements = {y: x * 2, z: y * 3} >>> cg = ComputationGraph([a]) >>> theano.pprint(a) # doctest: +NORMALIZE_WHITESPACE '(((x + TensorConstant{2}) + TensorConstant{3}) + TensorConstant{5})' >>> cg_new = cg.replace(replacements) >>> theano.pprint( ... cg_new.outputs[0]) # doctest: +NORMALIZE_WHITESPACE '(((x * TensorConstant{2}) * TensorConstant{3}) + TensorConstant{5})' First two sums turned into multiplications >>> float(function(cg_new.inputs, cg_new.outputs)(3.)[0]) 23.0 """ # Due to theano specifics we have to make one replacement in time replacements = OrderedDict(replacements) outputs_cur = self.outputs # `replacements` with previous replacements applied. We have to track # variables in the new graph corresponding to original replacements. replacement_keys_cur = [] replacement_vals_cur = [] # Sort `replacements` in topological order # variables in self.variables are in topological order remaining_replacements = replacements.copy() for variable in self.variables: if variable in replacements: if has_roles(variable, [AUXILIARY]): warnings.warn( "replace method was asked to replace a variable ({}) " "that is an auxiliary variable.".format(variable)) replacement_keys_cur.append(variable) # self.variables should not contain duplicates, # otherwise pop() may fail. replacement_vals_cur.append( remaining_replacements.pop(variable)) # if remaining_replacements is not empty if remaining_replacements: warnings.warn( "replace method was asked to replace a variable(s) ({}) " "that is not a part of the computational " "graph.".format(str(remaining_replacements.keys()))) # Replace step-by-step in topological order while replacement_keys_cur: replace_what = replacement_keys_cur[0] replace_by = replacement_vals_cur[0] # We also want to make changes in future replacements outputs_new = theano.clone(outputs_cur + replacement_keys_cur[1:] + replacement_vals_cur[1:], replace={replace_what: replace_by}) # Reconstruct outputs, keys, and values outputs_cur = outputs_new[:len(outputs_cur)] replacement_keys_cur = outputs_new[len(outputs_cur ):len(outputs_cur) + len(replacement_keys_cur) - 1] replacement_vals_cur = outputs_new[len(outputs_cur) + len(replacement_keys_cur):] return ComputationGraph(outputs_cur)
class NamedElementStack(Mapping): def __init__(self, init_elements, valid_element=callable): self._queue = OrderedDict() for element in reversed(init_elements): if valid_element(element): self.add(element) else: self.add(*element) def add(self, element, name=None): if name is None: name = element if name in self._queue: if name is element: raise ValueError( "You can't add the same un-named instance twice") else: raise ValueError( "You can't add the same name again, use replace instead") self._queue[name] = element def clear(self): self._queue.clear() def replace(self, old, new): if old not in self._queue: raise ValueError( "You can't replace unless one already exists, use add instead") to_be_replaced = self._queue[old] if to_be_replaced is old: # re-insert with new name in old slot self._replace_with_new_name(old, new) else: self._queue[old] = new return to_be_replaced def remove(self, old): if old not in self._queue: raise ValueError( "You can only remove something that has been added") del self._queue[old] def _replace_with_new_name(self, old, new): self._queue[new] = new found_old = False for key in list(self._queue.keys()): if not found_old: if key == old: found_old = True continue elif key != new: self._queue.move_to_end(key) del self._queue[old] def __iter__(self): elements = self._queue.values() if not isinstance(elements, Sequence): elements = list(elements) return iter(reversed(elements)) def __add__(self, other): if not isinstance(other, NamedElementStack): raise NotImplementedError( "You can only combine with another Stack") combined = self._queue.copy() combined.update(other._queue) return NamedElementStack(combined.items()) def __contains__(self, element): return element in self._queue def __getitem__(self, element): return self._queue[element] def __len__(self): return len(self._queue) def __reversed__(self): elements = self._queue.values() if not isinstance(elements, Sequence): elements = list(elements) return iter(elements)
class AnsibleConfig(object): def __init__( self, ansible_file, playbook, log_dir=None, environmental_config=None, remote_user=None, extra_vars=[]): self._ansible_file = ansible_file self._playbook = playbook self._env = OrderedDict() self._log_dir = log_dir self._remote_user = remote_user self._extra_vars = extra_vars self._env['ANSIBLE_CONFIG'] = self._ansible_file if remote_user: self._env['ANSIBLE_REMOTE_USER'] = self._remote_user if environmental_config: self._env.update(self._environment_config_to_dict(environmental_config)) logger.debug('ansible_file: {f}'.format(f=self.ansible_file)) logger.debug('playbook: {pb}'.format(pb=self.playbook)) logger.debug('remoteuser: {ru}'.format(ru=self.remote_user)) logger.debug('environmental_config: {envcfg}'.format(envcfg=environmental_config)) logger.debug('env_str: {env_str}'.format(env_str=self.env_str)) logger.debug('extra_vars: {ev}'.format(ev=self.extra_vars)) @property def env(self): return self._env.copy() @property def env_str(self): return ' '.join(['{k}={v}'.format(k=k, v=v) for k, v in self._env.items()]) @property def ansible_file(self): return self._ansible_file @property def playbook(self): return self._playbook @property def remote_user(self): return self._remote_user @property def extra_vars(self): return self._extra_vars @property def log_dir(self): return self._log_dir @property def log_path(self): return self._env.get('ANSIBLE_LOG_PATH', None) @staticmethod def _environment_config_to_dict(configs): result = OrderedDict() for item in configs: try: key, value = item.split('=') result[key] = value except ValueError: raise Exception( "The ansible configuration '{item}' is mall-formated".format(item=item)) return result
class NicosCmdClient(NicosClient): def __init__(self, conndata): NicosClient.__init__(self, self.put_error) # connection data as an object self.conndata = conndata # whether to suppress printing history and other info on connection self.quiet_connect = False # various state variables self.in_question = False self.in_editing = False self.tip_shown = False # number of automatic reconnect tries before giving up self.reconnect_count = 0 self.reconnect_time = 0 # current script, line within it and filename of script self.current_script = [''] self.current_line = -1 self.current_filename = '' # pending requests (i.e. scripts) in the daemon self.pending_requests = OrderedDict() # filename of last edited/simulated script self.last_filename = '' # instrument name from NICOS, pre-filled with server name self.instrument = conndata.host.split('.')[0] # script directory from NICOS self.scriptpath = '.' # execution mode of the NICOS session self.current_mode = MASTER # messages queueing up while the editor is running self.message_queue = [] # whether a stop is pending self.stop_pending = False # whether we are in debugging mode self.debug_mode = False # whether we are in spy mode (entering commands disabled) self.spy_mode = False # detected text-mode browser for help display self.browser = None # used for determining how much history to print by default self.tsize = terminalSize() # output stream to print to self.out = sys.stdout # uuid of the last simulation self.simuuid = '' # whether we display timestamps with subsecond precision self.subsec_ts = False # current ETA information self.cur_eta = '' # set up readline for line in DEFAULT_BINDINGS.splitlines(): readline.parse_and_bind(line) readline.set_completer(self.completer) readline.set_history_length(10000) self.histfile = os.environ.get('NICOS_HISTORY_FILE', path.expanduser('~/.nicoshistory')) if path.isfile(self.histfile): readline.read_history_file(self.histfile) self.completions = [] # set up "wakeup" pipe to notify readline of output and changed prompt self.wakeup_pipe_r, self.wakeup_pipe_w = os.pipe() # pre-set prompt to sane default self.set_status('disconnected') # -- low-level terminal input/output routines def readline(self, prompt, add_history=True): """Read a line from the user. This function basically reimplements the readline module's readline_until_enter_or_signal C function, with the addition that we set new prompts and update the display periodically. Thanks to ctypes this is possible without a custom C module. """ # pylint: disable=global-statement global readline_result term_encoding = sys.stdout.encoding or 'utf-8' librl.rl_callback_handler_install(to_encoding(prompt, term_encoding), c_readline_finish_callback) readline_result = Ellipsis while readline_result is Ellipsis: try: res = select.select([sys.stdin, self.wakeup_pipe_r], [], [], 1)[0] except select.error as e: if e.args[0] == errno.EINTR: continue librl.rl_callback_handler_remove() raise except: librl.rl_callback_handler_remove() raise if sys.stdin in res: librl.rl_callback_read_char() if self.wakeup_pipe_r in res: os.read(self.wakeup_pipe_r, 1) if not self.in_question: # question has an alternate prompt that never changes librl.rl_set_prompt(to_encoding(self.prompt, term_encoding)) librl.rl_forced_update_display() if readline_result: # add to history, but only if requested and not the same as the # previous history entry if add_history and readline.get_history_item( readline.get_current_history_length()) != readline_result: librl.add_history(readline_result) elif readline_result is None: raise EOFError elif readline_result is False: raise StateChange return readline_result.decode(term_encoding, 'ignore') def put(self, string): """Put a line of output, preserving the prompt afterwards.""" self.out.write('\r\x1b[K%s\n' % string) self.out.flush() os.write(self.wakeup_pipe_w, b' ') def put_error(self, string): """Put a client error message.""" self.put(colorize('red', '# ERROR: ' + string)) def put_client(self, string): """Put a client info message.""" self.put(colorize('bold', '# ' + string)) def ask_passwd(self, question): """Prompt user for a password.""" return getpass.getpass(colorize('bold', '# %s ' % question)) def ask_question(self, question, chars='', default='', on_intr=''): """Prompt user for input to a question.""" # add hints of what can be entered if chars: question += ' (%s)' % ('/'.join(chars.upper())) if default: question += ' [%s]' % default self.in_question = True try: try: # see set_status() for an explanation of the special chars here ans = self.readline('\x01\r\x1b[K' + colorize('bold', '\x02# ' + question + ' \x01') + '\x02', add_history=False) except (KeyboardInterrupt, EOFError): return on_intr if chars: # we accept any string; if it's beginning with one of the chars, # that is the result, otherwise it's the default value ans = ans.lower() for char in chars: if ans.startswith(char): return char return default if not ans: ans = default return ans finally: self.in_question = False def ask_input(self, question): """Prompt user for a line of input.""" self.in_question = True try: try: # see set_status() for an explanation of the special chars here ans = self.readline('\x01\r\x1b[K' + colorize('bold', '\x02# ' + question + ' \x01') + '\x02', add_history=False) except (KeyboardInterrupt, EOFError): return '' return ans finally: self.in_question = False # -- event (signal) handlers def initial_update(self): """Called after connection is established.""" # request current full status state = self.ask('getstatus') if state is None: return if not self.quiet_connect: self.put_client( 'Connected to %s:%s as %s. ' 'Replaying output (enter "/log" to see more)...' % (self.host, self.port, self.conndata.user)) output = self.ask('getmessages', str(self.tsize[1] - 3), default=[]) for msg in output: self.put_message(msg) if not self.tip_shown: self.put_client('Loaded setups: %s. Enter "/help" for help ' 'with the client commands.' % (', '.join(state['setups'][1]) or '(none)')) self.tip_shown = True else: self.put_client('Loaded setups: %s.' % (', '.join(state['setups'][1]) or '(none)')) else: self.put_client('Connected to %s:%s as %s. ' % (self.host, self.port, self.conndata.user)) self.signal('processing', {'script': state['script'], 'reqid': '0'}) self.signal('status', state['status']) self.signal('eta', state['eta']) self.current_mode = state['mode'] self.scriptpath = self.eval('session.experiment.scriptpath', '.') self.instrument = self.eval('session.instrument.instrument', self.instrument) for req in state['requests']: self.pending_requests[req['reqid']] = req self.set_status(self.status) stcolmap = {'idle': 'blue', 'running': 'fuchsia', 'paused': 'red', 'disconnected': 'darkgray'} modemap = {MASTER: '', SLAVE: 'slave,', SIMULATION: 'simmode,', MAINTENANCE: 'maintenance,'} def set_status(self, status): """Update the current execution status, and set a new prompt.""" self.status = status if self.stop_pending: pending = ' (stop pending)' elif self.pending_requests: pending = ' (%d pending)' % len(self.pending_requests) else: pending = '' # \x01/\x02 are markers recognized by readline as "here come" # zero-width control characters; ESC[K means "clear whole line" self.prompt = '\x01' + colorize( self.stcolmap[status], '\r\x1b[K\x02# ' + (self.instrument or '') + '[%s%s]%s %s \x01' % (self.modemap[self.current_mode], status, pending, self.spy_mode and 'spy>' or '>>')) + '\x02' os.write(self.wakeup_pipe_w, b' ') def showhelp(self, html): """Handles the "showhelp" signal. Uses html2text to get nicely-formatted text from the html. """ htmlconv = HTML2Text() htmlconv.ignore_links = True self.put(htmlconv.handle(html)) def handle_eta(self, data): """Handles the "eta" signal.""" state, eta = data if state in (SIM_STATES['pending'], SIM_STATES['running']): self.cur_eta = '<calculation in progress>' elif state == SIM_STATES['failed']: self.cur_eta = '<calculation failed>' elif state == SIM_STATES['success'] and eta > currenttime(): self.cur_eta = formatEndtime(eta - currenttime()) def put_message(self, msg, sim=False): """Handles the "message" signal.""" if msg[0] == 'nicos': namefmt = '' else: namefmt = '%-10s: ' % msg[0] levelno = msg[2] if levelno == ACTION: action = namefmt + msg[3].rstrip() self.out.write('\x1b]0;NICOS%s\x07' % (action and ' (%s)' % action or '')) return else: if self.subsec_ts: timesuf = '.%.06d] ' % ((msg[1] % 1) * 1000000) else: timesuf = '] ' if levelno <= DEBUG: timefmt = strftime('[%H:%M:%S', localtime(msg[1])) newtext = colorize('lightgray', timefmt + timesuf) + \ colorize('darkgray', namefmt + msg[3].rstrip()) elif levelno <= INFO: timefmt = strftime('[%H:%M:%S', localtime(msg[1])) newtext = colorize('lightgray', timefmt + timesuf) + \ namefmt + msg[3].rstrip() elif levelno == INPUT: newtext = colorize('darkgreen', msg[3].rstrip()) elif levelno <= WARNING: timefmt = strftime('[%Y-%m-%d %H:%M:%S', localtime(msg[1])) newtext = colorize('purple', timefmt + timesuf + namefmt + levels[levelno] + ': ' + msg[3].rstrip()) else: timefmt = strftime('[%Y-%m-%d %H:%M:%S', localtime(msg[1])) newtext = colorize('red', timefmt + timesuf + namefmt + levels[levelno] + ': ' + msg[3].rstrip()) if sim: newtext = '(sim) ' + newtext self.put(newtext) # pylint: disable=W0221 def signal(self, name, data=None, exc=None): """Handles any kind of signal/event sent by the daemon.""" try: # try to order the elifs by frequency if name == 'message': if self.in_editing: self.message_queue.append(data) else: self.put_message(data) elif name == 'status': status, line = data # pylint:disable=W0633 if status == STATUS_IDLE or status == STATUS_IDLEEXC: new_status = 'idle' self.stop_pending = False elif status != STATUS_INBREAK: new_status = 'running' else: new_status = 'paused' if status != self.status: self.set_status(new_status) if line != self.current_line: self.current_line = line elif name == 'cache': if data[1].endswith('/scriptpath'): self.scriptpath = self.eval( 'session.experiment.scriptpath', '.') elif name == 'processing': script = data.get('script') if script is None: return self.current_filename = data.get('name') or '' script = script.splitlines() or [''] if script != self.current_script: self.current_script = script self.pending_requests.pop(data['reqid'], None) self.set_status(self.status) elif name == 'request': if 'script' in data: self.pending_requests[data['reqid']] = data self.set_status(self.status) elif name == 'blocked': removed = [_f for _f in (self.pending_requests.pop(reqid, None) for reqid in data) if _f] if removed: self.put_client('%d script(s) or command(s) removed from ' 'queue.' % len(removed)) self.show_pending() self.set_status(self.status) elif name == 'updated': if 'script' in data: self.pending_requests[data['reqid']] = data elif name == 'rearranged': old_reqs = self.pending_requests.copy() self.pending_requests.clear() for reqid in data: self.pending_requests[reqid] = old_reqs[reqid] elif name == 'connected': self.reconnect_count = 0 self.initial_update() elif name == 'disconnected': self.put_client('Disconnected from server, use /reconnect to ' 'try reconnecting.') self.current_mode = MASTER self.debug_mode = False self.pending_requests.clear() self.set_status('disconnected') elif name == 'showhelp': self.showhelp(data[1]) elif name == 'simmessage': if data[5] in [self.simuuid, '0']: if not self.in_editing: self.put_message(data, sim=True) elif name == 'simresult': if data and data[2] in [self.simuuid, '0']: timing, devinfo, _ = data # pylint: disable=W0633 if timing < 0: self.put_client('Dry run resulted in an error.') return self.put_client('Simulated minimum runtime: %s ' '(finishes approximately %s). Device ranges:' % (formatDuration(timing, precise=False), formatEndtime(timing))) if devinfo: dnwidth = max(map(len, devinfo)) sorteditems = sorted(iteritems(devinfo), key=lambda x: x[0].lower()) for devname, (_, dmin, dmax, aliases) in sorteditems: aliascol = 'aliases: ' + ', '.join(aliases) if aliases else '' self.put('# %-*s: %10s <-> %-10s %s' % (dnwidth, devname, dmin, dmax, aliascol)) elif name == 'mode': self.current_mode = data self.set_status(self.status) elif name == 'setup': self.scriptpath = self.eval('session.experiment.scriptpath', '.') self.instrument = self.eval('session.instrument.instrument', self.instrument) elif name == 'debugging': self.debug_mode = data readline_finish_callback(False) elif name == 'plugplay': if data[0] == 'added': self.put_client('new sample environment detected: load ' 'setup %r to activate' % data[1]) elif data[0] == 'removed': self.put_client('sample environment removed: unload ' 'setup %r to clear devices' % data[1]) elif name == 'eta': self.handle_eta(data) elif name == 'broken': self.put_error(data) self.reconnect_count = self.RECONNECT_TRIES self.reconnect_time = self.RECONNECT_INTERVAL_SHORT self.schedule_reconnect() elif name == 'failed': if self.reconnect_count: self.schedule_reconnect() else: self.put_error(data) elif name == 'error': self.put_error(data) # and we ignore all other signals except Exception as e: self.put_error('In %s event handler: %s.' % (name, e)) # -- reconnect handling def schedule_reconnect(self): def reconnect(): if self.reconnect_count: self.connect(self.conndata, eventmask=EVENTMASK) self.reconnect_count -= 1 if self.reconnect_count <= self.RECONNECT_TRIES_LONG: self.reconnect_time = self.RECONNECT_INTERVAL_LONG threading.Timer(self.reconnect_time / 1000., reconnect).start() # -- command handlers def ask_connect(self, ask_all=True): hostport = '%s:%s' % (self.conndata.host, self.conndata.port) if hostport in (':', ':1301') or ask_all: default = '' if hostport in (':', ':1301') else hostport default = default or 'localhost' server = self.ask_question('Server host:port?', default=default) if not server: return try: host, port = server.split(':', 1) port = int(port) except ValueError: host = server port = DEFAULT_PORT self.conndata.host = host self.conndata.port = port if not self.conndata.user or ask_all: user = self.ask_question('User name?', default=self.conndata.user or 'guest') self.conndata.user = user if self.conndata.password is None or ask_all: password = self.ask_passwd('Password?') self.conndata.password = password self.instrument = self.conndata.host.split('.')[0] try: self.connect(self.conndata, eventmask=EVENTMASK) except RuntimeError as err: self.put_error('Cannot connect: %s.' % err) def help(self, arg): """Implements the "/help" command.""" if not arg: arg = 'main' if arg not in HELP: arg = 'main' helptext = HELP[arg] for line in helptext.splitlines(): self.put('# ' + line) def edit_file(self, arg): """Implements the "/edit" command.""" if not arg: if path.isfile(self.current_filename): arg = self.current_filename if not arg: self.put_error('Need a file name as argument.') return fpath = path.join(self.scriptpath, path.expanduser(arg)) if not os.environ.get('EDITOR'): os.environ['EDITOR'] = 'vi' self.in_editing = True cwd = os.getcwd() if path.isdir(self.scriptpath): os.chdir(self.scriptpath) try: ret = os.system('$EDITOR "' + fpath + '"') finally: os.chdir(cwd) self.in_editing = False for msg in self.message_queue: self.put_message(msg) self.message_queue = [] if ret != 0 or not path.isfile(fpath): return # if the editor exited successfully (and the file exists) we try to be # smart about offering the user a choice of running, simulating or # updating the current script self.last_filename = fpath if self.status == 'running': if fpath == self.current_filename: # current script edited: most likely we want to update it if self.ask_question('Update running script?', chars='yn', default='n') == 'y': return self.command('update', fpath) else: # another script edited: updating will likely fail reply = self.ask_question('Queue or dry-run file?', chars='qdn') if reply == 'q': # this will automatically queue return self.command('run!', fpath) elif reply == 'd': return self.command('sim', fpath) else: # no script is running at the moment: offer to run it reply = self.ask_question('Run or dry-run file?', chars='rdn') if reply == 'r': return self.command('run', fpath) elif reply == 'd': return self.command('sim', fpath) def print_where(self): """Implements the "/where" command.""" if self.status in ('running', 'paused'): self.put_client('Printing current script.') for i, line in enumerate(self.current_script): if i+1 == self.current_line: self.put(colorize('darkgreen', '---> ' + line)) else: self.put(' ' + line) self.put_client('End of script.') if self.cur_eta: self.put_client('Estimated finishing time: ' + self.cur_eta) else: self.put_client('No script is running.') def _iter_pending(self): for reqid, script in iteritems(self.pending_requests): if 'name' in script and script['name']: short = script['name'] elif 'script' in script: lines = script['script'].splitlines() if len(lines) == 1: short = lines[0] else: short = lines[0] + ' ...' else: short = '(stop)' yield reqid, short def show_pending(self): if not self.pending_requests: self.put_client('No scripts or commands are pending.') return self.put_client('Showing pending scripts or commands. ' 'Use "/cancel" to remove one or more.') for _reqid, short in self._iter_pending(): self.put('# %s' % short) self.put_client('End of pending list.') def cancel_menu(self, arg): if not self.pending_requests: self.put_client('No scripts or commands are pending.') return if arg == '*': self.tell('unqueue', '*') return self.put_client('Showing pending scripts or commands.') indices = {} for index, (reqid, short) in enumerate(self._iter_pending(), start=1): indices[index] = reqid self.put('# %s %s' % (colorize('blue', '%2d' % index), short)) res = self.ask_question('Which script to cancel ("*" for all)?') if res == '*': self.tell('unqueue', '*') return try: reqid = indices[int(res)] except (ValueError, KeyError): self.put_error('Invalid selection.') return self.tell('unqueue', reqid) def debug_repl(self): """Called to handle remote debugging via Rpdb.""" self.in_question = True # suppress prompt changes try: while self.debug_mode: try: cmd = self.readline('\x01\r\x1b[K' + colorize('darkred', '\x02# (Rpdb) \x01') + '\x02') + '\n' except (EOFError, KeyboardInterrupt): cmd = '' except StateChange: if not self.debug_mode: return self.tell('debuginput', cmd) finally: self.in_question = False def plot_data(self, xterm_mode): try: xs, ys, _, names = self.eval( '__import__("nicos").commands.analyze._getData()[:4]') plotlines = txtplot(xs, ys, names[0], names[1], xterm_mode) except Exception as err: self.put_error('Could not plot: %s.' % str(err)) else: for line in plotlines: self.put(line) def stop_query(self, how): """Called on Ctrl-C (if running) or when "/stop" is entered.""" self.put_client('== %s ==' % how) self.put('# Please enter how to proceed:') self.put('# <I> ignore this interrupt') self.put('# <H> stop after current scan point') self.put('# <L> stop after current command') self.put('# <S> immediate stop') res = self.ask_question('Your choice?', chars='ihls').upper() if res == 'I': return elif res == 'H': # this is basically "stop at any well-defined breakpoint" self.tell('stop', BREAK_AFTER_STEP) self.stop_pending = True self.set_status(self.status) elif res == 'L': # this is "everywhere after a command in the script" self.tell('stop', BREAK_AFTER_LINE) self.stop_pending = True self.set_status(self.status) else: self.tell('emergency') def command(self, cmd, arg): """Called when a "/foo" command is entered at the prompt.""" # try to order elif cases by frequency if cmd in ('cmd', 'exec'): if cmd == 'cmd' and self.spy_mode: return self.command('eval', arg) # this is not usually entered as "/cmd foo", but only "foo" if self.status in ('running', 'paused'): reply = self.ask_question('A script is already running, ' 'queue or execute anyway?', chars='qxn') if reply == 'x': if self.status != 'idle': self.tell('exec', arg) else: self.run(arg) elif reply == 'q': self.run(arg) self.put_client('Command queued.') else: self.run(arg) elif cmd in ('r', 'run', 'run!'): if not arg: # since we remember the last edited file, we can offer # running it here if self.last_filename: reply = self.ask_question('Run last used file %r?' % path.basename(self.last_filename), chars='yn', default='y') if reply == 'y': self.command('run', self.last_filename) return self.put_error('Need a file name as argument.') return fpath = path.join(self.scriptpath, path.expanduser(arg)) try: code = open(fpath).read() except Exception as e: self.put_error('Unable to open file: %s.' % e) return if self.status in ('running', 'paused') and cmd != 'run!': if self.ask_question('A script is already running, ' 'queue script?', chars='yn', default='y') == 'y': self.run(code, fpath) else: self.run(code, fpath) elif cmd == 'update': if not arg: # always take the current filename, if it still exists if path.isfile(self.current_filename): arg = self.current_filename if not arg: self.put_error('Need a file name as argument.') return fpath = path.join(self.scriptpath, path.expanduser(arg)) try: code = open(fpath).read() except Exception as e: self.put_error('Unable to open file: %s.' % e) return reason = self.ask_input('Reason for updating:') self.tell('update', code, reason) elif cmd in ('sim', 'simulate'): if not arg: self.put_error('Need a file name or code as argument.') return fpath = path.join(self.scriptpath, path.expanduser(arg)) self.last_filename = fpath # detect whether we have a filename or potential Python code if path.isfile(fpath) or fpath.endswith(('.py', '.txt')): try: code = open(fpath).read() except Exception as e: self.put_error('Unable to open file: %s.' % e) return self.simulate(fpath, code) else: self.simulate('', arg) elif cmd in ('e', 'edit'): self.edit_file(arg) elif cmd == 'break': self.tell('break', BREAK_AFTER_STEP) elif cmd in ('cont', 'continue'): self.tell('continue') elif cmd in ('s', 'stop'): if self.status == 'running': self.stop_query('Stop request') else: self.tell('emergency') elif cmd in ('fin', 'finish'): self.tell('finish') elif cmd == 'pending': self.show_pending() elif cmd == 'cancel': self.cancel_menu(arg) elif cmd == 'disconnect': if self.isconnected: self.disconnect() elif cmd == 'connect': self.reconnect_count = 0 if self.isconnected: self.put_error('Already connected. Use /disconnect first.') else: self.ask_connect() elif cmd in ('re', 'reconnect'): self.reconnect_count = 0 # no automatic reconnect if self.isconnected: self.disconnect() self.ask_connect(ask_all=False) elif cmd in ('q', 'quit'): if self.isconnected: self.disconnect() return 0 # i.e. exit with success elif cmd in ('h', 'help', '?'): self.help(arg) elif cmd == 'log': if arg: n = str(int(arg)) # make sure it's an integer else: n = '*' # as a slice index, this means "unlimited" # this can take a while to transfer, but we don't want to cache # messages in this client just for this command messages = self.ask('getmessages', n) if messages is None: return self.put_client('Printing %s previous messages.' % (n if n != '*' else 'all')) for msg in messages: self.put_message(msg) self.put_client('End of messages.') elif cmd in ('w', 'where'): self.print_where() elif cmd == 'wait': if arg: time.sleep(float(arg)) else: # this command is mainly meant for testing and scripting purposes time.sleep(0.1) while self.status != 'idle': time.sleep(0.1) elif cmd == 'trace': trace = self.ask('gettrace') if trace is None: return self.put_client('Current stacktrace of script execution:') for line in trace.splitlines(): if line: self.put('# ' + line) self.put_client('End of stacktrace.') elif cmd == 'debugclient': import pdb pdb.set_trace() elif cmd == 'debug': self.tell('debug', arg) elif cmd == 'eval': timefmt = colorize('lightgray', strftime('[%H:%M:%S]')) self.put('%s -> %s' % (timefmt, self.eval(arg, None, stringify=True))) elif cmd == 'spy': if not self.spy_mode: self.put_client('Spy mode on: normal input is evaluated as ' 'an expression, use /exec to execute as script.') else: self.put_client('Spy mode off.') self.spy_mode = not self.spy_mode self.set_status(self.status) elif cmd == 'plot': self.plot_data(xterm_mode=(arg == 'x')) elif cmd == 'subsec': self.subsec_ts = not self.subsec_ts else: self.put_error('Unknown command %r.' % cmd) # -- command-line completion support def simulate(self, fpath, code): self.simuuid = str(uuid1()) self.tell('simulate', fpath, code, self.simuuid) def complete_filename(self, fn, word): """Try to complete a script filename.""" # script filenames are relative to the current scriptpath; nevertheless # the user can override this by giving an absolute path to the script initpath = path.join(self.scriptpath, fn) candidates = [] # omit the part already on the line, but not what readline considers the # current "word" omit = len(initpath) - len(word) # complete directories and .py/.txt script files for f in glob.glob(initpath + '*'): if path.isdir(f): candidates.append(f[omit:] + '/') elif path.isfile(f) and f.endswith(('.py', '.txt')): candidates.append(f[omit:]) return candidates commands = ['run', 'simulate', 'edit', 'update', 'break', 'continue', 'stop', 'where', 'disconnect', 'connect', 'reconnect', 'quit', 'help', 'log', 'pending', 'cancel', 'eval', 'spy', 'debug', 'trace', 'subsec', 'plot'] def completer(self, text, state): """Try to complete the command line. Called by readline.""" if state == 0: # we got a a new bit of text to complete... line = readline.get_line_buffer() # handle line without command if not line.startswith('/'): line = '/exec ' + line # split into command and arguments parts = line[1:].split(None, 1) if len(parts) < 2 and not line.endswith(' '): # complete client command names self.completions = [cmd for cmd in self.commands if cmd.startswith(text)] elif parts[0] in ('r', 'run', 'e', 'edit', 'update', 'sim', 'simulate'): # complete filenames try: fn = parts[1] except IndexError: fn = '' self.completions = self.complete_filename(fn, text) elif parts[0] in ('eval', 'exec'): # complete code -- ask the server to complete for us try: self.completions = self.ask('complete', text, parts[1], default=[]) except Exception: self.completions = [] else: # no completions for other commands self.completions = [] try: return self.completions[state] except IndexError: return None # -- main loop def handle(self, cmd): """Handle a command line.""" # dispatch either as a client command... if cmd.startswith('/'): args = cmd[1:].split(None, 1) + ['', ''] return self.command(args[0], args[1]) elif cmd: # or as "normal" Python code to execute return self.command('cmd', cmd) # an empty line is ignored def main(self): """Connect and then run the main read-send-print loop.""" try: self.command('reconnect', '') while 1: if self.debug_mode: self.debug_repl() try: cmd = self.readline(self.prompt) except KeyboardInterrupt: # offer the user a choice of ways of stopping if self.status == 'running' and not self.spy_mode: self.stop_query('Keyboard interrupt') continue except EOFError: self.command('quit', '') return 0 except StateChange: continue ret = self.handle(cmd) if ret is not None: return ret finally: try: readline.write_history_file(self.histfile) except IOError: pass def main_with_command(self, command): self.quiet_connect = True self.command('reconnect', '') self.handle(command) time.sleep(0.1) while self.status != 'idle': time.sleep(0.1) self.command('quit', '') return 0
class CallSequence(object): """@brief Call sequence manager. Contains an ordered sequence of tasks. Each task has a name and associated callable. The CallSequence class itself is callable, so instances can be nested as tasks within other CallSequences. When tasks within a sequence are called, they may optionally return a new CallSequence instance. If this happens, the new sequence is executed right away, before continuing with the next task in the original sequence. A CallSequence can be iterated over. It will return tuples of (task-name, callable). """ def __init__(self, *args): """@brief Constructor. The constructor accepts an arbitrary number of parameters describing an ordered set of tasks. Each parameter must be a 2-tuple with the first element being the task's name and the second element a callable that implements the task. If you need to pass parameters to the callable, use a lambda. """ self._validate_tasks(args) self._calls = OrderedDict(args) def _validate_tasks(self, tasks): for i in tasks: assert len(i) == 2 assert type(i[0]) is str assert isinstance(i[1], Callable) @property def sequence(self): """@brief Returns an OrderedDict of the call sequence. Task names are keys. """ return self._calls @sequence.setter def sequence(self, seq): """@brief Replace the entire call sequence. Accepts either an OrderedDict or a list of 2-tuples like the constructor. """ if isinstance(seq, OrderedDict): self._calls = seq elif type(seq) is list and len(seq) and type(seq[0]) is tuple: self._calls = OrderedDict(seq) @property def count(self): """@brief Returns the number of tasks in the sequence.""" return len(self._calls) def clear(self): """@brief Remove all tasks from the sequence.""" self._calls = OrderedDict() def copy(self): """@brief Duplicate the sequence.""" new_seq = CallSequence() new_seq._calls = self._calls.copy() return new_seq def remove_task(self, name): """@brief Remove a task with the given name. @exception KeyError Raised if no task with the specified name exists. """ del self._calls[name] return self def has_task(self, name): """@brief Returns a boolean indicating presence of the named task in the sequence.""" return name in self._calls def get_task(self, name): """@brief Return the callable for the named task. @exception KeyError Raised if no task with the specified name exists. """ return self._calls[name] def replace_task(self, name, replacement): """@brief Change the callable associated with a task.""" assert isinstance(replacement, Callable) if name not in self._calls: raise KeyError(name) else: # OrderedDict preserves the order when changing the value of a key # that is already in the dict. self._calls[name] = replacement return self def wrap_task(self, name, wrapper): """@brief Wrap an existing task with a new callable. The wrapper is expected to take a single parameter, the return value from the original task. This allows for easy filtering of a new call sequence returned by the original task. """ if name not in self._calls: raise KeyError(name) # Get original callable. orig = self._calls[name] # OrderedDict preserves the order when changing the value of a key # that is already in the dict. self._calls[name] = lambda: wrapper(orig()) return self def append(self, *args): """@brief Append a new task or tasks to the sequence. Like the constructor, this method takes any number of arguments. Each must be a 2-tuple task description. """ self._validate_tasks(args) # Insert iterable. self._calls.update(args) return self def insert_before(self, beforeTaskName, *args): """@brief Insert a task or tasks before a named task. @param beforeTaskName The name of an existing task. The new tasks will be inserted prior to this task. After the task name parameter, any number of task description 2-tuples may be passed. @exception KeyError Raised if the named task does not exist in the sequence. """ self._validate_tasks(args) if not self.has_task(beforeTaskName): raise KeyError(beforeTaskName) seq = list(self._calls.items()) for i, v in enumerate(seq): if v[0] == beforeTaskName: for c in args: # List insert() inserts before the given index. seq.insert(i, c) i += 1 break self._calls = OrderedDict(seq) return self def insert_after(self, afterTaskName, *args): """@brief Insert a task or tasks after a named task. @param afterTaskName The name of an existing task. The new tasks will be inserted after this task. After the task name parameter, any number of task description 2-tuples may be passed. @exception KeyError Raised if the named task does not exist in the sequence. """ self._validate_tasks(args) if not self.has_task(afterTaskName): raise KeyError(afterTaskName) seq = list(self._calls.items()) for i, v in enumerate(seq): if v[0] == afterTaskName: for c in args: # List insert() inserts before the given index. seq.insert(i + 1, c) i += 1 break self._calls = OrderedDict(seq) return self def invoke(self): """@brief Execute each task in order. A task may return a CallSequence, in which case the new sequence is immediately executed. """ for name, call in self._calls.items(): LOG.debug("Running task %s", name) resultSequence = call() # Invoke returned call sequence. if resultSequence is not None and isinstance( resultSequence, CallSequence): # LOG.debug("Invoking returned call sequence: %s", resultSequence) resultSequence.invoke() def __call__(self, *args, **kwargs): """@brief Another way to execute the tasks. Supports nested CallSequences. """ self.invoke() def __iter__(self): """@brief Iterate over the sequence.""" return iter(self._calls.items()) def __repr__(self): s = "<%s@%x: " % (self.__class__.__name__, id(self)) for name, task in self._calls.items(): s += "\n%s: %s" % (name, task) s += ">" return s