def _generate_image_metadata(self): # First - read old image config, we'll update it instead of # generating one from scratch metadata = OrderedDict(self.old_image_config) # Update image creation date metadata['created'] = self.date # Remove unnecessary or old fields metadata.pop("container", None) # Remove squashed layers from history metadata['history'] = metadata['history'][:len(self.layers_to_move)] # Remove diff_ids for squashed layers metadata['rootfs']['diff_ids'] = metadata['rootfs'][ 'diff_ids'][:len(self.layer_paths_to_move)] history = {'comment': '', 'created': self.date} if self.layer_paths_to_squash: # Add diff_ids for the squashed layer metadata['rootfs']['diff_ids'].append( "sha256:%s" % self.diff_ids[-1]) else: history['empty_layer'] = True # Add new entry for squashed layer to history metadata['history'].append(history) if self.squash_id: # Update image id, should be one layer below squashed layer metadata['config']['Image'] = self.squash_id else: metadata['config']['Image'] = "" return metadata
class LRUCache(object): def __init__(self, capacity): """ :type capacity: int """ self.dictionary = OrderedDict() self.capacity = capacity def get(self, key): """ :rtype: int """ if key in self.dictionary: temp = self.dictionary.pop(key) self.dictionary[key] = temp return temp else: return -1 def set(self, key, value): """ :type key: int :type value: int :rtype: nothing """ if key in self.dictionary: self.dictionary.pop(key)() self.dictionary[key] = value else: if len(self.dictionary) >= self.capacity: self.dictionary.popitem(last=False) self.dictionary[key] = value else: self.dictionary[key] = value
class Cache(object): def __init__(self): self.reset_caches() def reset_caches(self): self._category_cache = OrderedDict() self._search_cache = OrderedDict() def search_cache(self, search): old = self._search_cache.pop(search, None) if old is None or old[0] <= self.db.last_modified(): matches = self.search_for_books(search) or [] self._search_cache[search] = (utcnow(), frozenset(matches)) if len(self._search_cache) > 50: self._search_cache.popitem(last=False) else: self._search_cache[search] = old return self._search_cache[search][1] def categories_cache(self, restrict_to=frozenset([])): base_restriction = self.search_cache('') if restrict_to: restrict_to = frozenset(restrict_to).intersection(base_restriction) else: restrict_to = base_restriction old = self._category_cache.pop(frozenset(restrict_to), None) if old is None or old[0] <= self.db.last_modified(): categories = self.db.get_categories(ids=restrict_to) self._category_cache[restrict_to] = (utcnow(), categories) if len(self._category_cache) > 20: self._category_cache.popitem(last=False) else: self._category_cache[frozenset(restrict_to)] = old return self._category_cache[restrict_to][1]
def trace(self, key, readby=False, strict=False): """ Return the sequence of operations required to compute the node ``key``. If ``readby = True``, then return the sequence of operations that will depend on ``key``, instead. With ``strict = True``, drop ``key`` from the result. """ if key not in self: return [] # OrderedDicts, besides preserving the scheduling order, also prevent # scheduling the same node more than once found = OrderedDict() queue = OrderedDict([(key, self[key])]) while queue: k, v = queue.popitem(last=False) reads = self.extract(k, readby=readby) if set(reads).issubset(set(found.values())): # All dependencies satisfied, schedulable found[k] = v else: # Tensors belong to other traces, so they can be scheduled straight away tensors = [i for i in reads if i.is_Tensor] found = OrderedDict(list(found.items()) + [(i.lhs, i) for i in tensors]) # Postpone the rest until all dependening nodes got scheduled scalars = [i for i in reads if i.is_Scalar] queue = OrderedDict([(i.lhs, i) for i in scalars] + [(k, v)] + list(queue.items())) if strict is True: found.pop(key) return tuple(found.values())
def find_unique_url_multi_dict(file): current_dict = OrderedDict() dicts = [current_dict] with open(file) as f: line_no = 0 for url in f: line_no += 1 if len(current_dict) > 100: current_dict = OrderedDict() dicts.append(current_dict) if url in current_dict: current_dict.pop(url) else: current_dict[url] = line_no while dicts: d = dicts.pop(0) for k, v in d.iteritems(): not_found = True for d1 in dicts: if k not in d1: continue else: not_found = False break if not_found: return k, v
def smallest_subarray_covering_set(A, Q): """Find smallest range (i,j) s.t. all q in Q are in A[i..j].""" # handle 0-length covering if not Q: return 0, -1 # start with no best and an empty candidate covering range Q = set(Q) # want O(1) membership test min_size = None min_covering_range = None cand_locs = OrderedDict() # stream elements of A, maintaining the shortest covering range # that ends at the current element for j, elem in enumerate(A): # skip elements that can't contribute to a covering if elem not in Q: continue # extend the candidate range with the current elem, # removing any previous instance of the same elem cand_locs.pop(elem, None) cand_locs[elem] = j # will be added in final position # if the new candidate is legal and better, make it the current best if len(cand_locs) == len(Q): i = cand_locs.itervalues().next() # get front position, O(1) time if min_size is None or j - i < min_size: min_size = j - i min_covering_range = i, j return min_covering_range
def __new__(mcs, name, bases, attrs): # Collect fields from current class. current_fields = [] for key, value in list(attrs.items()): if isinstance(value, Field): current_fields.append((key, value)) attrs.pop(key) current_fields.sort(key=lambda x: x[1].creation_counter) attrs["declared_fields"] = OrderedDict(current_fields) new_class = super(DeclarativeFieldsMetaclass, mcs).__new__(mcs, name, bases, attrs) # Walk through the MRO. declared_fields = OrderedDict() for base in reversed(new_class.__mro__): # Collect fields from base class. if hasattr(base, "declared_fields"): declared_fields.update(base.declared_fields) # Field shadowing. for attr in base.__dict__.keys(): if attr in declared_fields: declared_fields.pop(attr) new_class.base_fields = declared_fields new_class.declared_fields = declared_fields return new_class
def _topological_sort(self, goal_info_by_goal): dependees_by_goal = OrderedDict() def add_dependee(goal, dependee=None): dependees = dependees_by_goal.get(goal) if dependees is None: dependees = set() dependees_by_goal[goal] = dependees if dependee: dependees.add(dependee) for goal, goal_info in goal_info_by_goal.items(): add_dependee(goal) for dependency in goal_info.goal_dependencies: add_dependee(dependency, goal) satisfied = set() while dependees_by_goal: count = len(dependees_by_goal) for goal, dependees in dependees_by_goal.items(): unsatisfied = len(dependees - satisfied) if unsatisfied == 0: satisfied.add(goal) dependees_by_goal.pop(goal) yield goal_info_by_goal[goal] break if len(dependees_by_goal) == count: for dependees in dependees_by_goal.values(): dependees.difference_update(satisfied) # TODO(John Sirois): Do a better job here and actually collect and print cycle paths # between Goals/Tasks. The developer can most directly address that data. raise self.GoalCycleError('Cycle detected in goal dependencies:\n\t{0}' .format('\n\t'.join('{0} <- {1}'.format(goal, list(dependees)) for goal, dependees in dependees_by_goal.items())))
class Lineup(object): def __init__(self, lineup): self.lineup = lineup self.healthy_pokemon = OrderedDict() self.fainted_pokemon = OrderedDict() for pokemon in lineup: if pokemon.health > 0: self.healthy_pokemon[pokemon] = True else: self.fainted_pokemon[pokemon] = True def fainted(self, pokemon): if pokemon in self.healthy_pokemon: self.healthy_pokemon.pop(pokemon) self.fainted_pokemon[pokemon] = True def pokemon(self): return self.lineup def get_starters(self, size): starters = set() for pokemon in self.healthy_pokemon.keys(): starters.add(pokemon) if len(starters) == size: break return starters
class CompoundIndex: def __init__(self, field=[]): self.insidedoc = OrderedDict() if field: if isinstance(field, list): self.addListIndex(field) else: if field.startswith('_'): self.addDescIndex(field) else: self.addIndex(field) def addDescIndex(self, field): self.insidedoc.__setitem__(field, -1) def addIndex(self, field): self.insidedoc.__setitem__(field, 1) def addListIndex(self, list): for x in list: if x.startswith('_'): self.insidedoc.__setitem__(x.split('_', 1)[1], -1) else: self.insidedoc.__setitem__(x, 1) def removeIndex(self, field): self.insidedoc.pop(field) def __str__(self): return json.dumps(self.insidedoc)
def __new__(mcs, name, bases, attrs): # Collect transitions from current class. current_transitions = [] for key, value in list(attrs.items()): if isinstance(value, Transition): current_transitions.append((key, value)) attrs.pop(key) current_transitions.sort(key=lambda x: x[1].creation_counter) attrs['declared_transitions'] = OrderedDict(current_transitions) new_class = (super(DeclarativeTransitionsMetaclass, mcs).__new__(mcs, name, bases, attrs)) # Walk through the MRO. declared_transitions = OrderedDict() for base in reversed(new_class.__mro__): # Collect transitions from base class. if hasattr(base, 'declared_transitions'): declared_transitions.update(base.declared_transitions) # Field shadowing. for attr, value in base.__dict__.items(): if value is None and attr in declared_transitions: declared_transitions.pop(attr) new_class.declared_transitions = declared_transitions return new_class
def run(state, include=None, exclude=None): """ Run data validations for state. State is required. Optionally filter validations using include/exclude flags. """ if include and exclude: sys.exit("ERROR: You can not use both include and exclude flags!") state_mod = load_module(state, ['validate']) # Load all validations in order found validations = OrderedDict() for name in dir(state_mod.validate): if name.startswith('validate_'): func = getattr(state_mod.validate, name) validations[name] = func # Filter validations based in include/exclude flags if include: to_run = split_args(include) for val in validations: if val not in to_run: validations.pop(val) if exclude: to_skip = split_args(exclude) for val in validations: if val in to_skip: validations.pop(val) # Run remaining validations run_validation(state, list(validations.values()))
def parse_strand_chart_data (self): """ Make a data structure suitable for HighCharts for the strand alignment plot """ self.bismark_strand_samples = list() self.bismark_directional_mode = 0 series = OrderedDict() series['OB'] = list() series['CTOB'] = list() series['CTOT'] = list() series['OT'] = list() for sn in sorted(self.bismark_data.keys()): self.bismark_strand_samples.append(sn) series['OB'].append(int(self.bismark_data[sn].get('aln_strand_ob', 0))) series['CTOB'].append(int(self.bismark_data[sn].get('aln_strand_ctob', 0))) series['CTOT'].append(int(self.bismark_data[sn].get('aln_strand_ctot', 0))) series['OT'].append(int(self.bismark_data[sn].get('aln_strand_ot', 0))) if 'aln_strand_directional' in self.bismark_data[sn]: self.bismark_directional_mode += 1 if self.bismark_directional_mode == len(self.bismark_strand_samples): series.pop('CTOB', None) series.pop('CTOT', None) self.bismark_strand_plot_series = list() for cat in series: self.bismark_strand_plot_series.append({ 'name': cat, 'data': series[cat] })
class MissingPonies(object): def __init__(self, owned_ponies, inventory_ponies, all_ponies): owned_ponies_id = set(owned_ponies.keys()) | set(inventory_ponies.keys()) all_ponies_id = set(all_ponies.keys()) not_owned = all_ponies_id - owned_ponies_id self._ponies = OrderedDict([(ID, name) for ID, name in all_ponies.items() if ID in not_owned]) def items(self): return self._ponies.items() def values(self): return self._ponies.values() def keys(self): return self._ponies.keys() def __contains__(self, item): return item in self._ponies def __isub__(self, pony_id): self._ponies.pop(pony_id, None) return self def remove(self, pony_id): self -= pony_id
def parse_recipe(self, response, title, picture): hxs = HtmlXPathSelector(response) section = None sections = OrderedDict() for node in hxs.select(xp_ingredients): text = html_to_text(node.extract()).strip() if not text: continue if node.select('strong'): section = text continue sections.setdefault(section, []).append(text) ingredients = sections.pop(None, None) or sections.pop(sections.keys()[-1]) extra_ingredients = [x for y in sections.values() for x in y] yield CocktailItem( title=title, picture=picture, url=response.url, ingredients=ingredients, extra_ingredients=extra_ingredients )
def bismark_strand_chart (self): """ Make the strand alignment plot """ # Specify the order of the different possible categories keys = OrderedDict() keys['strand_ob'] = { 'name': 'Original bottom strand' } keys['strand_ctob'] = { 'name': 'Complementary to original bottom strand' } keys['strand_ctot'] = { 'name': 'Complementary to original top strand' } keys['strand_ot'] = { 'name': 'Original top strand' } # See if we have any directional samples directional = 0 d_mode = '' for sn in self.bismark_data['alignment'].values(): if 'strand_directional' in sn.keys(): directional += 1 if directional == len(self.bismark_data['alignment']): keys.pop('strand_ctob', None) keys.pop('strand_ctot', None) d_mode = '<p>All samples were run with <code>--directional</code> mode; alignments to complementary strands (CTOT, CTOB) were ignored.</p>' elif directional > 0: d_mode = '<p>{} samples were run with <code>--directional</code> mode; alignments to complementary strands (CTOT, CTOB) were ignored.</p>'.format(directional) # Config for the plot config = { 'id': 'bismark_strand_alignment', 'title': 'Alignment to Individual Bisulfite Strands', 'ylab': '% Reads', 'cpswitch_c_active': False, 'cpswitch_counts_label': 'Number of Reads' } return d_mode + plots.bargraph.plot(self.bismark_data['alignment'], keys, config)
def get_rack_units(self, face=RACK_FACE_FRONT, exclude=None, remove_redundant=False): """ Return a list of rack units as dictionaries. Example: {'device': None, 'face': 0, 'id': 48, 'name': 'U48'} Each key 'device' is either a Device or None. By default, multi-U devices are repeated for each U they occupy. :param face: Rack face (front or rear) :param exclude: PK of a Device to exclude (optional); helpful when relocating a Device within a Rack :param remove_redundant: If True, rack units occupied by a device already listed will be omitted """ elevation = OrderedDict() for u in reversed(range(1, self.u_height + 1)): elevation[u] = {'id': u, 'name': 'U{}'.format(u), 'face': face, 'device': None} # Add devices to rack units list if self.pk: for device in Device.objects.select_related('device_type__manufacturer', 'device_role')\ .annotate(devicebay_count=Count('device_bays'))\ .exclude(pk=exclude)\ .filter(rack=self, position__gt=0)\ .filter(Q(face=face) | Q(device_type__is_full_depth=True)): if remove_redundant: elevation[device.position]['device'] = device for u in range(device.position + 1, device.position + device.device_type.u_height): elevation.pop(u, None) else: for u in range(device.position, device.position + device.device_type.u_height): elevation[u]['device'] = device return [u for u in elevation.values()]
def get_round_results(self, competition, round, valid_session_rounds, scores): results = OrderedDict() for score in scores: session_entry = score.target.session_entry if session_entry.session_round.shot_round.id is not round.id: continue if session_entry.session_round not in valid_session_rounds: continue categories = self.get_categories_for_entry(competition, session_entry.competition_entry) for category in categories: if category not in results: results[category] = {} if session_entry.competition_entry not in results[category]: results[category][session_entry.competition_entry] = [] results[category][session_entry.competition_entry].append(score) for category, scores in results.items(): scores = OrderedDict((entry, rounds) for entry, rounds in scores.items() if len(rounds) >= 2) if not scores: results.pop(category) continue for entry in scores: scores[entry] = sorted(scores[entry], key=lambda s: s.target.session_entry.session_round.session.start)[:2] new_scores = [ScoreMock( disqualified=any(s.disqualified for s in sub_scores), retired=any(s.disqualified for s in sub_scores), target=sub_scores[0].target, score=sum(s.score for s in sub_scores), hits=sum(s.hits for s in sub_scores), golds=sum(s.golds for s in sub_scores), xs=sum(s.xs for s in sub_scores), ) for entry, sub_scores in scores.items()] if not self.leaderboard: new_scores = filter(lambda s: s.score > 0, new_scores) results[category] = self.sort_results(new_scores) return results
class LRUCache(object): def __init__(self, capacity): """ :type capacity: int """ self.c = capacity self.mapi = OrderedDict() def get(self, key): """ :rtype: int """ if key in self.mapi: ret = self.mapi.pop(key) self.mapi[key] = ret return ret return -1 def set(self, key, value): """ :type key: int :type value: int :rtype: nothing """ if key in self.mapi: self.mapi.pop(key) self.mapi[key] = value if len(self.mapi) > self.c: self.mapi.popitem(last=False)
def iet_make(stree): """Create an IET from a ScheduleTree.""" nsections = 0 queues = OrderedDict() for i in stree.visit(): if i == stree: # We hit this handle at the very end of the visit return List(body=queues.pop(i)) elif i.is_Exprs: exprs = [Increment(e) if e.is_Increment else Expression(e) for e in i.exprs] body = ExpressionBundle(i.shape, i.ops, i.traffic, body=exprs) elif i.is_Conditional: body = Conditional(i.guard, queues.pop(i)) elif i.is_Iteration: # Order to ensure deterministic code generation uindices = sorted(i.sub_iterators, key=lambda d: d.name) # Generate Iteration body = Iteration(queues.pop(i), i.dim, i.dim._limits, offsets=i.limits, direction=i.direction, uindices=uindices) elif i.is_Section: body = Section('section%d' % nsections, body=queues.pop(i)) nsections += 1 elif i.is_Halo: body = HaloSpot(i.halo_scheme, body=queues.pop(i)) queues.setdefault(i.parent, []).append(body) assert False
def convert_block(data): block_id = data.findtext('key') params = OrderedDict(sorted( (param.findtext('key'), param.findtext('value')) for param in data.findall('param') )) if block_id == "import": params["imports"] = params.pop("import") states = OrderedDict() x, y = ast.literal_eval(params.pop('_coordinate', '(10, 10)')) states['coordinate'] = yaml.ListFlowing([x, y]) states['rotation'] = int(params.pop('_rotation', '0')) enabled = params.pop('_enabled', 'True') states['state'] = ( 'enabled' if enabled in ('1', 'True') else 'bypassed' if enabled == '2' else 'disabled' ) block = OrderedDict() if block_id != 'options': block['name'] = params.pop('id') block['id'] = block_id block['parameters'] = params block['states'] = states return block
def longestConsecutiveSeq(self, num): seqDic = OrderedDict() for each in num: seqDic[each] = each maxlength = 0 for key in num: left = key - 1 right = key + 1 cnt = 1 while left in seqDic: cnt += 1 seqDic.pop(left) left -= 1 while right in seqDic: cnt += 1 seqDic.pop(right) right += 1 if cnt > maxlength: maxlength = cnt return maxlength
class LRUCache(OrderedDict): """ #实现一个LRU算法的dict,用于记录order_id #避免storm平台重发log记录,造成redis数据偏大! """ def __init__(self, capacity): """ @param capacity 字典长度 """ self.capacity = capacity self.cache = OrderedDict() def get(self,key): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: value = None return value def set(self,key,value): if self.cache.has_key(key): value = self.cache.pop(key) self.cache[key] = value else: if len(self.cache) == self.capacity: #pop出第一个item self.cache.popitem(last = False) self.cache[key] = value else: self.cache[key] = value
def __new__(mcs, name, bases, attrs): # Collect fields from current class. current_fields = [] for key, value in attrs.items(): if isinstance(value, BaseField): current_fields.append((key, value)) current_fields.sort(key=lambda x: x[1].creation_counter) attrs['declared_fields'] = OrderedDict(current_fields) new_class = (super(MetaField, mcs) .__new__(mcs, name, bases, attrs)) # Walk through the MRO. declared_fields = OrderedDict() for base in reversed(new_class.__mro__): # Collect fields from base class. if hasattr(base, 'declared_fields'): declared_fields.update(base.declared_fields) # Field shadowing. for attr, value in base.__dict__.items(): if value is None and attr in declared_fields: declared_fields.pop(attr) # Store these fields on the class. new_class.base_fields = declared_fields new_class.declared_fields = declared_fields return new_class
class LimitedMemoryBackend(BasicInterface, dc.api.CacheBackend): def __init__(self, argument_dict): '''If argument_dict contains a value for max_kbytes this the total memory limit in kByte that is enforced on the internal cache dictionary, otherwise it's set to sys.maxint. If argument_dict contains a value for max_keys this maximum amount of cache values kept in the internal cache dictionary, otherwise it's set to sys.maxlen. If necessary values are deleted from the cache in FIFO order. ''' self.logger.debug('LimitedMemoryBackend args {}'.format(pformat(argument_dict))) self._max_keys = argument_dict.get('max_keys', sys.maxsize) self._max_bytes = argument_dict.get('max_kbytes', sys.maxint / 1024) * 1024 self._cache = OrderedDict() def get(self, key): return self._cache.get(key, dc.api.NO_VALUE) def print_limit(self, additional_size=0): self.logger.info('LimitedMemoryBackend at {}({}) keys -- {}({}) Byte' .format(len(self._cache), self._max_keys, memory.getsizeof(self._cache) / 8, self._max_bytes)) def _enforce_limits(self, new_value): additional_size = memory.getsizeof(new_value) / 8 while len(self._cache) > 0 and not (len(self._cache) <= self._max_keys and (memory.getsizeof(self._cache) + additional_size) / 8 <= self._max_bytes): self.logger.debug('shrinking limited memory cache') self._cache.popitem(last=False) def set(self, key, value): self._enforce_limits(value) self._cache[key] = value def delete(self, key): self._cache.pop(key)
def to_representation(self, instance): """ Serialize objects -> primitives. """ ret = OrderedDict() fields = [field for field in self.fields.values() if not field.write_only] # geo structure if self.Meta.id_field is not False: ret["id"] = "" ret["type"] = "Feature" ret["geometry"] = {} ret["properties"] = OrderedDict() for field in fields: field_name = field.field_name if field.read_only and instance is None: continue value = field.to_representation(field.get_attribute(instance)) if self.Meta.id_field is not False and field_name == self.Meta.id_field: ret["id"] = value elif field_name == self.Meta.geo_field: ret["geometry"] = value elif not getattr(field, 'write_only', False): ret["properties"][field_name] = value if self.Meta.id_field is False: ret.pop(self.Meta.model._meta.pk.name) return ret
class WorkerQueue(object): def __init__(self): self.queue = OrderedDict() def ready(self, worker): self.queue.pop(worker.address, None) self.queue[worker.address] = worker def purge(self): """Look for & kill expired workers.""" t = time.time() expired = [] for address, worker in self.queue.items(): if t > worker.expiry: # Worker expired expired.append(address) for address in expired: self.queue.pop(address, None) def size(self): return len(self.queue.items()) def next(self): if len(self.queue.items()) == 0: return None address, worker = self.queue.popitem(False) return address
def extract_properties_from_manifest(manifest_src): '''return a dictionary with the properties of a task read from a task manifest if it exists''' if not any(key.lower() in manifest_src.lower() for key in MANIFEST_ALIASES_GROUPED['manifest_version']): raise VersionError('Missing Task Manifest Version, must be v0') props = ordered_load(manifest_src) if not isinstance(props, dict): raise TypeError('Docstring should be a yaml dictionary. Got %s instead.' % type(props).__name__) props = OrderedDict((MANIFEST_ALIASING.get(p.lower(), p.lower()), v) for p, v in props.items()) if 'manifest_version' not in props: # note that it MUST be present because we check for it in the entire string just above # probably this is broken yaml raise KeyError('Missing Task Manifest Version, must be present') if props['manifest_version'] != 1: raise VersionError('Unknown Task Manifest Version %s' % props['manifest_version']) props = parse_args_from_manifest(props) props.pop('manifest_version') return props
def _diff_trees(lhs, rhs, index, *path): def _fq(name): return "/".join(reversed((name,) + path)) # Diff the trees (and keep deterministic order) rhs_tree_ids = OrderedDict((o.name, o.id) for o in rhs.tree_ids) for o in lhs.tree_ids: rhs_id = rhs_tree_ids.pop(o.name, None) # remove so won't be picked up as added, below if rhs_id == o.id: # no change continue elif rhs_id is None: # removed yield (_fq(o.name), o.id, None) rhs_tree = Object(_id=None, tree_ids=[], blob_ids=[], other_ids=[]) else: # changed rhs_tree = index[rhs_id] for difference in _diff_trees(index[o.id], rhs_tree, index, o.name, *path): yield difference for name, id in rhs_tree_ids.items(): # added yield (_fq(name), None, id) lhs_tree = Object(_id=None, tree_ids=[], blob_ids=[], other_ids=[]) for difference in _diff_trees(lhs_tree, index[id], index, name, *path): yield difference # Diff the blobs (and keep deterministic order) rhs_blob_ids = OrderedDict((o.name, o.id) for o in rhs.blob_ids) for o in lhs.blob_ids: rhs_id = rhs_blob_ids.pop(o.name, None) if rhs_id == o.id: continue # no change elif rhs_id is None: yield (_fq(o.name), o.id, None) else: yield (_fq(o.name), o.id, rhs_id) for name, id in rhs_blob_ids.items(): yield (_fq(name), None, id)
class PointSpace(object): """Given a point will assign numeric IDs""" def __init__(self): self.points = OrderedDict() self.point_to_id = {} self._updated = True def _update(self): if self._updated: return current_id = 0 for point in self.points: self.point_to_id[point] = current_id current_id += 1 self._updated = True def add(self, point): self.points[point] = True self._updated = False def delete(self, point): self.points.pop(point, None) self._updated = False def get_id(self, point): self._update() return self.point_to_id.get(point)
class Watchdog(BaseCacheClient): """Main device for running the watchdog service.""" parameters = { 'watch': Param('The configuration of things to watch', type=listof(dictof(str, anytype))), 'mailreceiverkey': Param( 'Cache key that updates the receivers for ' 'any mail notifiers we have configured', type=str), # these would be adevs, but adevs don't allow this flexible # configuration with dictionaries 'notifiers': Param('Configures the notifiers for each warning type', type=dictof(str, listof(str))), } parameter_overrides = { 'prefix': Override(mandatory=False, default='nicos/'), } def doInit(self, mode): BaseCacheClient.doInit(self, mode) # cache of all interesting keys with current values self._keydict = LCDict() # put status constants in key dict to simplify status conditions for stval, stname in status.statuses.items(): self._keydict[stname.upper()] = stval # set to true during connect action self._process_updates = False # current setups self._setups = set() # mapping entry ids to entrys self._entries = {} # (mangled) key to update mail receivers self._mailreceiverkey = self.mailreceiverkey.replace('/', '_').lower() # mapping cache keys to entries that check this key self._keymap = {'session_mastersetup': set()} if self._mailreceiverkey: self._keymap[self._mailreceiverkey] = set() # current warnings: mapping entry ids to the string description self._warnings = OrderedDict() # current count loop pause reasons: mapping like self._warnings self._pausecount = OrderedDict() # create all notifier devices self._all_notifiers = [] self._notifiers = {'': []} for key, devnames in iteritems(self.notifiers): self._notifiers[key] = notiflist = [] for devname in devnames: dev = session.getDevice(devname, Notifier) notiflist.append(dev) self._all_notifiers.append(dev) # process entries in the default watchlist for entry_dict in self.watch: self._add_entry(entry_dict, 'watchdog') # start a thread checking for modification of the setup file createThread('refresh checker', self._checker) def _checker(self): setupname = session.explicit_setups[0] fn = session._setup_info[setupname]['filenames'] watchFileContent(fn, self.log) self.log.info('setup file changed; restarting watchdog process') os.execv(sys.executable, [sys.executable] + sys.argv) def _add_entry(self, entryd, source): entryd = dict(entryd) # convert back from readonlydict logprefix = 'entry %s from setup %r' % (entryd, source) # sanity-check the input if not entryd.get('condition'): self.log.warning('%s: missing "condition" key', logprefix) return if not entryd.get('message'): self.log.warning('%s: missing "message" key', logprefix) return if entryd.get('scriptaction') not in (None, 'pausecount', 'stop', 'immediatestop'): self.log.warning( '%s: scriptaction is invalid, needs ' "to be one of 'pausecount', 'stop' or " "'immediatestop'", logprefix) entryd.pop('scriptaction') entry = Entry(entryd) entry.from_setup = source if entry.id in self._entries: self.log.error('%s: duplicate entry, ignoring', logprefix) return if entry.type and entry.type not in self._notifiers: log_msg = ( '%s: the condition type %r is not valid, must be ' 'one of %r' % (logprefix, entry.type, ', '.join(map(repr, self._notifiers)))) if 'default' in self._notifiers: self.log.warning(log_msg + '; using default') entry.type = 'default' else: self.log.warning(log_msg + '; ignoring notifiers') entry.type = '' try: cond = Expression(self.log, entry.condition, entry.setup) if entry.gracetime: cond = DelayedTrigger(self.log, cond, entry.gracetime) if entry.precondition: precond = Expression(self.log, entry.precondition, entry.setup) if entry.precondtime: precond = DelayedTrigger(self.log, precond, entry.precondtime) cond = Precondition(self.log, precond, cond) if not entry.enabled: cond.enabled = False entry.cond_obj = cond except Exception: self.log.error( '%s: could not construct condition, ignoring ' 'this condition', logprefix) return for key in cond.interesting_keys(): self._keymap.setdefault(key, set()).add(entry) self._entries[entry.id] = entry def _remove_entry(self, eid): entry = self._entries.pop(eid, None) if entry: for key in entry.cond_obj.interesting_keys(): self._keymap[key].discard(entry) # cache client API def _connect_action(self): # inhibit direct processing of updates self._process_updates = False try: BaseCacheClient._connect_action(self) # now process all keys we got time = currenttime() for key in list(self._keydict): try: self._process_key(time, key, self._keydict[key]) except Exception: self.log.warning('error handling first update for key %s', key, exc=1) finally: self._process_updates = True self.storeSysInfo('watchdog') self._queue.put('watchdog/%s\n' % OP_SUBSCRIBE) self._publish_config() def _wait_data(self): t = currenttime() for entry in itervalues(self._entries): if entry.cond_obj.tick(t) or entry.cond_obj.is_expired(t): self._check_state(entry, t) def _handle_msg(self, time, ttlop, ttl, tsop, key, op, value): if key.startswith('watchdog/'): self._handle_control_msg(key, cache_load(value), time, op != OP_TELL) return key = key[len(self._prefix):].replace('/', '_').lower() # do we need the key for conditions? if key not in self._keymap: return expired = op == OP_TELLOLD or not value value = None if expired else cache_load(value) if not expired: self._keydict[key] = value else: self._keydict.pop(key, None) time = float(time) if self._process_updates: self._process_key(time, key, value) # internal watchdog API def _handle_control_msg(self, key, value, time, expired): if key == 'watchdog/enable': time = currenttime() for (eid, enabled) in value[1]: entry = self._entries.get(eid) if entry: entry.cond_obj.enabled = enabled entry.cond_obj.update(time, self._keydict) self._check_state(entry, time) self.log.info('updated enabled conditions by user request') elif key == 'watchdog/reset': # reset all condition enables to their initial state # (e.g. due to NewExperiment) for entry in itervalues(self._entries): if entry.enabled != entry.cond_obj.enabled: entry.cond_obj.enabled = entry.enabled entry.cond_obj.update(time, self._keydict) self._check_state(entry, time) self.log.info('enable status of all conditions reset') self._publish_config() def _publish_config(self): # publish current condition info in the cache self._put_message('configured', None, [c.serialize() for c in itervalues(self._entries)]) def _process_key(self, time, key, value): # check setups? if key == 'session_mastersetup' and value: self._setups_updated(time, set(value)) return # update notification targets? if key == self._mailreceiverkey and value: self._update_mailreceivers(value) return for entry in self._keymap.get(key, ()): entry.cond_obj.update(time, self._keydict) self._check_state(entry, time) def _setups_updated(self, time, new_setups): prev_setups, self._setups = self._setups, new_setups # check if we need to remove some conditions for entry in listvalues(self._entries): if entry.from_setup != 'watchdog': if entry.from_setup not in self._setups: self._remove_entry(entry.id) # check if we need to add some conditions session.readSetups() # refresh setup info for new_setup in self._setups - prev_setups: info = session._setup_info.get(new_setup) if info and info['watch_conditions']: self.log.info('adding conditions from setup %s', new_setup) for entry_dict in info['watch_conditions']: self._add_entry(entry_dict, new_setup) # trigger an update of all conditions for entry in itervalues(self._entries): entry.cond_obj.new_setups(self._setups) entry.cond_obj.update(time, self._keydict) self._check_state(entry, time) # update everyone els self._publish_config() self.log.info('new setups list: %s', ', '.join(self._setups)) def _check_state(self, entry, time): """Check if the state of this entry changed and we need to emit a warning or a clear it. """ eid = entry.id if entry.cond_obj.is_expired(time): if eid not in self._warnings or self._warnings[eid][0]: self._emit_expired_warning(entry) elif entry.cond_obj.triggered: if eid not in self._warnings or not self._warnings[eid][0]: self._emit_warning(entry) else: if eid in self._warnings: self._clear_warning(entry) self.log.debug('new conditions: %s', self._warnings) def _emit_warning(self, entry): """Emit a warning that this condition is now triggered.""" self.log.info('got a new warning for %r', entry) warning_desc = strftime('%Y-%m-%d %H:%M') + ' -- ' + entry.message if entry.action: warning_desc += ' -- executing %r' % entry.action if entry.scriptaction == 'pausecount': self._pausecount[entry.id] = entry.message self._update_pausecount_str() warning_desc += ' -- counting paused' elif entry.scriptaction: self._put_message('scriptaction', entry, (entry.scriptaction, entry.message)) if entry.type: for notifier in self._notifiers[entry.type]: notifier.send('New warning from NICOS', warning_desc) self._put_message('warning', entry, entry.message) self._warnings[entry.id] = (True, warning_desc) self._update_warnings_str() if entry.action: self._put_message('action', entry, entry.action) self._spawn_action(entry.action) def _emit_expired_warning(self, entry): """Emit a warning that this condition has missing information.""" self.log.info('value(s) missing for %r', entry) warning_desc = (strftime('%Y-%m-%d %H:%M') + ' -- current value(s) missing, ' 'cannot check condition %r' % entry.condition) # warn that we cannot check the condition anymore if entry.type: for notifier in self._notifiers[entry.type]: notifier.send('New warning from NICOS', warning_desc) self._put_message('warning', entry, warning_desc) self._warnings[entry.id] = (False, warning_desc) self._update_warnings_str() def _clear_warning(self, entry): """Clear a previously emitted warning for this condition.""" self.log.info('condition %r normal again', entry) was_real_warning = self._warnings.pop(entry.id)[0] if entry.scriptaction == 'pausecount': self._pausecount.pop(entry.id, None) self._update_pausecount_str() self._update_warnings_str() if was_real_warning: self._put_message('resolved', entry, '') if entry.okmessage and entry.type: msg = strftime('%Y-%m-%d %H:%M -- ') msg += '%s\n\nWarning was: %s' % (entry.okmessage, entry.message) for notifier in self._notifiers[entry.type]: notifier.send('NICOS warning resolved', msg) if entry.okaction: self._put_message('action', entry, entry.okaction) self._spawn_action(entry.okaction) # internal helper methods def _put_message(self, msgtype, entry, message): if entry is not None: message = [currenttime(), message, entry.id] self._queue.put('watchdog/%s%s%s\n' % (msgtype, OP_TELL, cache_dump(message))) def _update_mailreceivers(self, emails): self.log.info('updating any Mailer receivers to %s', emails) for notifier in self._all_notifiers: if isinstance(notifier, Mailer): # we're in slave mode, so _setROParam is necessary to set params notifier._setROParam('receivers', emails) def _update_warnings_str(self): self._put_message('warnings', None, '\n'.join(v[1] for v in self._warnings.values())) def _update_pausecount_str(self): self._put_message('pausecount', None, ', '.join(self._pausecount.values())) def _spawn_action(self, action): self.log.warning('will execute action %r', action) script = path.join(config.nicos_root, 'bin', 'nicos-script') createSubprocess([ sys.executable, script, '-M', # start in maintenance mode '-S', '60', # abort after 60 seconds '-A', 'watchdog-action', # appname for the logfiles ','.join(self._setups), # setups to load action ]) # code to execute
class DynamoDBBackend(BaseBackend): def __init__(self): self.tables = OrderedDict() def create_table(self, name, **params): table = Table(name, **params) self.tables[name] = table return table def delete_table(self, name): return self.tables.pop(name, None) def update_table_throughput(self, name, new_read_units, new_write_units): table = self.tables[name] table.read_capacity = new_read_units table.write_capacity = new_write_units return table def put_item(self, table_name, item_attrs): table = self.tables.get(table_name) if not table: return None return table.put_item(item_attrs) def get_item(self, table_name, hash_key_dict, range_key_dict): table = self.tables.get(table_name) if not table: return None hash_key = DynamoType(hash_key_dict) range_key = DynamoType(range_key_dict) if range_key_dict else None return table.get_item(hash_key, range_key) def query(self, table_name, hash_key_dict, range_comparison, range_value_dicts): table = self.tables.get(table_name) if not table: return None, None hash_key = DynamoType(hash_key_dict) range_values = [ DynamoType(range_value) for range_value in range_value_dicts ] return table.query(hash_key, range_comparison, range_values) def scan(self, table_name, filters): table = self.tables.get(table_name) if not table: return None, None, None scan_filters = {} for key, (comparison_operator, comparison_values) in filters.iteritems(): dynamo_types = [DynamoType(value) for value in comparison_values] scan_filters[key] = (comparison_operator, dynamo_types) return table.scan(scan_filters) def delete_item(self, table_name, hash_key_dict, range_key_dict): table = self.tables.get(table_name) if not table: return None hash_key = DynamoType(hash_key_dict) range_key = DynamoType(range_key_dict) if range_key_dict else None return table.delete_item(hash_key, range_key)
class ConnectionState: def __init__(self, *, dispatch, chunker, handlers, syncer, http, loop, **options): self.loop = loop self.http = http self.max_messages = max(options.get('max_messages', 5000), 100) self.dispatch = dispatch self.chunker = chunker self.syncer = syncer self.is_bot = None self.handlers = handlers self.shard_count = None self._ready_task = None self._fetch_offline = options.get('fetch_offline_members', True) self.heartbeat_timeout = options.get('heartbeat_timeout', 60.0) self._listeners = [] activity = options.get('activity', None) if activity: if not isinstance(activity, _ActivityTag): raise TypeError('activity parameter must be one of Game, Streaming, or Activity.') activity = activity.to_dict() status = options.get('status', None) if status: if status is Status.offline: status = 'invisible' else: status = str(status) self._activity = activity self._status = status self.clear() def clear(self): self.user = None self._users = weakref.WeakValueDictionary() self._emojis = {} self._calls = {} self._guilds = {} self._voice_clients = {} # LRU of max size 128 self._private_channels = OrderedDict() # extra dict to look up private channels by user id self._private_channels_by_user = {} self._messages = deque(maxlen=self.max_messages) def process_listeners(self, listener_type, argument, result): removed = [] for i, listener in enumerate(self._listeners): if listener.type != listener_type: continue future = listener.future if future.cancelled(): removed.append(i) continue try: passed = listener.predicate(argument) except Exception as exc: future.set_exception(exc) removed.append(i) else: if passed: future.set_result(result) removed.append(i) if listener.type == ListenerType.chunk: break for index in reversed(removed): del self._listeners[index] def call_handlers(self, key, *args, **kwargs): try: func = self.handlers[key] except KeyError: pass else: func(*args, **kwargs) @property def self_id(self): u = self.user return u.id if u else None @property def voice_clients(self): return list(self._voice_clients.values()) def _get_voice_client(self, guild_id): return self._voice_clients.get(guild_id) def _add_voice_client(self, guild_id, voice): self._voice_clients[guild_id] = voice def _remove_voice_client(self, guild_id): self._voice_clients.pop(guild_id, None) def _update_references(self, ws): for vc in self.voice_clients: vc.main_ws = ws def store_user(self, data): # this way is 300% faster than `dict.setdefault`. user_id = int(data['id']) try: return self._users[user_id] except KeyError: user = User(state=self, data=data) if user.discriminator != '0000': self._users[user_id] = user return user def get_user(self, id): return self._users.get(id) def store_emoji(self, guild, data): emoji_id = int(data['id']) self._emojis[emoji_id] = emoji = Emoji(guild=guild, state=self, data=data) return emoji @property def guilds(self): return list(self._guilds.values()) def _get_guild(self, guild_id): return self._guilds.get(guild_id) def _add_guild(self, guild): self._guilds[guild.id] = guild def _remove_guild(self, guild): self._guilds.pop(guild.id, None) for emoji in guild.emojis: self._emojis.pop(emoji.id, None) del guild @property def emojis(self): return list(self._emojis.values()) def get_emoji(self, emoji_id): return self._emojis.get(emoji_id) @property def private_channels(self): return list(self._private_channels.values()) def _get_private_channel(self, channel_id): try: value = self._private_channels[channel_id] except KeyError: return None else: self._private_channels.move_to_end(channel_id) return value def _get_private_channel_by_user(self, user_id): return self._private_channels_by_user.get(user_id) def _add_private_channel(self, channel): channel_id = channel.id self._private_channels[channel_id] = channel if self.is_bot and len(self._private_channels) > 128: _, to_remove = self._private_channels.popitem(last=False) if isinstance(to_remove, DMChannel): self._private_channels_by_user.pop(to_remove.recipient.id, None) if isinstance(channel, DMChannel): self._private_channels_by_user[channel.recipient.id] = channel def add_dm_channel(self, data): channel = DMChannel(me=self.user, state=self, data=data) self._add_private_channel(channel) return channel def _remove_private_channel(self, channel): self._private_channels.pop(channel.id, None) if isinstance(channel, DMChannel): self._private_channels_by_user.pop(channel.recipient.id, None) def _get_message(self, msg_id): return utils.find(lambda m: m.id == msg_id, reversed(self._messages)) def _add_guild_from_data(self, guild): guild = Guild(data=guild, state=self) self._add_guild(guild) return guild def chunks_needed(self, guild): for _ in range(math.ceil(guild._member_count / 1000)): yield self.receive_chunk(guild.id) def _get_guild_channel(self, data): try: guild = self._get_guild(int(data['guild_id'])) except KeyError: channel = self.get_channel(int(data['channel_id'])) guild = None else: channel = guild and guild.get_channel(int(data['channel_id'])) return channel, guild async def request_offline_members(self, guilds): # get all the chunks chunks = [] for guild in guilds: chunks.extend(self.chunks_needed(guild)) # we only want to request ~75 guilds per chunk request. splits = [guilds[i:i + 75] for i in range(0, len(guilds), 75)] for split in splits: await self.chunker(split) # wait for the chunks if chunks: try: await utils.sane_wait_for(chunks, timeout=len(chunks) * 30.0, loop=self.loop) except asyncio.TimeoutError: log.info('Somehow timed out waiting for chunks.') async def _delay_ready(self): try: launch = self._ready_state.launch # only real bots wait for GUILD_CREATE streaming if self.is_bot: while not launch.is_set(): # this snippet of code is basically waiting 2 seconds # until the last GUILD_CREATE was sent launch.set() await asyncio.sleep(2, loop=self.loop) guilds = next(zip(*self._ready_state.guilds), []) if self._fetch_offline: await self.request_offline_members(guilds) for guild, unavailable in self._ready_state.guilds: if unavailable is False: self.dispatch('guild_available', guild) else: self.dispatch('guild_join', guild) # remove the state try: del self._ready_state except AttributeError: pass # already been deleted somehow # call GUILD_SYNC after we're done chunking if not self.is_bot: log.info('Requesting GUILD_SYNC for %s guilds', len(self.guilds)) await self.syncer([s.id for s in self.guilds]) except asyncio.CancelledError: pass else: # dispatch the event self.call_handlers('ready') self.dispatch('ready') finally: self._ready_task = None def parse_ready(self, data): if self._ready_task is not None: self._ready_task.cancel() self._ready_state = ReadyState(launch=asyncio.Event(), guilds=[]) self.clear() self.user = user = ClientUser(state=self, data=data['user']) self._users[user.id] = user guilds = self._ready_state.guilds for guild_data in data['guilds']: guild = self._add_guild_from_data(guild_data) if (not self.is_bot and not guild.unavailable) or guild.large: guilds.append((guild, guild.unavailable)) for relationship in data.get('relationships', []): try: r_id = int(relationship['id']) except KeyError: continue else: user._relationships[r_id] = Relationship(state=self, data=relationship) for pm in data.get('private_channels', []): factory, _ = _channel_factory(pm['type']) self._add_private_channel(factory(me=user, data=pm, state=self)) self.dispatch('connect') self._ready_task = asyncio.ensure_future(self._delay_ready(), loop=self.loop) def parse_resumed(self, data): self.dispatch('resumed') def parse_message_create(self, data): channel, _ = self._get_guild_channel(data) message = Message(channel=channel, data=data, state=self) self.dispatch('message', message) self._messages.append(message) if channel and channel._type in (0, 5): channel.last_message_id = message.id def parse_message_delete(self, data): raw = RawMessageDeleteEvent(data) found = self._get_message(raw.message_id) raw.cached_message = found self.dispatch('raw_message_delete', raw) if found is not None: self.dispatch('message_delete', found) self._messages.remove(found) def parse_message_delete_bulk(self, data): raw = RawBulkMessageDeleteEvent(data) found_messages = [message for message in self._messages if message.id in raw.message_ids] raw.cached_messages = found_messages self.dispatch('raw_bulk_message_delete', raw) if found_messages: self.dispatch('bulk_message_delete', found_messages) for msg in found_messages: self._messages.remove(msg) def parse_message_update(self, data): raw = RawMessageUpdateEvent(data) self.dispatch('raw_message_edit', raw) message = self._get_message(raw.message_id) if message is not None: older_message = copy.copy(message) if 'call' in data: # call state message edit message._handle_call(data['call']) elif 'content' not in data: # embed only edit message.embeds = [Embed.from_dict(d) for d in data['embeds']] else: message._update(channel=message.channel, data=data) self.dispatch('message_edit', older_message, message) def parse_message_reaction_add(self, data): emoji_data = data['emoji'] emoji_id = utils._get_as_snowflake(emoji_data, 'id') emoji = PartialEmoji.with_state(self, animated=emoji_data['animated'], id=emoji_id, name=emoji_data['name']) raw = RawReactionActionEvent(data, emoji) self.dispatch('raw_reaction_add', raw) # rich interface here message = self._get_message(raw.message_id) if message is not None: emoji = self._upgrade_partial_emoji(emoji) reaction = message._add_reaction(data, emoji, raw.user_id) user = self._get_reaction_user(message.channel, raw.user_id) if user: self.dispatch('reaction_add', reaction, user) def parse_message_reaction_remove_all(self, data): raw = RawReactionClearEvent(data) self.dispatch('raw_reaction_clear', raw) message = self._get_message(raw.message_id) if message is not None: old_reactions = message.reactions.copy() message.reactions.clear() self.dispatch('reaction_clear', message, old_reactions) def parse_message_reaction_remove(self, data): emoji_data = data['emoji'] emoji_id = utils._get_as_snowflake(emoji_data, 'id') emoji = PartialEmoji.with_state(self, animated=emoji_data['animated'], id=emoji_id, name=emoji_data['name']) raw = RawReactionActionEvent(data, emoji) self.dispatch('raw_reaction_remove', raw) message = self._get_message(raw.message_id) if message is not None: emoji = self._upgrade_partial_emoji(emoji) try: reaction = message._remove_reaction(data, emoji, raw.user_id) except (AttributeError, ValueError): # eventual consistency lol pass else: user = self._get_reaction_user(message.channel, raw.user_id) if user: self.dispatch('reaction_remove', reaction, user) def parse_presence_update(self, data): guild_id = utils._get_as_snowflake(data, 'guild_id') guild = self._get_guild(guild_id) if guild is None: log.warning('PRESENCE_UPDATE referencing an unknown guild ID: %s. Discarding.', guild_id) return user = data['user'] member_id = int(user['id']) member = guild.get_member(member_id) if member is None: if 'username' not in user: # sometimes we receive 'incomplete' member data post-removal. # skip these useless cases. return member, old_member = Member._from_presence_update(guild=guild, data=data, state=self) guild._add_member(member) else: old_member = Member._copy(member) user_update = member._presence_update(data=data, user=user) if user_update: self.dispatch('user_update', user_update[0], user_update[1]) self.dispatch('member_update', old_member, member) def parse_user_update(self, data): self.user._update(data) def parse_channel_delete(self, data): guild = self._get_guild(utils._get_as_snowflake(data, 'guild_id')) channel_id = int(data['id']) if guild is not None: channel = guild.get_channel(channel_id) if channel is not None: guild._remove_channel(channel) self.dispatch('guild_channel_delete', channel) else: # the reason we're doing this is so it's also removed from the # private channel by user cache as well channel = self._get_private_channel(channel_id) if channel is not None: self._remove_private_channel(channel) self.dispatch('private_channel_delete', channel) def parse_channel_update(self, data): channel_type = try_enum(ChannelType, data.get('type')) channel_id = int(data['id']) if channel_type is ChannelType.group: channel = self._get_private_channel(channel_id) old_channel = copy.copy(channel) channel._update_group(data) self.dispatch('private_channel_update', old_channel, channel) return guild_id = utils._get_as_snowflake(data, 'guild_id') guild = self._get_guild(guild_id) if guild is not None: channel = guild.get_channel(channel_id) if channel is not None: old_channel = copy.copy(channel) channel._update(guild, data) self.dispatch('guild_channel_update', old_channel, channel) else: log.warning('CHANNEL_UPDATE referencing an unknown channel ID: %s. Discarding.', channel_id) else: log.warning('CHANNEL_UPDATE referencing an unknown guild ID: %s. Discarding.', guild_id) def parse_channel_create(self, data): factory, ch_type = _channel_factory(data['type']) if factory is None: log.warning('CHANNEL_CREATE referencing an unknown channel type %s. Discarding.', data['type']) return channel = None if ch_type in (ChannelType.group, ChannelType.private): channel_id = int(data['id']) if self._get_private_channel(channel_id) is None: channel = factory(me=self.user, data=data, state=self) self._add_private_channel(channel) self.dispatch('private_channel_create', channel) else: guild_id = utils._get_as_snowflake(data, 'guild_id') guild = self._get_guild(guild_id) if guild is not None: channel = factory(guild=guild, state=self, data=data) guild._add_channel(channel) self.dispatch('guild_channel_create', channel) else: log.warning('CHANNEL_CREATE referencing an unknown guild ID: %s. Discarding.', guild_id) return def parse_channel_pins_update(self, data): channel_id = int(data['channel_id']) channel = self.get_channel(channel_id) if channel is None: log.warning('CHANNEL_PINS_UPDATE referencing an unknown channel ID: %s. Discarding.', channel_id) return last_pin = utils.parse_time(data['last_pin_timestamp']) if data['last_pin_timestamp'] else None try: # I have not imported discord.abc in this file # the isinstance check is also 2x slower than just checking this attribute # so we're just gonna check it since it's easier and faster and lazier channel.guild except AttributeError: self.dispatch('private_channel_pins_update', channel, last_pin) else: self.dispatch('guild_channel_pins_update', channel, last_pin) def parse_channel_recipient_add(self, data): channel = self._get_private_channel(int(data['channel_id'])) user = self.store_user(data['user']) channel.recipients.append(user) self.dispatch('group_join', channel, user) def parse_channel_recipient_remove(self, data): channel = self._get_private_channel(int(data['channel_id'])) user = self.store_user(data['user']) try: channel.recipients.remove(user) except ValueError: pass else: self.dispatch('group_remove', channel, user) def parse_guild_member_add(self, data): guild = self._get_guild(int(data['guild_id'])) if guild is None: log.warning('GUILD_MEMBER_ADD referencing an unknown guild ID: %s. Discarding.', data['guild_id']) return member = Member(guild=guild, data=data, state=self) guild._add_member(member) guild._member_count += 1 self.dispatch('member_join', member) def parse_guild_member_remove(self, data): guild = self._get_guild(int(data['guild_id'])) if guild is not None: user_id = int(data['user']['id']) member = guild.get_member(user_id) if member is not None: guild._remove_member(member) guild._member_count -= 1 self.dispatch('member_remove', member) else: log.warning('GUILD_MEMBER_REMOVE referencing an unknown guild ID: %s. Discarding.', data['guild_id']) def parse_guild_member_update(self, data): guild = self._get_guild(int(data['guild_id'])) user = data['user'] user_id = int(user['id']) if guild is None: log.warning('GUILD_MEMBER_UPDATE referencing an unknown guild ID: %s. Discarding.', data['guild_id']) return member = guild.get_member(user_id) if member is not None: old_member = copy.copy(member) member._update(data) self.dispatch('member_update', old_member, member) else: log.warning('GUILD_MEMBER_UPDATE referencing an unknown member ID: %s. Discarding.', user_id) def parse_guild_emojis_update(self, data): guild = self._get_guild(int(data['guild_id'])) if guild is None: log.warning('GUILD_EMOJIS_UPDATE referencing an unknown guild ID: %s. Discarding.', data['guild_id']) return before_emojis = guild.emojis for emoji in before_emojis: self._emojis.pop(emoji.id, None) guild.emojis = tuple(map(lambda d: self.store_emoji(guild, d), data['emojis'])) self.dispatch('guild_emojis_update', guild, before_emojis, guild.emojis) def _get_create_guild(self, data): if data.get('unavailable') is False: # GUILD_CREATE with unavailable in the response # usually means that the guild has become available # and is therefore in the cache guild = self._get_guild(int(data['id'])) if guild is not None: guild.unavailable = False guild._from_data(data) return guild return self._add_guild_from_data(data) async def _chunk_and_dispatch(self, guild, unavailable): chunks = list(self.chunks_needed(guild)) await self.chunker(guild) if chunks: try: await utils.sane_wait_for(chunks, timeout=len(chunks), loop=self.loop) except asyncio.TimeoutError: log.info('Somehow timed out waiting for chunks.') if unavailable is False: self.dispatch('guild_available', guild) else: self.dispatch('guild_join', guild) def parse_guild_create(self, data): unavailable = data.get('unavailable') if unavailable is True: # joined a guild with unavailable == True so.. return guild = self._get_create_guild(data) # check if it requires chunking if guild.large: if unavailable is False: # check if we're waiting for 'useful' READY # and if we are, we don't want to dispatch any # event such as guild_join or guild_available # because we're still in the 'READY' phase. Or # so we say. try: state = self._ready_state state.launch.clear() state.guilds.append((guild, unavailable)) except AttributeError: # the _ready_state attribute is only there during # processing of useful READY. pass else: return # since we're not waiting for 'useful' READY we'll just # do the chunk request here if wanted if self._fetch_offline: asyncio.ensure_future(self._chunk_and_dispatch(guild, unavailable), loop=self.loop) return # Dispatch available if newly available if unavailable is False: self.dispatch('guild_available', guild) else: self.dispatch('guild_join', guild) def parse_guild_sync(self, data): guild = self._get_guild(int(data['id'])) guild._sync(data) def parse_guild_update(self, data): guild = self._get_guild(int(data['id'])) if guild is not None: old_guild = copy.copy(guild) guild._from_data(data) self.dispatch('guild_update', old_guild, guild) else: log.warning('GUILD_UPDATE referencing an unknown guild ID: %s. Discarding.', data['id']) def parse_guild_delete(self, data): guild = self._get_guild(int(data['id'])) if guild is None: log.warning('GUILD_DELETE referencing an unknown guild ID: %s. Discarding.', data['id']) return if data.get('unavailable', False) and guild is not None: # GUILD_DELETE with unavailable being True means that the # guild that was available is now currently unavailable guild.unavailable = True self.dispatch('guild_unavailable', guild) return # do a cleanup of the messages cache self._messages = deque((msg for msg in self._messages if msg.guild != guild), maxlen=self.max_messages) self._remove_guild(guild) self.dispatch('guild_remove', guild) def parse_guild_ban_add(self, data): # we make the assumption that GUILD_BAN_ADD is done # before GUILD_MEMBER_REMOVE is called # hence we don't remove it from cache or do anything # strange with it, the main purpose of this event # is mainly to dispatch to another event worth listening to for logging guild = self._get_guild(int(data['guild_id'])) if guild is not None: try: user = User(data=data['user'], state=self) except KeyError: pass else: member = guild.get_member(user.id) or user self.dispatch('member_ban', guild, member) def parse_guild_ban_remove(self, data): guild = self._get_guild(int(data['guild_id'])) if guild is not None: if 'user' in data: user = self.store_user(data['user']) self.dispatch('member_unban', guild, user) def parse_guild_role_create(self, data): guild = self._get_guild(int(data['guild_id'])) if guild is None: log.warning('GUILD_ROLE_CREATE referencing an unknown guild ID: %s. Discarding.', data['guild_id']) return role_data = data['role'] role = Role(guild=guild, data=role_data, state=self) guild._add_role(role) self.dispatch('guild_role_create', role) def parse_guild_role_delete(self, data): guild = self._get_guild(int(data['guild_id'])) if guild is not None: role_id = int(data['role_id']) try: role = guild._remove_role(role_id) except KeyError: return else: self.dispatch('guild_role_delete', role) else: log.warning('GUILD_ROLE_DELETE referencing an unknown guild ID: %s. Discarding.', data['guild_id']) def parse_guild_role_update(self, data): guild = self._get_guild(int(data['guild_id'])) if guild is not None: role_data = data['role'] role_id = int(role_data['id']) role = guild.get_role(role_id) if role is not None: old_role = copy.copy(role) role._update(role_data) self.dispatch('guild_role_update', old_role, role) else: log.warning('GUILD_ROLE_UPDATE referencing an unknown guild ID: %s. Discarding.', data['guild_id']) def parse_guild_members_chunk(self, data): guild_id = int(data['guild_id']) guild = self._get_guild(guild_id) members = data.get('members', []) for member in members: m = Member(guild=guild, data=member, state=self) existing = guild.get_member(m.id) if existing is None or existing.joined_at is None: guild._add_member(m) log.info('Processed a chunk for %s members in guild ID %s.', len(members), guild_id) self.process_listeners(ListenerType.chunk, guild, len(members)) def parse_guild_integrations_update(self, data): guild = self._get_guild(int(data['guild_id'])) if guild is not None: self.dispatch('guild_integrations_update', guild) else: log.warning('GUILD_INTEGRATIONS_UPDATE referencing an unknown guild ID: %s. Discarding.', data['guild_id']) def parse_webhooks_update(self, data): channel = self.get_channel(int(data['channel_id'])) if channel is not None: self.dispatch('webhooks_update', channel) else: log.warning('WEBHOOKS_UPDATE referencing an unknown channel ID: %s. Discarding.', data['channel_id']) def parse_voice_state_update(self, data): guild = self._get_guild(utils._get_as_snowflake(data, 'guild_id')) channel_id = utils._get_as_snowflake(data, 'channel_id') if guild is not None: if int(data['user_id']) == self.user.id: voice = self._get_voice_client(guild.id) if voice is not None: ch = guild.get_channel(channel_id) if ch is not None: voice.channel = ch member, before, after = guild._update_voice_state(data, channel_id) if member is not None: self.dispatch('voice_state_update', member, before, after) else: log.warning('VOICE_STATE_UPDATE referencing an unknown member ID: %s. Discarding.', data['user_id']) else: # in here we're either at private or group calls call = self._calls.get(channel_id) if call is not None: call._update_voice_state(data) def parse_voice_server_update(self, data): try: key_id = int(data['guild_id']) except KeyError: key_id = int(data['channel_id']) vc = self._get_voice_client(key_id) if vc is not None: asyncio.ensure_future(vc._create_socket(key_id, data)) def parse_typing_start(self, data): channel, guild = self._get_guild_channel(data) if channel is not None: member = None user_id = utils._get_as_snowflake(data, 'user_id') if isinstance(channel, DMChannel): member = channel.recipient elif isinstance(channel, TextChannel) and guild is not None: member = guild.get_member(user_id) elif isinstance(channel, GroupChannel): member = utils.find(lambda x: x.id == user_id, channel.recipients) if member is not None: timestamp = datetime.datetime.utcfromtimestamp(data.get('timestamp')) self.dispatch('typing', channel, member, timestamp) def parse_relationship_add(self, data): key = int(data['id']) old = self.user.get_relationship(key) new = Relationship(state=self, data=data) self.user._relationships[key] = new if old is not None: self.dispatch('relationship_update', old, new) else: self.dispatch('relationship_add', new) def parse_relationship_remove(self, data): key = int(data['id']) try: old = self.user._relationships.pop(key) except KeyError: pass else: self.dispatch('relationship_remove', old) def _get_reaction_user(self, channel, user_id): if isinstance(channel, TextChannel): return channel.guild.get_member(user_id) return self.get_user(user_id) def get_reaction_emoji(self, data): emoji_id = utils._get_as_snowflake(data, 'id') if not emoji_id: return data['name'] try: return self._emojis[emoji_id] except KeyError: return PartialEmoji(animated=data['animated'], id=emoji_id, name=data['name']) def _upgrade_partial_emoji(self, emoji): emoji_id = emoji.id if not emoji_id: return emoji.name try: return self._emojis[emoji_id] except KeyError: return emoji def get_channel(self, id): if id is None: return None pm = self._get_private_channel(id) if pm is not None: return pm for guild in self.guilds: channel = guild.get_channel(id) if channel is not None: return channel def create_message(self, *, channel, data): return Message(state=self, channel=channel, data=data) def receive_chunk(self, guild_id): future = self.loop.create_future() listener = Listener(ListenerType.chunk, future, lambda s: s.id == guild_id) self._listeners.append(listener) return future
def make_table(dt, hide_bar=None): """ Build the HTML needed for a MultiQC table. :param data: MultiQC datatable object """ table_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4)))) table_id = report.save_htmlid(table_id) t_headers = OrderedDict() t_modal_headers = OrderedDict() t_rows = OrderedDict() t_rows_empty = OrderedDict() dt.raw_vals = defaultdict(lambda: dict()) empty_cells = dict() hidden_cols = 1 table_title = dt.pconfig.get('table_title') if table_title is None: table_title = table_id.replace("_", " ").title() for idx, k, header in dt.get_headers_in_order(): rid = header['rid'] # Build the table header cell shared_key = '' if header.get('shared_key', None) is not None: shared_key = ' data-shared-key={}'.format(header['shared_key']) hide = '' muted = '' checked = ' checked="checked"' if header.get('hidden', False) is True: hide = 'hidden' muted = ' text-muted' checked = '' hidden_cols += 1 data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \ .format(header['dmax'], header['dmin'], header['namespace'], shared_key) if header.get('namespace'): cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \ .format(header['namespace'], header['description'], header['title']) else: cell_contents = '<span class="mqc_table_tooltip" title="{} {}">{}</span>' \ .format(header['namespace'], header['description'], header['title']) t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \ .format(rid=rid, h=hide, da=data_attr, c=cell_contents) empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(rid=rid, h=hide) # Build the modal table row t_modal_headers[rid] = """ <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);"> <td class="sorthandle ui-sortable-handle">||</span></td> <td style="text-align:center;"> <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}"> </td> <td>{name}</td> <td>{title}</td> <td>{desc}</td> <td>{col_id}</td> <td>{sk}</td> </tr>""".format( rid=rid, muted=muted, checked=checked, tid=table_id, col=header['colour'], name=header['namespace'], title=header['title'], desc=header['description'], col_id='<code>{}</code>'.format(k), sk=header.get('shared_key', '') ) # Make a colour scale if header['scale'] == False: c_scale = None else: c_scale = mqc_colour.mqc_colour_scale(header['scale'], header['dmin'], header['dmax']) # Add the data table cells for (s_name, samp) in dt.data[idx].items(): if k in samp: val = samp[k] kname = '{}_{}'.format(header['namespace'], rid) dt.raw_vals[s_name][kname] = val # if "is_int" in header.keys(): # val = int(val) # print(val) if 'modify' in header and callable(header['modify']): val = header['modify'](val) try: dmin = header['dmin'] dmax = header['dmax'] percentage = ((float(val) - dmin) / (dmax - dmin)) * 100 percentage = min(percentage, 100) percentage = max(percentage, 0) except (ZeroDivisionError, ValueError): percentage = 0 try: if not header.get("is_int"): valstring = str(header['format'].format(val)) else: valstring = str(int(val)) except ValueError: try: valstring = str(header['format'].format(float(val))) except ValueError: valstring = str(val) except: valstring = str(val) # This is horrible, but Python locale settings are worse if config.thousandsSep_format is None: config.thousandsSep_format = '<span class="mqc_thousandSep"></span>' if config.decimalPoint_format is None: config.decimalPoint_format = '.' valstring = valstring.replace('.', 'DECIMAL').replace(',', 'THOUSAND') valstring = valstring.replace('DECIMAL', config.decimalPoint_format).replace('THOUSAND', config.thousandsSep_format) # Percentage suffixes etc if header.get('suffix') == "show_perc": suff_dict = header.get("suffix_dict") valstring += str(suff_dict.get(s_name)) else: valstring += header.get('suffix', '') # Conditional formatting cmatches = {cfck: False for cfc in config.table_cond_formatting_colours for cfck in cfc} # Find general rules followed by column-specific rules for cfk in ['all_columns', rid]: if cfk in config.table_cond_formatting_rules: # Loop through match types for ftype in cmatches.keys(): # Loop through array of comparison types for cmp in config.table_cond_formatting_rules[cfk].get(ftype, []): try: # Each comparison should be a dict with single key: val if 's_eq' in cmp and str(cmp['s_eq']).lower() == str(val).lower(): cmatches[ftype] = True if 's_contains' in cmp and str(cmp['s_contains']).lower() in str(val).lower(): cmatches[ftype] = True if 's_ne' in cmp and str(cmp['s_ne']).lower() != str(val).lower(): cmatches[ftype] = True if 'eq' in cmp and float(cmp['eq']) == float(val): cmatches[ftype] = True if 'ne' in cmp and float(cmp['ne']) != float(val): cmatches[ftype] = True if 'gt' in cmp and float(cmp['gt']) < float(val): cmatches[ftype] = True if 'lt' in cmp and float(cmp['lt']) > float(val): cmatches[ftype] = True except: logger.warn( "Not able to apply table conditional formatting to '{}' ({})".format(val, cmp)) # Apply HTML in order of config keys bgcol = None for cfc in config.table_cond_formatting_colours: for cfck in cfc: # should always be one, but you never know if cmatches[cfck]: bgcol = cfc[cfck] if bgcol is not None: valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(bgcol, valstring) # Build HTML if not header['scale']: if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][rid] = '<td class="{rid} {h}">{v}</td>'.format(rid=rid, h=hide, v=valstring) # coloring with quartiles elif header['scale'] == "quart": col_dict = header['col_dict'] bar_dict = header.get('bar_dict') col = ' background-color:{};'.format(col_dict.get(s_name)) if bar_dict: bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(bar_dict.get(s_name)+3, col) else: bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col) # bar percentage here val_html = '<span class="val">{}</span>'.format(valstring) wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html) if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html) else: if c_scale is not None: col = ' background-color:{};'.format(c_scale.get_colour(val)) else: col = '' bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col) val_html = '<span class="val">{}</span>'.format(valstring) wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html) if s_name not in t_rows: t_rows[s_name] = dict() t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide, c=wrapper_html) # Is this cell hidden or empty? if s_name not in t_rows_empty: t_rows_empty[s_name] = dict() t_rows_empty[s_name][rid] = header.get('hidden', False) or str(val).strip() == '' # Remove header if we don't have any filled cells for it if sum([len(rows) for rows in t_rows.values()]) == 0: t_headers.pop(rid, None) t_modal_headers.pop(rid, None) logger.debug('Removing header {} from general stats table, as no data'.format(k)) # # Put everything together # # Buttons above the table html = '' if not config.simple_output: # Copy Table Button html += """<div class="row"> <div class="col-sm-2"> <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}"> <span class="glyphicon glyphicon-copy"></span> Copy table </button> </div> """.format(tid=table_id) # <div class="progress-bar progress-bar-warning progress-bar-striped" style="width: 20%"> # <span class="sr-only">20% Complete (warning)</span> # </div> # <div class="progress-bar progress-bar-danger" style="width: 10%"> # <span class="sr-only">10% Complete (danger)</span> # </div> if not hide_bar: html += """ <div class="col-sm-2"> <div id="quartiles_bar"> <div class="progress"> <div class="progress-bar progress-bar-q1" style="width: 25%"> <span class="sr-only">35% Complete (success)</span> Q1 </div> <div class="progress-bar progress-bar-q2" style="width: 25%"> <span class="sr-only">35% Complete (success)</span> Q2 </div> <div class="progress-bar progress-bar-q3" style="width: 25%"> <span class="sr-only">35% Complete (success)</span> Q3 </div> <div class="progress-bar progress-bar-q4" style="width: 25%"> <span class="sr-only">35% Complete (success)</span> Q4 </div> </div> </div> </div> </div> """ else: html += """ </div> """ # Configure Columns Button # if len(t_headers) > 1: # html += """ # <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal"> # <span class="glyphicon glyphicon-th"></span> Configure Columns # </button> # """.format(tid=table_id) # Sort By Highlight button # html += """ # <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;"> # <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight # </button> # """.format(tid=table_id) # Scatter Plot Button # if len(t_headers) > 1: # html += """ # <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}"> # <span class="glyphicon glyphicon glyphicon-stats"></span> Plot # </button> # """.format(tid=table_id) # "Showing x of y columns" text row_visibilities = [all(t_rows_empty[s_name].values()) for s_name in t_rows_empty] visible_rows = [x for x in row_visibilities if not x] # html += """ # <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small> # """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows), ncols_vis = (len(t_headers)+1)-hidden_cols, ncols=len(t_headers)) # Add text # html += """ # <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small> # """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows), # ncols_vis=(len(t_headers) + 1) - hidden_cols, ncols=len(t_headers)) # Build the table itself collapse_class = 'mqc-table-collapse' if len(t_rows) > 10 and config.collapse_tables else '' html += """ <div id="{tid}_container" class="mqc_table_container"> <div class="table-responsive mqc-table-responsive {cc}"> <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}"> """.format(tid=table_id, title=table_title, cc=collapse_class) # Build the header row col1_header = dt.pconfig.get('col1_header', 'Sample Name') html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(col1_header, ''.join(t_headers.values())) # Build the table body html += '<tbody>' t_row_keys = t_rows.keys() if dt.pconfig.get('sortRows') is not False: t_row_keys = sorted(t_row_keys) for s_name in t_row_keys: # Hide the row if all cells are empty or hidden row_hidden = ' style="display:none"' if all(t_rows_empty[s_name].values()) else '' html += '<tr{}>'.format(row_hidden) # Sample name row header html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(sn=s_name) for k in t_headers: html += t_rows[s_name].get(k, empty_cells[k]) html += '</tr>' html += '</tbody></table></div>' if len(t_rows) > 10 and config.collapse_tables: html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>' html += '</div>' # Build the bootstrap modal to customise columns and order # if not config.simple_output: # html += """ # <!-- MultiQC Table Columns Modal --> # <div class="modal fade" id="{tid}_configModal" tabindex="-1"> # <div class="modal-dialog modal-lg"> # <div class="modal-content"> # <div class="modal-header"> # <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> # <h4 class="modal-title">{title}: Columns</h4> # </div> # <div class="modal-body"> # <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p> # <p> # <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button> # <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button> # </p> # <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}"> # <thead> # <tr> # <th class="sorthandle" style="text-align:center;">Sort</th> # <th style="text-align:center;">Visible</th> # <th>Group</th> # <th>Column</th> # <th>Description</th> # <th>ID</th> # <th>Scale</th> # </tr> # </thead> # <tbody> # {trows} # </tbody> # </table> # </div> # <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div> # </div> </div> </div>""".format( tid=table_id, title=table_title, trows=''.join(t_modal_headers.values()) ) # Save the raw values to a file if requested if dt.pconfig.get('save_file') is True: fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id)) util_functions.write_data_file(dt.raw_vals, fn) report.saved_raw_data[fn] = dt.raw_vals return html
def __new__(cls, name, bases, attrs): super_new = super(ResourceBase, cls).__new__ parents = [b for b in bases if isinstance(b, ResourceBase)] if not parents: # If this isn't a subclass of RestObject, don't do anything # special. return super_new(cls, name, bases, attrs) attrs['__ordered__'] = [key for key in attrs.keys() if key not in ('__module__', '__qualname__')] current_fields = [] for key, value in attrs.items(): if isinstance(value, Field): if getattr(value, 'attname', None) is None: value.attname = key value.name = key current_fields.append((key, value)) for key,value in current_fields: attrs.pop(key) current_fields.sort(key=lambda x: x[1].creation_counter) attrs['declared_fields'] = OrderedDict(current_fields) new_class = super_new(cls, name, bases, attrs) # Create the meta class. attr_meta = attrs.pop('Meta', None) abstract = getattr(attr_meta, 'abstract', False) if not attr_meta: meta = getattr(new_class, 'Meta', None) else: meta = attr_meta module = attrs.pop('__module__') app_config = apps.get_containing_app_config(module) # TODO: Verify the purpose and use of this. # Wrapped in a condition to avoid throwing an # AttributeError when meta is None. if meta: meta.app_config = app_config if getattr(meta, 'app_label', None) is None: if app_config is None: model_module = sys.modules[new_class.__module__] package_components = model_module.__name__.split('.') # find the last occurrence of 'models' package_components.reverse() try: app_label_index = package_components.index(MODELS_MODULE_NAME) + 1 except ValueError: #app_label_index = 1 app_label_index = 0 try: kwargs = {"app_label": package_components[app_label_index]} except IndexError: raise ImproperlyConfigured( 'Unable to detect the app label for model "%s." ' 'Ensure that its module, "%s", is located inside an ' 'installed app.' % ( new_class.__name__, model_module.__name__) ) else: kwargs = {"app_label": app_config.label} else: kwargs = {} if not 'resource_name' in meta.__dict__.copy(): meta.resource_name = new_class.__name__ opts = ResourceOptions(meta, **kwargs) new_class._meta = opts # Assign manager. manager = attrs.pop('objects', None) if manager is None: manager = ResourceManager() manager.object_class = new_class # Wrap default or custom managers such that it can only be used on # classes and not on instances. new_class.objects = ResourceManagerDescriptor(manager) # Walk through the MRO. declared_fields = OrderedDict() for base in reversed(new_class.__mro__): # Collect fields from base class. if hasattr(base, 'declared_fields'): declared_fields.update(base.declared_fields) # Field shadowing. for attr, value in base.__dict__.items(): if value is None and attr in declared_fields: declared_fields.pop(attr) primary_key = None for attr, value in declared_fields.items(): setattr(new_class, attr, value) if value.primary_key: if primary_key is not None: raise ImproperlyConfigured('Multiple primary keys.') else: primary_key = value opts._pk_attr = attr new_class.base_fields = declared_fields new_class.declared_fields = declared_fields new_class.DoesNotExist = RestServerException opts._fields = declared_fields opts.concrete_fields = declared_fields opts.concrete_model = new_class class State: db = opts.client adding = False new_class._state = State() if abstract: # Abstract resources are not registered, but subclasses should # still be non-abstract by default. Resetting the attribute to # False, following the behavior in Django models. attr_meta.abstract = False new_class.Meta = attr_meta elif hasattr(opts, 'resources'): # If resource registry was provided, use it to register new_class opts.resources.register(new_class._meta.app_label, new_class) return new_class
class ContextTree: ''' Code based on work by Mi et al., Context Tree for Adaptive Session-based Recommendation, 2018. Parameters -------- history_maxlen: max considered context length nb_candidates (only used for adaptive configuration): the number of recent candidates considered for adaptive configuration expert: type of expert for each context ''' def __init__(self, history_maxlen=50, nb_candidates=1000, expert='StdExpert', session_key='SessionId', item_key='ItemId', time_key='Time'): self.history_maxlen = history_maxlen self.nb_candidates = nb_candidates self.expert = eval(expert) self.item_key = item_key self.session_key = session_key self.time_key = time_key self.root = TreeRoot(self.expert) self.histories = History(history_maxlen) self.recent_candidates = OrderedDict() self.user_to_previous_recoms = {} def fit(self, train, items=None): ''' fit training data for static evalution Parameters -------- data: pandas.DataFrame Training data. It contains the transactions of the sessions. It has one column for session IDs, one for item IDs and one for the timestamp of the events (unix timestamps). It must have a header. Column names are arbitrary, but must correspond to the ones you set during the initialization of the network (session_key, item_key, time_key properties). ''' start_time = time.time() for index, row in train.iterrows(): self.fit_one_row(row, True) if index % 1000000 == 0: print(index, '---- Time: ', time.time() - start_time) def fit_one_row(self, row, first_in_session): ''' fit one row for static setting ''' current_session = row[self.session_key] current_item = row[self.item_key] history = self.histories.get_history(current_session) self.root.update(current_item, history) history.appendleft(current_item) self.root.expand(history) def fit_time_order_online(self, row, first_in_session): ''' fit one row for adpative configuration, we data is ordered by time nb_candidates is used to keep a pool of recent candidates ''' current_session = row[self.session_key] current_item = row[self.item_key] history = self.histories.get_history(current_session) self.root.update(current_item, history) history.appendleft(current_item) best_item_and_probas = self.root.get_n_most_probable( self.recent_candidates.keys(), history) predictions = [proba for rec, proba in best_item_and_probas] series = pd.Series( data=predictions, index=[int(rec) for rec, proba in best_item_and_probas]) if not first_in_session: predictions = preprocessing.normalize([predictions]) series = pd.Series( data=predictions[0], index=[int(rec) for rec, proba in best_item_and_probas]) self.user_to_previous_recoms[current_session] = series self.root.expand(history) self.recent_candidates.pop(current_item, None) self.recent_candidates[current_item] = True if len(self.recent_candidates) > self.nb_candidates: self.recent_candidates.popitem(last=False) def match_context(self, row, items_to_predict, normalize): ''' only used in static evaluation update the recommendation given next time with current row as input the model (CT) is not updated in static evalution for a fair comparison against other methods ''' current_session = row[self.session_key] current_item = row[self.item_key] history = self.histories.get_history(current_session) history.appendleft(current_item) best_item_and_probas = self.root.get_n_most_probable( items_to_predict, history) predictions = [proba for rec, proba in best_item_and_probas] if normalize: predictions = preprocessing.normalize([predictions]) #predictions = [i / sum(predictions) for i in predictions] series = pd.Series( data=predictions[0], index=[int(rec) for rec, proba in best_item_and_probas]) else: series = pd.Series( data=predictions, index=[int(rec) for rec, proba in best_item_and_probas]) self.user_to_previous_recoms[current_session] = series def predict_next(self, session_id, input_item_id, predict_for_item_ids, timestamp=0, skip=False, mode_type="view"): # print(input_item_id) ''' Gives predicton scores for a selected set of items on how likely they be the next item in the session. Parameters -------- session_id : int or string The session IDs of the event. input_item_id : int or string The item ID of the event. Must be in the set of item IDs of the training set. predict_for_item_ids : 1D array IDs of items for which the network should give prediction scores. Every ID must be in the set of item IDs of the training set. Returns -------- out : pandas.Series Prediction scores for selected items on how likely to be the next item of this session. Indexed by the item IDs. ''' row = {self.item_key: input_item_id, self.session_key: session_id} self.match_context(row, predict_for_item_ids, False) previous_recoms = self.user_to_previous_recoms.get(session_id) #print(previous_recoms[:5]) return previous_recoms def clear(self): del self.histories del self.recent_candidates del self.user_to_previous_recoms def support_users(self): ''' whether it is a session-based or session-aware algorithm (if return True, method "predict_with_training_data" must be defined as well) Parameters -------- Returns -------- True : if it is session-aware False : if it is session-based ''' return False
def get_tabulation_data(self, table_code): endpoint = settings.API_URL + '/1.0/tabulation/%s' % table_code r = r_session.get(endpoint) status_code = r.status_code # make sure we've requested a legit tabulation code if status_code == 200: tabulation_data = r.json(object_pairs_hook=OrderedDict) elif status_code == 404 or status_code == 400: error_data = r.json() raise_404_with_messages(self.request, error_data) else: raise Http404 # factories for organizing metadata on arbitrary sets of tables def table_dict_factory(): return { 'version_name': 'Standard Table', 'releases': { 'one_yr': '', 'three_yr': '', 'five_yr': '', }, } def table_ordereddict_factory(): return OrderedDict() def table_expanded_factory(): return {'table_metadata': self.get_table_data(table_code)} tables = OrderedDict() table_grid = OrderedDict() tables_expanded = OrderedDict() default_table = defaultdict(table_dict_factory) default_table_groups = defaultdict(table_ordereddict_factory) default_table_list = defaultdict(table_ordereddict_factory) default_expanded_list = defaultdict(table_ordereddict_factory) default_expanded_table = defaultdict(table_expanded_factory) # take API data and shape into dicts for: # * a grid with each table variant and which releases it's available for # * a list of each primary table, with its column metadata from the API for release in tabulation_data['tables_by_release']: # Sort data by length to have all the PR tables at the end. # Note that sorted() is guaranteed to be stable, per Python docs, # meaning that keys that compare equal (are equal length) will # remain in the same relative order. Assuming they were alphabetical # before this, this is guaranteed to list tables as # tableA, ... , tableI, tableAPR, ... , tableIPR. sorted_data = sorted(tabulation_data['tables_by_release'][release], key=lambda code: len(code)) for table_code in sorted_data: # is this a B or C table? parsed = parse_table_id(table_code) letter_code = parsed['table_type'] tables[letter_code] = default_table_groups[letter_code] # keep the grids separate, track which releases a table is in tables[letter_code] = default_table_list[letter_code] tables[letter_code][table_code] = default_table[table_code] tables[letter_code][table_code]['releases'][ release] = self.RELEASE_TRANSLATE_DICT[release] # get the variant names if parsed['racial']: if parsed['puerto_rico']: tables[letter_code][table_code][ 'version_name'] = parsed['race'] + " (Puerto Rico)" else: tables[letter_code][table_code][ 'version_name'] = parsed['race'] elif parsed['puerto_rico']: tables[letter_code][table_code][ 'version_name'] = "Standard Table (Puerto Rico)" tabulation_data['table_versions'] = tables.pop(self.table_group, None) tabulation_data['related_tables'] = { 'grid': tables, 'preview': {}, } for group, group_values in tables.iteritems(): preview_table = next(group_values.iteritems())[0] try: tabulation_data['related_tables']['preview'][ preview_table] = self.get_table_data(preview_table) tabulation_data['related_tables']['preview'][preview_table][ 'table_type'] = self.TABLE_TYPE_TRANSLATE_DICT[ preview_table.upper()[0]] except ValueError: continue return tabulation_data
class NamedNodeMap(UserDict): """Collection of Attr objects.""" def __init__(self, owner: Node) -> None: """Initialize with owner node. :arg Node owner: owner node of this object. """ self._owner = owner self._dict = OrderedDict() # type: OrderedDict[str, Attr] def __len__(self) -> int: return len(self._dict) def __contains__(self, item: object) -> bool: return item in self._dict def __getitem__(self, index: Union[int, str]) -> Optional[Attr]: if isinstance(index, int): return tuple(self._dict.values())[index] return None def __setitem__(self, attr: str, item: Attr) -> None: self._dict[attr] = item def __delitem__(self, attr: str) -> None: del self._dict[attr] def __iter__(self) -> Iterator[str]: for attr in self._dict.keys(): yield attr @property def length(self) -> int: """Return number of Attrs in this collection.""" return len(self) def getNamedItem(self, name: str) -> Optional[Attr]: """Get ``Attr`` object which has ``name``. If does not have ``name`` attr, return None. """ return self._dict.get(name, None) def setNamedItem(self, item: Attr) -> None: """Set ``Attr`` object in this collection.""" from wdom.web_node import WdomElement if not isinstance(item, Attr): raise TypeError('item must be an instance of Attr') if isinstance(self._owner, WdomElement): self._owner.js_exec( 'setAttribute', item.name, # type: ignore item.value) self._dict[item.name] = item item._owner = self._owner def removeNamedItem(self, item: Attr) -> Optional[Attr]: """Set ``Attr`` object and return it (if exists).""" from wdom.web_node import WdomElement if not isinstance(item, Attr): raise TypeError('item must be an instance of Attr') if isinstance(self._owner, WdomElement): self._owner.js_exec('removeAttribute', item.name) removed_item = self._dict.pop(item.name, None) if removed_item: removed_item._owner = self._owner return removed_item def item(self, index: int) -> Optional[Attr]: """Return ``index``-th attr node.""" if 0 <= index < len(self): return self._dict[tuple(self._dict.keys())[index]] return None def toString(self) -> str: """Return string representation of collections.""" return ' '.join(attr.html for attr in self._dict.values())
class TVDepsGraph(object): cachedir = None cache_indices = None def __init__(self, state, clone=False): self.tasks = OrderedDict() self.data = OrderedDict() self.resources = OrderedDict() self.allocators = OrderedDict() self._evaluated = False self.state = state if clone: return from .TVAction import TVAction, TVTask from .TVArray import TVArrayBase, TVArray, TVBaseShard, TVReadShard, TVArraySeries #1. Initial build of depsgraph from parent register for a in self.state.tasks: task = TVDepsTask(a) self.tasks[len(self.tasks) + 1] = task for a in self.state.data: assert isinstance(a, TVArray) aname = a.name() d = TVDepsData(a) assert aname not in self.data, namestr(aname) self.data[aname] = d a._cache.graph = self if isinstance(a, TVBaseShard): continue aname2 = aname if a._parent is not None and aname[-1] is None: aname2 = aname[0], 0 assert aname2 not in self.resources, namestr(aname2) r = TVDepsResource(a) self.resources[aname2] = r for a in self.state.data_containers: assert isinstance(a, TVArraySeries), type(a) r = TVDepsResource(a) aname = (a.name()[0], None) assert aname not in self.resources, namestr(aname) self.resources[aname] = r outputs = set() output_names = set() for tasknr, task in self.tasks.items(): a = task.tvtask() for arg, argname in zip(a._inputs, a._inputs_names): if isinstance(arg, TVArray): assert arg.name() in self.data, namestr(arg.name()) task.data.append(arg.name()) if a._accumulate: for arg in a._outputs: prev = arg._previous() assert prev.name() in self.data, namestr(prev) task.data.append(prev.name()) for arg, argname in zip(a._outputs, range(1, len(a._outputs) + 1)): assert isinstance(arg, TVArray), (str(a), arg) assert arg.name() in self.data, namestr(arg.name()) assert arg.name() not in output_names, namestr(arg.name()) assert id(arg) not in outputs, namestr(arg.name()) outputs.add(id(arg)) output_names.add(arg.name()) task.resources.append(arg.name()[:2]) self.data[arg.name()].task = tasknr r = TVDepsResource(None) self.resources["@return"] = r for v in self.state.return_: mode, index = v assert mode == "data" a = self.state.data[index] if a._parent is not None and isinstance(a._parent, TVArraySeries): a = a._parent._children[-1] assert a.name() in self.data, namestr(a) r.data.append(a.name()) # 2: remove unassigned data for name0 in list(self.data.keys()): ok = False for name in list(self.data.keys()): if name[:2] != name0[:2]: continue if self.data[name].task is not None: ok = True if not ok: for r in self.resources.values(): rdata = [d for d in r.data] if name in rdata: ok = True break if not ok: for r in self.tasks.values(): rdata = [d for d in r.data] if name in rdata: ok = True break if ok: break if not ok: self.data.pop(name0) # 3: check that every data has dtype and shape for d in self.data: r = self.data[d] tvdata = r.tvdata() if tvdata is None: continue assert tvdata.dtype is not None, d assert tvdata.shape is not None, d # 4: check consistency within containers for c in self.state.data_containers: assert isinstance(c, TVArraySeries) firstchild = c._children[0] for child in c._children[1:]: assert child.dtype == firstchild.dtype, (namestr(c.name()), child.dtype, firstchild.dtype) assert len(child.shape) == len(firstchild.shape), (namestr( c.name()), child.shape, firstchild.shape) # 5: rewrite the depsgraph based on handlers for rname, r in list(self.resources.items()): if rname not in self.resources: continue #if we were deleted by a previous handler if self.resources[rname] is not r: continue #if we were replaced by a previous handler tvdata = r.tvdata() if tvdata is None: continue if tvdata._parent is not None: continue assert isinstance(tvdata, TVArraySeries), r exceptions = [] for handler in _handlers: try: handler(self, rname) break except: raise ### e = traceback.format_exc() exceptions.append(e) else: for e, handler in zip(exceptions, _handlers): print handler.__name__ print e print raise Exception("All handlers failed for resource '%s'" % rname) # 6: add dependencies for shards, remove writeshards that point to eliminated data: for dname, d in list(self.data.items()): dname2 = dname[:2] if isinstance(d.tvdata(), TVWriteShard): if dname2 not in self.data: self.data.pop(dname) continue assert dname2 in self.data, dname2 resname = (namestr(dname2) + "::join", None) if resname not in self.resources: self.resources[resname] = TVDepsResource(None) r = self.data[dname2].resource if r is None: self.data[dname2].resource = resname else: self.resources[r].resources.append(resname) if d.tvdata()._parent._cache_active: cache = TVAllocator("cache", tvname=dname2) cachename = (namestr(dname2) + "::cache", None) self.allocators[cachename] = cache self.resources[resname].allocators.append(cachename) self.resources[resname].data.append(dname) elif isinstance(d.tvdata(), TVReadShard): assert d.resource is None, (dname, d.resource) resname = (namestr(dname2) + "::split", None) if resname not in self.resources: self.resources[resname] = TVDepsResource(None) self.resources[resname].data.append(dname2) d.resource = resname # 7: re-check that every data has dtype and shape (null data not allowed) for d in self.data: r = self.data[d] assert isinstance(r, TVDepsData), d tvdata = r.tvdata() assert tvdata is not None, d assert tvdata.dtype is not None, d assert tvdata.shape is not None, d # 8: calculate signatures for every task for task in self.tasks.values(): t = task.tvtask() t.calc_signature() # 9: build hashtree for non cache-active data (such as those computed on the GPU) for d in self.data: r = self.data[d] data = r.tvdata() if isinstance(data, TVReadShard): continue if not data._cache_active and (not hasattr(data, "_hash") or data._hash is None): if r.hashtree is None: r.hashtree = make_hashtree(d, self) def prune(self): """ Eliminates all tasks with a cachekey, and everything that does not depend on return """ #TODO: will run out of stack space for large depsgraphs tracing_data = set() tracing_resources = set() tracing_tasks = set() dep_data = set() dep_resources = set() dep_tasks = set() dep_allocators = set() def trace_resource(name): if name in dep_resources: return r = self.resources[name] tracing_resources.add(name) for d in r.resources: if d in tracing_resources: raise ValueError( "Resource '%s' and resource '%s' are in a cyclical dependency" % (name, d)) trace_resource(d) for d in r.data: if d in tracing_resources: raise ValueError( "Resource '%s' and data '%s' are in a cyclical dependency" % (name, d)) trace_data(d) for d in r.allocators: ok = True if self.cachedir is None: alloc = self.allocators[d] if alloc.alloc_command == "cache": ok = False if ok: dep_allocators.add(d) dep_resources.add(name) tracing_resources.remove(name) def trace_data(name): if name in dep_data: return r = self.data[name] if r.tvdata() is not None and r.tvdata()._cache.cachekey: dep_data.add(name) for t in self.tasks: task = self.tasks[t] if name in task.data: task._cachekeys.append(r.tvdata()) return tracing_data.add(name) d = r.task if d is not None: if d in tracing_tasks: raise ValueError( "Data '%s' and task %d are in a cyclical dependency" % (namestr(name), d)) trace_task(d) d = r.resource if d is not None: if d in tracing_resources: raise ValueError( "Data '%s' and resource '%s' are in a cyclical dependency" % (namestr(name), d)) trace_resource(d) dep_data.add(name) tracing_data.remove(name) def trace_task(name): if name in dep_tasks: return r = self.tasks[name] tracing_tasks.add(name) for d in r.resources: if d in tracing_resources: raise ValueError( "Task %d and resource '%s' are in a cyclical dependency" % (namestr(name), d)) trace_resource(d) for d in r.data: if d in tracing_resources: raise ValueError( "Task %d and data '%s' are in a cyclical dependency" % (namestr(name), d)) trace_data(d) dep_tasks.add(name) tracing_tasks.remove(name) trace_resource("@return") for d in list(self.data.keys()): if d not in dep_data: self.data.pop(d) for d in list(self.resources.keys()): if d == "@return": continue if d not in dep_resources: self.resources.pop(d) for d in list(self.tasks.keys()): if d not in dep_tasks: self.tasks.pop(d) for d in list(self.allocators.keys()): if d not in dep_allocators: self.allocators.pop(d) retdep = self.resources["@return"].data progress = True while progress: progress = False for d in list(self.data.keys()): r = self.data[d] r.cleanup(self) if d in retdep: continue if r.empty(): progress = True self.data.pop(d) for d in list(self.resources.keys()): if d == "@return": continue r = self.resources[d] r.cleanup(self) if r.empty(): progress = True for a in r.allocators: self.allocators.pop(a) self.resources.pop(d) for d in list(self.tasks.keys()): r = self.tasks[d] r.cleanup(self) if r.empty(): progress = True self.tasks.pop(d) #print "PRUNE", len(self.tasks) def __str__(self): s = "@return:\n" namestrs = [namestr(dd) for dd in self.resources["@return"].data] s += " " + " ".join(namestrs) + "\n" s += "Tasks:\n" for tnr, t in self.tasks.items(): s += " " + str(tnr) + ":\n" s += str(t) s += "Data:\n" for dname in sorted(list(self.data.keys())): d = self.data[dname] s += " " + namestr(dname) + ":\n" s += str(d) s += "Resources:\n" for rname in sorted(list(self.resources.keys())): r = self.resources[rname] if rname == "@return": continue s += " " + namestr(rname) + ":\n" s += str(r) s += "Allocators:\n" for aname in sorted(list(self.allocators.keys())): a = self.allocators[aname] s += " " + namestr(aname) + ":\n" s += str(a) return s def clone(self): newstate = self.state.clone() clone = TVDepsGraph(newstate, clone=True) for k, v in self.tasks.items(): d = v.dict(self.state) clone.tasks[k] = TVDepsTask.fromdict(d, clone.state) for k, v in self.data.items(): d = v.dict(self.state) clone.data[k] = TVDepsData.fromdict(d, clone.state) for k, v in self.resources.items(): d = v.dict(self.state) clone.resources[k] = TVDepsResource.fromdict(d, clone.state) for k, v in self.allocators.items(): clone.allocators[k] = v.clone() for a in newstate.data: a._cache.graph = clone if self.cachedir is not None: clone.cachedir = self.cachedir clone.cache_indices = self.cache_indices return clone def define_cache(self, cachedir): from .TVCache import CacheIndex, cache_indices as global_cache_indices if cachedir is None: self.cachedir = None self.cache_indices = None return self.cachedir = cachedir if not os.path.exists(self.cachedir): os.mkdir(self.cachedir) for d in self.data.values(): data = d.tvdata() if data._cache is not None: data._cache.searched = False cache_indices = {} for action in ("::shard", "::join") + tuple(self.state.actions): k = (self.cachedir, action) if k not in global_cache_indices: dir = k[0] + os.sep + k[1] c = CacheIndex(dir) global_cache_indices[k] = c else: c = global_cache_indices[k] cache_indices[action] = c self.cache_indices = cache_indices def search_cache_hashes(self, allocator): from .TVArray import TVBaseShard, TVReadShard, TVWriteShard if self.cachedir is None: return global progress done = set() progress = True def search_cache_hash(dname): global progress if dname in done: return d = self.data[dname] data = d.tvdata() shards, shards2 = [], [] if isinstance(data, TVBaseShard): dname2 = dname[:2] if isinstance(data, TVReadShard): if dname2 not in done: return if data._parent._cache_active: data.search_cache_hash() done.add(dname) progress = True return elif data.get_hash() is not None: #print "HASH!", dname done.add(dname) progress = True return else: for sh in self.data: if len(sh) < 3: continue if sh[:2] != dname: continue shards.append(sh) shards2 = [ sh for sh in shards if isinstance(self.data[sh].tvdata(), TVWriteShard) ] if d.task is None and not len(shards2): #print "UNUSED!", dname done.add(dname) progress = True return if len(shards2): #print "SEARCHIN?", dname deps = shards2 else: task = self.tasks[d.task] #print "SEARCHING", dname, str(task).split("\n")[0] deps = task.data if not d.tvdata()._cache_active: ok = True for dep in deps: search_cache_hash(dep) if dep not in done: ok = False break if ok: done.add(dname) progress = True return for dep in deps: dd = self.data[dep] ddd = dd.tvdata() if isinstance(ddd, TVBaseShard): if dep not in done: return elif ddd.get_hash() is None: ok = False if not ddd._cache_active: if dd.hashtree.bindable(self): ok = True if not ok: return #print "SEARCH", dname if len(shards2): data._cache.search_join() else: data._cache.search() if data._cache.arrayhash is not None: #print "HASH2!", dname data.set_hash(data._cache.arrayhash) done.add(dname) progress = True #else: # print "MISSED", task while progress: progress = False for d in self.data: search_cache_hash(d) def evaluate(self, cls_evaluator, args, kwargs): from . import get_mode, set_mode #TODO: check that all previous stages are complete assert not self._evaluated self.state.bind_args(args, kwargs) evaluator = cls_evaluator(self) self.data_unpruned = self.data.copy() oldmode = get_mode() set_mode("search_cache") self.search_cache_hashes(evaluator.allocator) #prune depsgraph based on cachekeys self.prune() log.set_plan(str(self)) try: set_mode("evaluate") evaluator.init() evaluator.evaluate_resource("@return") evaluator.run() r = self.resources["@return"] ret = [] for d in r.data: dat = self.data[d] ret.append(dat.tvdata().get_data()) finally: set_mode(oldmode) for d in self.data.values(): dd = d.tvdata() if dd is not None: try: dd.set_data(None) except AttributeError: pass self._evaluated = True self.cache_indices = None ret = tuple(ret) if len(ret) == 1: ret = ret[0] return ret
def __sub__(self, other): """ Allows to remove common fields of two schema.Struct from self by using '-' operator. If two Struct have common field names, the removal is conducted recursively. If a child struct has no fields inside, it will be removed from its parent. Here are examples: Example 1 s1 = Struct( ('a', Scalar()), ('b', Scalar()), ) s2 = Struct(('a', Scalar())) s1 - s2 == Struct(('b', Scalar())) Example 2 s1 = Struct( ('b', Struct( ('c', Scalar()), ('d', Scalar()), )) ) s2 = Struct( ('b', Struct(('c', Scalar()))), ) s1 - s2 == Struct( ('b', Struct( ('d', Scalar()), )), ) Example 3 s1 = Struct( ('a', Scalar()), ('b', Struct( ('d', Scalar()), )) ) s2 = Struct( ('b', Struct( ('c', Scalar()) ('d', Scalar()) )), ) s1 - s2 == Struct( ('a', Scalar()), ) """ if not isinstance(other, Struct): return NotImplemented children = OrderedDict(self.get_children()) for name, right_field in other.get_children(): if name in children: left_field = children[name] if type(left_field) == type(right_field): if isinstance(left_field, Struct): child = left_field - right_field if child.get_children(): children[name] = child continue children.pop(name) else: raise TypeError( "Type of left_field, " + str(type(left_field)) + ", is not the same as that of right_field, " + str(type(right_field)) + ", yet they have the same field name, " + name) return Struct(*(children.items()))
def _extract_metadata(self): """ Reads the metadata in the header Returns ------- meas_parms : OrderedDict Ordered dictionary of Ordered dictionaries (one per image / force channel, etc.) other_parms : OrderedDict Ordered Dictionary of Ordered dictionaries containing all other metadata """ other_parms = OrderedDict() meas_parms = OrderedDict() curr_category = '' temp_dict = OrderedDict() with open(self.file_path, "rb") as file_handle: for ind, line in enumerate(file_handle): line = line.decode("utf-8", 'ignore') trimmed = line.strip().replace("\\", "").replace('@', '') split_data = trimmed.split(':') # First account for wierdly formatted metadata that if len(split_data) == 3: split_data = [ split_data[1] + '_' + split_data[0], split_data[-1] ] elif len(split_data) > 3: # Date: split_ind = trimmed.index(':') split_data = [trimmed[:split_ind], trimmed[split_ind + 1:]] # At this point, split_data should only contain either 1 (class header) or 2 elements if len(split_data) == 1: if len(temp_dict) > 0: if 'Ciao image list' in curr_category or 'Ciao force image list' in curr_category: # In certain cases the same class name occurs multiple times. # Append suffix to existing name and to this name count = 0 for class_name in meas_parms.keys(): if curr_category in class_name: count += 1 if count == 0: meas_parms[curr_category] = temp_dict.copy() else: if count == 1: for class_name in meas_parms.keys(): if curr_category == class_name: # Remove and add back again with suffix # This should only ever happen once. # The next time we come across the same class, all elements already have # suffixes meas_parms[curr_category + '_0'] = meas_parms.pop( curr_category) break meas_parms[curr_category + '_' + str(count)] = temp_dict.copy() else: curr_category = curr_category.replace('Ciao ', '') other_parms[curr_category] = temp_dict.copy() if "*File list end" in trimmed: break curr_category = split_data[0].replace('*', '') temp_dict = OrderedDict() elif len(split_data) == 2: split_data = [item.strip() for item in split_data] try: num_val = float(split_data[1]) if int(num_val) == num_val: num_val = int(num_val) temp_dict[split_data[0]] = num_val except ValueError: temp_dict[split_data[0]] = split_data[1] else: print(split_data) return meas_parms, other_parms
class CBindings(object): def __init__(self, hfiles, includes = [], include_dirs = [], predefs = {}, prelude = [], debug = False): self.hfiles = hfiles self.types = OrderedDict() self.funcs = OrderedDict() self.macros = OrderedDict() self.macros.update(predefs) self.include_dirs = include_dirs lines = [] self._included = set() for hfile in hfiles: lines.extend(self._load_with_includes(hfile, includes)) lines[0:0] = prelude text = self._preprocess(lines) if debug: with open("tmp.c", "w") as f: f.write(text) self.packed_lines = set(self._get_packed_regions(text)) parser = CParser() ast = parser.parse(text, "tmp.c") visitor = CtypesGenVisitor(self) visitor.visit(ast) def _get_packed_regions(self, text): lines = text.splitlines() in_packed = False for i, l in enumerate(lines, 1): l = l.strip().lower() if not l.startswith("#pragma") or "pack" not in l: if in_packed: yield i continue if "push" in l: if in_packed: raise ValueError("Detected #pragma pack without a matching pop") in_packed = True elif "pop" in l: if not in_packed: raise ValueError("Detected #pragma pop without a matching pack") in_packed = False else: raise ValueError("Unknown #pragma pack option %r" % (l,)) if in_packed: raise ValueError("Detected #pragma pack without a matching pop") @classmethod def _find_inc(cls, name, include_dirs): for path in include_dirs: fn = os.path.join(path, name) if os.path.isfile(fn): return fn raise Exception("Cannot open %r - not found in %r" % (name, include_dirs)) def _load_with_includes(self, rootfile, includes): include_dirs = [os.path.dirname(os.path.abspath(rootfile))] + self.include_dirs lines = strip_comments(open(rootfile, "rU").read()+"\n\n\n").splitlines() i = -1 while i + 1 < len(lines): i += 1 line = lines[i] m = include_pattern.match(line) if not m: continue del lines[i] i -= 1 filename = m.group(1) if filename not in includes or filename in self._included: continue self._included.add(filename) fullname = self._find_inc(filename, include_dirs) lines2 = strip_comments(open(fullname, "rU").read()+"\n\n\n").splitlines() lines[i+1:i+1] = lines2 return [l.strip() for l in lines if l.strip()] def _preprocess(self, lines): lines2 = [] for line in lines: m = define_pattern.match(line) if m: name, args, body = m.groups() self.macros[name] = ([a.strip() for a in args.split(",")] if args else (), body) continue m = undef_pattern.match(line) if m: name = m.group(1) self.macros.pop(name, None) continue if ifdef_pattern.match(line) or ifndef_pattern.match(line) or endif_pattern.match(line): continue if if_pattern.match(line) or elif_pattern.match(line) or else_pattern.match(line): continue replaced = True while replaced: replaced = False tokens = delimiters.split(line) for name, (args, body) in self.macros.items(): while name in tokens: replaced = True i = tokens.index(name) tokens[i:i+1] = body line = "".join(tokens).strip() lines2.append(line) return "\n".join(lines2) def export(self, filename): m = PythonModule() m.comment("Auto-generated file; do not edit directly") m.comment(time.ctime()) m.comment("Generated from %s" % (", ".join(self.hfiles,))) m.sep() m.import_("sys") m.import_("ctypes") self.emit_prelude(m) m.sep() generated_macros = set() for name, (args, value) in self.macros.items(): if not self.filter_macro(name, args, value): continue self.emit_macro(m, name, args, value, generated_macros) m.sep() self.emit_get_calling_conv(m) m.sep() for t in self.types.values(): if not self.filter_type(t): continue if isinstance(t, CEnum): self.emit_enum(m, t) elif isinstance(t, CStruct): self.emit_struct_decl(m, t) elif isinstance(t, CUnion): self.emit_union_decl(m, t) elif isinstance(t, CTypedef): pass else: raise TypeError(t) m.sep() for t in self.types.values(): if not self.filter_type(t): continue if isinstance(t, CTypedef): self.emit_typedef(m, t) m.sep() for t in self.types.values(): if not self.filter_type(t): continue if isinstance(t, CStruct): self.emit_struct_fields(m, t) elif isinstance(t, CUnion): self.emit_union_fields(m, t) m.sep() m.stmt("_the_dll = UnloadedDLL") for f in self.funcs.values(): if self.filter_func(f): self.emit_func_unloaded(m, f) m.sep() with m.def_("load_dll", "dllname"): m.stmt("global _the_dll") with m.if_("_the_dll"): m.raise_("ValueError('DLL already loaded')") m.stmt("dll = ctypes.CDLL(dllname)") m.sep() for f in self.funcs.values(): if not self.filter_func(f): continue self.emit_func_prototype(m, f) m.sep() m.stmt("_the_dll = dll") self.before_funcs_hook(m) for f in self.funcs.values(): if not self.filter_func(f): continue self.emit_func(m, f) m.stmt("all_types = [") for t in self.types.values(): if not self.filter_type(t): continue m.stmt(" {0},", t.name) m.stmt("]") m.sep() m.stmt("all_funcs = [") for f in self.funcs.values(): if not self.filter_func(f): continue m.stmt(" {0},", f.name) m.stmt("]") m.dump(filename) return m def filter_macro(self, name, args, value): return True def filter_type(self, tp): return True def filter_func(self, func): return True def emit_prelude(self, m): m.from_("cbinder.lib", "CEnum", "UnloadedDLL") def emit_get_calling_conv(self, m): with m.if_("sys.platform == 'win32'"): m.stmt("_get_calling_conv = ctypes.WINFUNCTYPE)") with m.else_(): m.stmt("_get_calling_conv = ctypes.CFUNCTYPE)") def emit_macro(self, m, name, args, value, generated_macros): if not value: return tokens = delimiters.split(value) identifiers = [tok for tok in tokens if identifier_pattern.match(tok)] if any((tok not in generated_macros and tok not in args) for tok in identifiers): return python_tokens = [] while tokens: t = tokens.pop(0) if not t.strip(): continue if t == "##": if not tokens or not python_tokens: #print "unexpected end of tokens!" return python_tokens[-1] = "str(%s)" % (python_tokens[-1],) python_tokens.append("+ str(%s)" % (tokens.pop(0),)) elif t == "#": if not tokens: #print "unexpected end of tokens!" return python_tokens.append("str(%s)" % (tokens.pop(0),)) else: python_tokens.append(t) code = " ".join(t.encode("unicode_escape") for t in python_tokens).strip() if args: code = "lambda %s: %s" % (", ".join(args), code) try: compile(code, "?", "eval") except Exception: return m.stmt("{0} = {1}", name, code) generated_macros.add(name) def emit_struct_decl(self, m, tp): with m.class_(tp.name, ["ctypes.Structure"]): if tp.packed: m.stmt("_packed_ = 1") for name, type in tp.members: # @ReservedAssignment m.stmt("{0} = {1!r}", name, type.get_ctype()) m.sep() with m.def_("__repr__", "self"): m.return_("'{0}({1})' % ({2})", tp.name, ", ".join("%s = %%r" % (n,) for n, _ in tp.members), ", ".join("self.%s" % (n,) for n, _ in tp.members)) def emit_struct_fields(self, m, tp): m.stmt("{0}._fields_ = [", tp.name) for name, type in tp.members: # @ReservedAssignment m.stmt(" ({0!r}, {1}),", name, type.get_ctype()) m.stmt("]") m.sep() def emit_union_decl(self, m, tp): with m.class_(tp.name, ["ctypes.Union"]): if tp.packed: m.stmt("_packed_ = 1") for name, type in tp.members: # @ReservedAssignment m.stmt("{0} = {1!r}", name, type.get_ctype()) m.sep() with m.def_("__repr__", "self"): m.return_("'{0}({1})' % ({2})", tp.name, ", ".join("%s = %%r" % (n,) for n, _ in tp.members), ", ".join("self.%s" % (n,) for n, _ in tp.members)) def emit_union_fields(self, m, tp): self.emit_struct_fields(m, tp) def emit_enum(self, m, tp): with m.class_(tp.name, ["CEnum"]): m.stmt("_names_ = {0!r}", dict(tp.members)) m.stmt("_values_ = {0!r}", dict((v, k) for k, v in tp.members)) for name, value in tp.members: m.stmt("{0} = {1}", name, value) self.emit_enum_top_level_fields(m, tp) m.sep() def emit_enum_top_level_fields(self, m, tp): if tp.name.startswith("_anon_"): for k, _ in tp.members: m.stmt("{0} = {1}.{0}", k, tp.name) m.sep() def emit_typedef(self, m, tp): m.stmt("{0} = {1}", tp.name, tp.type.get_ctype()) def emit_func_prototype(self, m, func): m.stmt("global _{0}", func.name) m.stmt("_{0} = dll.{0}", func.name) restype = func.type.get_ctype() if restype == "void": restype = None m.stmt("_{0}.restype = {1}", func.name, restype) m.stmt("_{0}.argtypes = [{1}]", func.name, ", ".join(arg.get_ctype() for _, arg in func.args)) def emit_func_unloaded(self, m, func): m.stmt("_{0} = UnloadedDLL", func.name) def before_funcs_hook(self, m): pass def emit_func(self, m, func): with m.def_(func.name, *(n for n, _ in func.args)): textargs = ", ".join("%s" % (arg.as_pretty_type(name),) for name, arg in func.args) m.stmt("'''{0}({1})'''", func.type.as_pretty_type(func.name), textargs) m.return_("_{0}({1})", func.name, ", ".join(n for n, _ in func.args))
class ThumbnailCache(object): ' This is a persistent disk cache to speed up loading and resizing of covers ' def __init__(self, max_size=1024, # The maximum disk space in MB name='thumbnail-cache', # The name of this cache (should be unique in location) thumbnail_size=(100, 100), # The size of the thumbnails, can be changed location=None, # The location for this cache, if None cache_dir() is used test_mode=False, # Used for testing min_disk_cache=0): # If the size is set less than or equal to this value, the cache is disabled. self.location = os.path.join(location or cache_dir(), name) if max_size <= min_disk_cache: max_size = 0 self.max_size = int(max_size * (1024**2)) self.group_id = 'group' self.thumbnail_size = thumbnail_size self.size_changed = False self.lock = Lock() self.min_disk_cache = min_disk_cache if test_mode: self.log = self.fail_on_error def log(self, *args, **kwargs): kwargs['file'] = sys.stderr prints(*args, **kwargs) def fail_on_error(self, *args, **kwargs): msg = ' '.join(args) raise CacheError(msg) def _do_delete(self, path): try: os.remove(path) except EnvironmentError as err: self.log('Failed to delete cached thumbnail file:', as_unicode(err)) def _load_index(self): 'Load the index, automatically removing incorrectly sized thumbnails and pruning to fit max_size' try: os.makedirs(self.location) except OSError as err: if err.errno != errno.EEXIST: self.log('Failed to make thumbnail cache dir:', as_unicode(err)) self.total_size = 0 self.items = OrderedDict() order = self._read_order() def listdir(*args): try: return os.listdir(os.path.join(*args)) except EnvironmentError: return () # not a directory or no permission or whatever entries = ('/'.join((parent, subdir, entry)) for parent in listdir(self.location) for subdir in listdir(self.location, parent) for entry in listdir(self.location, parent, subdir)) invalidate = set() try: with open(os.path.join(self.location, 'invalidate'), 'rb') as f: raw = f.read().decode('utf-8') except EnvironmentError as err: if getattr(err, 'errno', None) != errno.ENOENT: self.log('Failed to read thumbnail invalidate data:', as_unicode(err)) else: try: os.remove(os.path.join(self.location, 'invalidate')) except EnvironmentError as err: self.log('Failed to remove thumbnail invalidate data:', as_unicode(err)) else: def record(line): try: uuid, book_id = line.partition(' ')[0::2] book_id = int(book_id) return (uuid, book_id) except Exception: return None invalidate = {record(x) for x in raw.splitlines()} items = [] try: for entry in entries: try: uuid, name = entry.split('/')[0::2] book_id, timestamp, size, thumbnail_size = name.split('-') book_id, timestamp, size = int(book_id), float(timestamp), int(size) thumbnail_size = tuple(map(int, thumbnail_size.partition('x')[0::2])) except (ValueError, TypeError, IndexError, KeyError, AttributeError): continue key = (uuid, book_id) path = os.path.join(self.location, entry) if self.thumbnail_size == thumbnail_size and key not in invalidate: items.append((key, Entry(path, size, timestamp, thumbnail_size))) self.total_size += size else: self._do_delete(path) except EnvironmentError as err: self.log('Failed to read thumbnail cache dir:', as_unicode(err)) self.items = OrderedDict(sorted(items, key=lambda x:order.get(x[0], 0))) self._apply_size() def _invalidate_sizes(self): if self.size_changed: size = self.thumbnail_size remove = tuple(key for key, entry in iteritems(self.items) if size != entry.thumbnail_size) for key in remove: self._remove(key) self.size_changed = False def _remove(self, key): entry = self.items.pop(key, None) if entry is not None: self._do_delete(entry.path) self.total_size -= entry.size def _apply_size(self): while self.total_size > self.max_size and self.items: entry = self.items.popitem(last=False)[1] self._do_delete(entry.path) self.total_size -= entry.size def _write_order(self): if hasattr(self, 'items'): try: data = '\n'.join(group_id + ' ' + unicode_type(book_id) for (group_id, book_id) in self.items) with lopen(os.path.join(self.location, 'order'), 'wb') as f: f.write(data.encode('utf-8')) except EnvironmentError as err: self.log('Failed to save thumbnail cache order:', as_unicode(err)) def _read_order(self): order = {} try: with lopen(os.path.join(self.location, 'order'), 'rb') as f: for line in f.read().decode('utf-8').splitlines(): parts = line.split(' ', 1) if len(parts) == 2: order[(parts[0], int(parts[1]))] = len(order) except Exception as err: if getattr(err, 'errno', None) != errno.ENOENT: self.log('Failed to load thumbnail cache order:', as_unicode(err)) return order def shutdown(self): with self.lock: self._write_order() def set_group_id(self, group_id): with self.lock: self.group_id = group_id def set_thumbnail_size(self, width, height): new_size = (width, height) with self.lock: if new_size != self.thumbnail_size: self.thumbnail_size = new_size self.size_changed = True return True return False def insert(self, book_id, timestamp, data): if self.max_size < len(data): return with self.lock: if not hasattr(self, 'total_size'): self._load_index() self._invalidate_sizes() ts = ('%.2f' % timestamp).replace('.00', '') path = '%s%s%s%s%d-%s-%d-%dx%d' % ( self.group_id, os.sep, book_id % 100, os.sep, book_id, ts, len(data), self.thumbnail_size[0], self.thumbnail_size[1]) path = os.path.join(self.location, path) key = (self.group_id, book_id) e = self.items.pop(key, None) self.total_size -= getattr(e, 'size', 0) try: with open(path, 'wb') as f: f.write(data) except EnvironmentError as err: d = os.path.dirname(path) if not os.path.exists(d): try: os.makedirs(d) with open(path, 'wb') as f: f.write(data) except EnvironmentError as err: self.log('Failed to write cached thumbnail:', path, as_unicode(err)) return self._apply_size() else: self.log('Failed to write cached thumbnail:', path, as_unicode(err)) return self._apply_size() self.items[key] = Entry(path, len(data), timestamp, self.thumbnail_size) self.total_size += len(data) self._apply_size() def __len__(self): with self.lock: try: return len(self.items) except AttributeError: self._load_index() return len(self.items) def __contains__(self, book_id): with self.lock: try: return (self.group_id, book_id) in self.items except AttributeError: self._load_index() return (self.group_id, book_id) in self.items def __getitem__(self, book_id): with self.lock: if not hasattr(self, 'total_size'): self._load_index() self._invalidate_sizes() key = (self.group_id, book_id) entry = self.items.pop(key, None) if entry is None: return None, None if entry.thumbnail_size != self.thumbnail_size: try: os.remove(entry.path) except EnvironmentError as err: if getattr(err, 'errno', None) != errno.ENOENT: self.log('Failed to remove cached thumbnail:', entry.path, as_unicode(err)) self.total_size -= entry.size return None, None self.items[key] = entry try: with open(entry.path, 'rb') as f: data = f.read() except EnvironmentError as err: self.log('Failed to read cached thumbnail:', entry.path, as_unicode(err)) return None, None return data, entry.timestamp def invalidate(self, book_ids): with self.lock: if hasattr(self, 'total_size'): for book_id in book_ids: self._remove((self.group_id, book_id)) elif os.path.exists(self.location): try: raw = '\n'.join('%s %d' % (self.group_id, book_id) for book_id in book_ids) with open(os.path.join(self.location, 'invalidate'), 'ab') as f: f.write(raw.encode('ascii')) except EnvironmentError as err: self.log('Failed to write invalidate thumbnail record:', as_unicode(err)) @property def current_size(self): with self.lock: if not hasattr(self, 'total_size'): self._load_index() return self.total_size def empty(self): with self.lock: try: os.remove(os.path.join(self.location, 'order')) except EnvironmentError: pass if not hasattr(self, 'total_size'): self._load_index() for entry in itervalues(self.items): self._do_delete(entry.path) self.total_size = 0 self.items = OrderedDict() def __hash__(self): return id(self) def set_size(self, size_in_mb): if size_in_mb <= self.min_disk_cache: size_in_mb = 0 size_in_mb = max(0, size_in_mb) with self.lock: self.max_size = int(size_in_mb * (1024**2)) if hasattr(self, 'total_size'): self._apply_size()
class WorkerQueue(object): def __init__(self): self.queue = OrderedDict() self.context = None self.frontend = None self.backend = None self.poll_both = None self.poll_workers = None self.heartbeat_at = 0.0 self.__conf = _QueueConf() self.HEARTBEAT_LIVENESS = self.__conf.getWorkerHeartBeatLiveness() self.HEARTBEAT_INTERVAL = self.__conf.getHeartBeatInterval() # Paranoid Pirate Protocol constants self.PPP_READY = "\x01" # Signals worker is ready self.PPP_HEARTBEAT = "\x02" # Signals worker heartbeat def connect(self): self.context = zmq.Context(1) self.frontend = self.context.socket(zmq.ROUTER) self.backend = self.context.socket(zmq.ROUTER) self.frontend.bind("tcp://*:{0}".format(self.__conf.getFrontEndPort())) # For clients self.backend.bind("tcp://*:{0}".format(self.__conf.getBackEndPort())) # For workers self.poll_workers = zmq.Poller() self.poll_workers.register(self.backend, zmq.POLLIN) self.poll_both = zmq.Poller() self.poll_both.register(self.frontend, zmq.POLLIN) self.poll_both.register(self.backend, zmq.POLLIN) self.heartbeat_at = time.time() + self.HEARTBEAT_INTERVAL def ready(self, worker): self.queue.pop(worker.address, None) self.queue[worker.address] = worker def run(self): heartbeat_at = time.time() + self.HEARTBEAT_INTERVAL while True: if len(self.queue) > 0: poller = self.poll_both else: poller = self.poll_workers socks = dict(poller.poll(self.HEARTBEAT_INTERVAL * 1000)) # Handle worker activity on backend if socks.get(self.backend) == zmq.POLLIN: # Use worker address for LRU routing frames = self.backend.recv_multipart() if not frames: break address = frames[0] self.ready(_Worker(address, self.HEARTBEAT_LIVENESS, self.HEARTBEAT_INTERVAL)) # Validate control message, or return reply to client msg = frames[1:] if len(msg) == 1: if msg[0] not in (self.PPP_READY, self.PPP_HEARTBEAT): logging.error("Invalid message from worker: %s" % msg) else: self.frontend.send_multipart(msg) # Send heartbeats to idle workers if it's time if time.time() >= heartbeat_at: for worker in self.queue: msg = [worker, self.PPP_HEARTBEAT] self.backend.send_multipart(msg) heartbeat_at = time.time() + self.HEARTBEAT_INTERVAL if socks.get(self.frontend) == zmq.POLLIN: frames = self.frontend.recv_multipart() if not frames: break frames.insert(0, self.next()) self.backend.send_multipart(frames) self.purge() def purge(self): """Look for & kill expired workers.""" t = time.time() expired = [] for address,worker in self.queue.iteritems(): if t > worker.expiry: # Worker expired expired.append(address) for address in expired: logging.info("[WORKER]: Idle worker expired: %s" % address) self.queue.pop(address, None) def next(self): address, worker = self.queue.popitem(False) return address
class SymbolTable(object): # pylint: disable=too-many-public-methods '''Encapsulates the symbol table and provides methods to add new symbols and look up existing symbols. Nested scopes are supported and, by default, the add and lookup methods take any ancestor symbol tables into consideration (ones attached to nodes that are ancestors of the node that this symbol table is attached to). :param node: reference to the Schedule or Container to which this \ symbol table belongs. :type node: :py:class:`psyclone.psyir.nodes.Schedule`, \ :py:class:`psyclone.psyir.nodes.Container` or NoneType :raises TypeError: if node argument is not a Schedule or a Container. ''' def __init__(self, node=None): # Dict of Symbol objects with the symbol names as keys. Make # this ordered so that different versions of Python always # produce code with declarations in the same order. self._symbols = OrderedDict() # Ordered list of the arguments. self._argument_list = [] # Dict of tags. Some symbols can be identified with a tag. self._tags = {} # Reference to the node to which this symbol table belongs. # pylint: disable=import-outside-toplevel from psyclone.psyir.nodes import Schedule, Container if node and not isinstance(node, (Schedule, Container)): raise TypeError( "Optional node argument to SymbolTable should be a " "Schedule or a Container but found '{0}'." "".format(type(node).__name__)) self._node = node @property def node(self): ''' :returns: the Schedule or Container to which this symbol table belongs. :rtype: :py:class:`psyclone.psyir.nodes.Schedule`, \ :py:class:`psyclone.psyir.nodes.Container` or NoneType ''' return self._node def parent_symbol_table(self, scope_limit=None): '''If this symbol table is enclosed in another scope, return the symbol table of the next outer scope. Otherwise return None. :param scope_limit: optional Node which limits the symbol \ search space to the symbol tables of the nodes within the \ given scope. If it is None (the default), the whole \ scope (all symbol tables in ancestor nodes) is searched \ otherwise ancestors of the scope_limit node are not \ searched. :type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \ `NoneType` :returns: the 'parent' SymbolTable of the current SymbolTable (i.e. the one that encloses this one in the PSyIR hierarchy). :rtype: :py:class:`psyclone.psyir.symbols.SymbolTable` or NoneType ''' # Validate the supplied scope_limit if scope_limit is not None: # pylint: disable=import-outside-toplevel from psyclone.psyir.nodes import Node if not isinstance(scope_limit, Node): raise TypeError( "The scope_limit argument '{0}', is not of type `Node`." "".format(str(scope_limit))) # We use the Node with which this table is associated in order to # move up the Node hierarchy if self.node: search_next = self.node while search_next is not scope_limit and search_next.parent: search_next = search_next.parent if hasattr(search_next, 'symbol_table'): return search_next.symbol_table return None def get_symbols(self, scope_limit=None): '''Return symbols from this symbol table and all symbol tables associated with ancestors of the node that this symbol table is attached to. If there are name duplicates we only return the one from the closest ancestor including self. It accepts an optional scope_limit argument. :param scope_limit: optional Node which limits the symbol \ search space to the symbol tables of the nodes within the \ given scope. If it is None (the default), the whole \ scope (all symbol tables in ancestor nodes) is searched \ otherwise ancestors of the scope_limit node are not \ searched. :type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \ `NoneType` :returns: ordered dictionary of symbols indexed by symbol name. :rtype: OrderedDict[str] = :py:class:`psyclone.psyir.symbols.Symbol` ''' all_symbols = OrderedDict() current = self while current: for symbol_name, symbol in current.symbols_dict.items(): if symbol_name not in all_symbols: all_symbols[symbol_name] = symbol current = current.parent_symbol_table(scope_limit) return all_symbols def get_tags(self, scope_limit=None): '''Return tags from this symbol table and all symbol tables associated with ancestors of the node that this symbol table is attached to. If there are tag duplicates we only return the one from the closest ancestor including self. It accepts an optional scope_limit argument. :param scope_limit: optional Node which limits the symbol \ search space to the symbol tables of the nodes within the \ given scope. If it is None (the default), the whole \ scope (all symbol tables in ancestor nodes) is searched \ otherwise ancestors of the scope_limit node are not \ searched. :type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \ `NoneType` :returns: ordered dictionary of symbols indexed by tag. :rtype: OrderedDict[str] = :py:class:`psyclone.psyir.symbols.Symbol` ''' all_tags = OrderedDict() current = self while current: for tag, symbol in current.tags_dict.items(): if tag not in all_tags: all_tags[tag] = symbol current = current.parent_symbol_table(scope_limit) return all_tags def shallow_copy(self): '''Create a copy of the symbol table with new instances of the top-level data structures but keeping the same existing symbol objects. Symbols added to the new symbol table will not be added in the original but the existing objects are still the same. :returns: a shallow copy of this symbol table. :rtype: :py:class:`psyclone.psyir.symbols.SymbolTable` ''' # pylint: disable=protected-access new_st = SymbolTable() new_st._symbols = copy.copy(self._symbols) new_st._argument_list = copy.copy(self._argument_list) new_st._tags = copy.copy(self._tags) new_st._node = self.node return new_st def deep_copy(self): '''Create a copy of the symbol table with new instances of the top-level data structures and also new instances of the symbols contained in these data structures. Modifying a symbol attribute will not affect the equivalent named symbol in the original symbol table. :returns: a deep copy of this symbol table. :rtype: :py:class:`psyclone.psyir.symbols.SymbolTable` ''' # pylint: disable=protected-access new_st = SymbolTable(self.node) # Make a copy of each symbol in the symbol table for symbol in self.symbols: new_st.add(symbol.copy()) # Prepare the new argument list new_arguments = [] for name in [arg.name for arg in self.argument_list]: new_arguments.append(new_st.lookup(name)) new_st.specify_argument_list(new_arguments) # Prepare the new tag dict for tag, symbol in self._tags.items(): new_st._tags[tag] = new_st.lookup(symbol.name) return new_st @staticmethod def _normalize(key): '''Normalises the symboltable key strings. :param str key: an input key. :returns: the normalized key. :rtype: str ''' # The symbol table is currently case insensitive new_key = key.lower() return new_key def new_symbol(self, root_name=None, tag=None, shadowing=False, symbol_type=None, **symbol_init_args): ''' Create a new symbol. Optional root_name and shadowing arguments can be given to choose the name following the rules of next_available_name(). An optional tag can also be given. By default it creates a generic symbol but a symbol_type argument and any additional initialization keyword arguments of this symbol_type can be provided to refine the created Symbol. :param root_name: optional name to use when creating a new \ symbol name. This will be appended with an integer if the name \ clashes with an existing symbol name. :type root_name: str or NoneType :param str tag: optional tag identifier for the new symbol. :param bool shadowing: optional logical flag indicating whether the \ name can be overlapping with a symbol in any of the ancestors \ symbol tables. Defaults to False. :param symbol_type: class type of the new symbol. :type symbol_type: type object of class (or subclasses) of \ :py:class:`psyclone.psyir.symbols.Symbol` :param symbol_init_args: arguments to create a new symbol. :type symbol_init_args: unwrapped Dict[str] = object :raises TypeError: if the type_symbol argument is not the type of a \ Symbol object class or one of its subclasses. ''' # Only type-check symbol_type, the other arguments are just passed down # and type checked inside each relevant method. if symbol_type is not None: if not (isinstance(symbol_type, type) and Symbol in inspect.getmro(symbol_type)): raise TypeError( "The symbol_type parameter should be a type class of " "Symbol or one of its sub-classes but found '{0}' instead." .format(type(symbol_type).__name__)) else: symbol_type = Symbol available_name = self.next_available_name(root_name, shadowing) symbol = symbol_type(available_name, **symbol_init_args) self.add(symbol, tag) return symbol def symbol_from_tag(self, tag, root_name=None, **new_symbol_args): ''' Lookup a tag, if it doesn't exist create a new symbol with the given tag. By default it creates a generic Symbol with the tag as the root of the symbol name. Optionally, a different root_name or any of the arguments available in the new_symbol() method can be given to refine the name and the type of the created Symbol. :param str tag: tag identifier. :param str root_name: optional name of the new symbol if it needs \ to be created. Otherwise it is ignored. :param new_symbol_args: arguments to create a new symbol. :type new_symbol_args: unwrapped Dict[str, object] :returns: symbol associated with the given tag. :rtype: :py:class:`psyclone.psyir.symbols.Symbol` :raises SymbolError: if the symbol already exists but the type_symbol \ argument does not match the type of the symbol \ found. ''' try: symbol = self.lookup_with_tag(tag) # Check that the symbol found matches the requested description if 'symbol_type' in new_symbol_args: symbol_type = new_symbol_args['symbol_type'] if not isinstance(symbol, new_symbol_args['symbol_type']): raise SymbolError( "Expected symbol with tag '{0}' to be of type '{1}' " "but found type '{2}'.".format(tag, symbol_type.__name__, type(symbol).__name__)) # TODO #1057: If the symbol is found and some unmatching arguments # were given it should also fail here. return symbol except KeyError: if not root_name: root_name = tag return self.new_symbol(root_name, tag, **new_symbol_args) def next_available_name(self, root_name=None, shadowing=False): '''Return a name that is not in the symbol table and therefore can be used to declare a new symbol. If the `root_name` argument is not supplied or if it is an empty string then the name is generated internally, otherwise the `root_name` is used. If required, an additional integer is appended to avoid clashes. If the shadowing argument is True (is False by default), the names in parent symbol tables will not be considered. :param str root_name: optional name to use when creating a new \ symbol name. This will be appended with an integer if the name \ clashes with an existing symbol name. :param bool shadowing: optional logical flag indicating whether the \ name can be overlapping with a symbol in any of the ancestors \ symbol tables. Defaults to False. :returns: the new unique symbol name. :rtype: str :raises TypeError: if the root_name argument is not a string \ or None. :raises TypeError: if the shadowing argument is not a bool. ''' if not isinstance(shadowing, bool): raise TypeError("Argument shadowing should be of type bool" " but found '{0}'.".format( type(shadowing).__name__)) if shadowing: symbols = self._symbols else: # If symbol shadowing is not permitted, the list of symbols names # that can't be used includes all the symbols from all the ancestor # symbol tables. symbols = self.get_symbols() if root_name is not None: if not isinstance(root_name, six.string_types): raise TypeError( "Argument root_name should be of type str or NoneType but " "found '{0}'.".format(type(root_name).__name__)) if not root_name: root_name = Config.get().psyir_root_name candidate_name = root_name idx = 1 while candidate_name in symbols: candidate_name = "{0}_{1}".format(root_name, idx) idx += 1 return candidate_name def add(self, new_symbol, tag=None): '''Add a new symbol to the symbol table if the symbol name is not already in use. :param new_symbol: the symbol to add to the symbol table. :type new_symbol: :py:class:`psyclone.psyir.symbols.Symbol` :param str tag: a tag identifier for the new symbol, by default no \ tag is given. :raises InternalError: if the new_symbol argument is not a \ symbol. :raises KeyError: if the symbol name is already in use. :raises KeyError: if a tag is supplied and it is already in \ use. ''' if not isinstance(new_symbol, Symbol): raise InternalError( "Symbol '{0}' is not a symbol, but '{1}'.'".format( new_symbol, type(new_symbol).__name__)) key = self._normalize(new_symbol.name) if key in self._symbols: raise KeyError("Symbol table already contains a symbol with" " name '{0}'.".format(new_symbol.name)) if tag: if tag in self.get_tags(): raise KeyError( "This symbol table, or an outer scope ancestor symbol " "table, already contains the tag '{0}' for the symbol" " '{1}', so it can not be associated with symbol '{2}'.". format(tag, self.lookup_with_tag(tag), new_symbol.name)) self._tags[tag] = new_symbol self._symbols[key] = new_symbol def swap_symbol_properties(self, symbol1, symbol2): '''Swaps the properties of symbol1 and symbol2 apart from the symbol name. Argument list positions are also updated appropriately. :param symbol1: the first symbol. :type symbol1: :py:class:`psyclone.psyir.symbols.Symbol` :param symbol2: the second symbol. :type symbol2: :py:class:`psyclone.psyir.symbols.Symbol` :raises KeyError: if either of the supplied symbols are not in \ the symbol table. :raises TypeError: if the supplied arguments are not symbols, \ or the names of the symbols are the same in the SymbolTable \ instance. ''' for symbol in [symbol1, symbol2]: if not isinstance(symbol, Symbol): raise TypeError("Arguments should be of type 'Symbol' but " "found '{0}'.".format(type(symbol).__name__)) if symbol.name not in self._symbols: raise KeyError("Symbol '{0}' is not in the symbol table." "".format(symbol.name)) if symbol1.name == symbol2.name: raise ValueError("The symbols should have different names, but " "found '{0}' for both.".format(symbol1.name)) tmp_symbol = symbol1.copy() symbol1.copy_properties(symbol2) symbol2.copy_properties(tmp_symbol) # Update argument list if necessary index1 = None if symbol1 in self._argument_list: index1 = self._argument_list.index(symbol1) index2 = None if symbol2 in self._argument_list: index2 = self._argument_list.index(symbol2) if index1 is not None: self._argument_list[index1] = symbol2 if index2 is not None: self._argument_list[index2] = symbol1 def specify_argument_list(self, argument_symbols): ''' Sets-up the internal list storing the order of the arguments to this kernel. :param list argument_symbols: ordered list of the DataSymbols \ representing the kernel arguments. :raises ValueError: if the new argument_list is not consistent with \ the existing entries in the SymbolTable. ''' self._validate_arg_list(argument_symbols) self._argument_list = argument_symbols[:] def lookup(self, name, visibility=None, scope_limit=None): '''Look up a symbol in the symbol table. The lookup can be limited by visibility (e.g. just show public methods) or by scope_limit (e.g. just show symbols up to a certain scope). :param str name: name of the symbol. :param visibilty: the visibility or list of visibilities that the \ symbol must have. :type visibility: [list of] :py:class:`psyclone.symbols.Visibility` :param scope_limit: optional Node which limits the symbol \ search space to the symbol tables of the nodes within the \ given scope. If it is None (the default), the whole \ scope (all symbol tables in ancestor nodes) is searched \ otherwise ancestors of the scope_limit node are not \ searched. :type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \ `NoneType` :returns: the symbol with the given name and, if specified, visibility. :rtype: :py:class:`psyclone.psyir.symbols.Symbol` :raises TypeError: if the name argument is not a string. :raises SymbolError: if the name exists in the Symbol Table but does \ not have the specified visibility. :raises TypeError: if the visibility argument has the wrong type. :raises KeyError: if the given name is not in the Symbol Table. ''' if not isinstance(name, six.string_types): raise TypeError( "Expected the name argument to the lookup() method to be " "a str but found '{0}'." "".format(type(name).__name__)) try: symbol = self.get_symbols(scope_limit)[self._normalize(name)] if visibility: if not isinstance(visibility, list): vis_list = [visibility] else: vis_list = visibility if symbol.visibility not in vis_list: vis_names = [] # Take care here in case the 'visibility' argument # is of the wrong type for vis in vis_list: if not isinstance(vis, Symbol.Visibility): raise TypeError( "the 'visibility' argument to lookup() must be" " an instance (or list of instances) of " "Symbol.Visibility but got '{0}' when " "searching for symbol '{1}'".format( type(vis).__name__, name)) vis_names.append(vis.name) raise SymbolError( "Symbol '{0}' exists in the Symbol Table but has " "visibility '{1}' which does not match with the " "requested visibility: {2}".format( name, symbol.visibility.name, vis_names)) return symbol except KeyError as err: six.raise_from( KeyError("Could not find '{0}' in the Symbol Table." "".format(name)), err) def lookup_with_tag(self, tag, scope_limit=None): '''Look up a symbol by its tag. The lookup can be limited by scope_limit (e.g. just show symbols up to a certain scope). :param str tag: tag identifier. :param scope_limit: optional Node which limits the symbol \ search space to the symbol tables of the nodes within the \ given scope. If it is None (the default), the whole \ scope (all symbol tables in ancestor nodes) is searched \ otherwise ancestors of the scope_limit node are not \ searched. :returns: symbol with the given tag. :rtype: :py:class:`psyclone.psyir.symbols.Symbol` :raises TypeError: if the tag argument is not a string. :raises KeyError: if the given tag is not in the Symbol Table. ''' if not isinstance(tag, six.string_types): raise TypeError( "Expected the tag argument to the lookup_with_tag() method " "to be a str but found '{0}'.".format(type(tag).__name__)) try: return self.get_tags(scope_limit)[tag] except KeyError as err: six.raise_from( KeyError("Could not find the tag '{0}' in the Symbol Table." "".format(tag)), err) def __contains__(self, key): '''Check if the given key is part of the Symbol Table. :param str key: key to check for existance. :returns: whether the Symbol Table contains the given key. :rtype: bool ''' return self._normalize(key.lower()) in self._symbols def imported_symbols(self, csymbol): ''' Examines the contents of this symbol table to see which DataSymbols (if any) are imported from the supplied ContainerSymbol (which must be present in the SymbolTable). :param csymbol: the ContainerSymbol to search for imports from. :type csymbol: :py:class:`psyclone.psyir.symbols.ContainerSymbol` :returns: list of DataSymbols that are imported from the supplied \ ContainerSymbol. If none are found then the list is empty. :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol` :raises TypeError: if the supplied object is not a ContainerSymbol. :raises KeyError: if the supplied object is not in this SymbolTable. ''' if not isinstance(csymbol, ContainerSymbol): raise TypeError( "imported_symbols() expects a ContainerSymbol but got an " "object of type '{0}'".format(type(csymbol).__name__)) # self.lookup(name) will raise a KeyError if there is no symbol with # that name in the table. if self.lookup(csymbol.name) is not csymbol: raise KeyError("The '{0}' entry in this SymbolTable is not the " "supplied ContainerSymbol.".format(csymbol.name)) return [ symbol for symbol in self.global_symbols if symbol.interface.container_symbol is csymbol ] def swap(self, old_symbol, new_symbol): ''' Remove the `old_symbol` from the table and replace it with the `new_symbol`. :param old_symbol: the symbol to remove from the table. :type old_symbol: :py:class:`psyclone.psyir.symbols.Symbol` :param new_symbol: the symbol to add to the table. :type new_symbol: :py:class:`psyclone.psyir.symbols.Symbol` :raises TypeError: if either old/new_symbol are not Symbols. :raises SymbolError: if `old_symbol` and `new_symbol` don't have \ the same name (after normalising). ''' if not isinstance(old_symbol, Symbol): raise TypeError("Symbol to remove must be of type Symbol but " "got '{0}'".format(type(old_symbol).__name__)) if not isinstance(new_symbol, Symbol): raise TypeError("Symbol to add must be of type Symbol but " "got '{0}'".format(type(new_symbol).__name__)) # The symbol table is not case sensitive so we must normalise the # symbol names before comparing them. if (self._normalize(old_symbol.name) != self._normalize( new_symbol.name)): raise SymbolError( "Cannot swap symbols that have different names, got: '{0}' " "and '{1}'".format(old_symbol.name, new_symbol.name)) # TODO #898 remove() does not currently check for any uses of # old_symbol. self.remove(old_symbol) self.add(new_symbol) def remove(self, symbol): ''' Remove the supplied symbol from the Symbol Table. This has a high potential to leave broken links, so this method checks for some references to the removed symbol depending on the symbol type. Currently, generic Symbols, ContainerSymbols and RoutineSymbols are supported. Support for removing other types of Symbol will be added as required. TODO #898. This method should check for any references/uses of the target symbol. :param symbol: the container symbol to remove. :type symbol: :py:class:`psyclone.psyir.symbols.ContainerSymbol` :raises TypeError: if the supplied symbol is not of type Symbol. :raises NotImplementedError: the removal of this symbol type is not \ supported yet. :raises KeyError: if the supplied symbol is not in the symbol table. :raises ValueError: if the supplied container symbol is referenced \ by one or more DataSymbols. :raises InternalError: if the supplied symbol is not the same as the \ entry with that name in this SymbolTable. ''' if not isinstance(symbol, Symbol): raise TypeError("remove() expects a Symbol argument but found: " "'{0}'.".format(type(symbol).__name__)) # pylint: disable=unidiomatic-typecheck if not (isinstance(symbol, (ContainerSymbol, RoutineSymbol)) or type(symbol) == Symbol): raise NotImplementedError( "remove() currently only supports generic Symbol, " "ContainerSymbol and RoutineSymbol types but got: '{0}'" "".format(type(symbol).__name__)) # pylint: enable=unidiomatic-typecheck # Since we are manipulating the _symbols dict directly we must use # the normalised name of the symbol. norm_name = self._normalize(symbol.name) if norm_name not in self._symbols: raise KeyError("Cannot remove Symbol '{0}' from symbol table " "because it does not exist.".format(symbol.name)) # Sanity-check that the entry in the table is the symbol we've # been passed. if self._symbols[norm_name] is not symbol: raise InternalError( "The Symbol with name '{0}' in this symbol table is not the " "same Symbol object as the one that has been supplied to the " "remove() method.".format(symbol.name)) # We can only remove a ContainerSymbol if no DataSymbols are # being imported from it if (isinstance(symbol, ContainerSymbol) and self.imported_symbols(symbol)): raise ValueError( "Cannot remove ContainerSymbol '{0}' because symbols " "{1} are imported from it - remove them first.".format( symbol.name, [sym.name for sym in self.imported_symbols(symbol)])) # If the symbol had a tag, it should be disassociated for tag, tagged_symbol in list(self._tags.items()): if symbol is tagged_symbol: del self._tags[tag] self._symbols.pop(norm_name) @property def argument_list(self): ''' Checks that the contents of the SymbolTable are self-consistent and then returns the list of kernel arguments. :returns: ordered list of arguments. :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol` :raises InternalError: if the entries of the SymbolTable are not \ self-consistent. ''' try: self._validate_arg_list(self._argument_list) self._validate_non_args() except ValueError as err: # If the SymbolTable is inconsistent at this point then # we have an InternalError. six.raise_from(InternalError(str(err.args)), err) return self._argument_list @staticmethod def _validate_arg_list(arg_list): ''' Checks that the supplied list of Symbols are valid kernel arguments. :param arg_list: the proposed kernel arguments. :type param_list: list of :py:class:`psyclone.psyir.symbols.DataSymbol` :raises TypeError: if any item in the supplied list is not a \ DataSymbol. :raises ValueError: if any of the symbols does not have an argument \ interface. ''' for symbol in arg_list: if not isinstance(symbol, DataSymbol): raise TypeError("Expected a list of DataSymbols but found an " "object of type '{0}'.".format(type(symbol))) if not symbol.is_argument: raise ValueError( "DataSymbol '{0}' is listed as a kernel argument but has " "an interface of type '{1}' rather than " "ArgumentInterface" "".format(str(symbol), type(symbol.interface))) def _validate_non_args(self): ''' Performs internal consistency checks on the current entries in the SymbolTable that do not represent kernel arguments. :raises ValueError: if a symbol that is not in the argument list \ has an argument interface. ''' for symbol in self.datasymbols: if symbol not in self._argument_list: # DataSymbols not in the argument list must not have a # Symbol.Argument interface if symbol.is_argument: raise ValueError( "Symbol '{0}' is not listed as a kernel argument and " "yet has an ArgumentInterface interface." "".format(str(symbol))) def get_unresolved_datasymbols(self, ignore_precision=False): ''' Create a list of the names of all of the DataSymbols in the table that do not have a resolved interface. If ignore_precision is True then those DataSymbols that are used to define the precision of other DataSymbols are ignored. If no unresolved DataSymbols are found then an empty list is returned. :param bool ignore_precision: whether or not to ignore DataSymbols \ that are used to define the precision of other DataSymbols. :returns: the names of those DataSymbols with unresolved interfaces. :rtype: list of str ''' unresolved_symbols = [ sym for sym in self.datasymbols if sym.is_unresolved ] if ignore_precision: unresolved_datasymbols = list( set(unresolved_symbols) - set(self.precision_datasymbols)) else: unresolved_datasymbols = unresolved_symbols return [sym.name for sym in unresolved_datasymbols] @property def symbols_dict(self): ''' :returns: ordered dictionary of symbols indexed by symbol name. :rtype: OrderedDict[str] = :py:class:`psyclone.psyir.symbols.Symbol` ''' return self._symbols @property def tags_dict(self): ''' :returns: ordered dictionary of symbols indexed by tag. :rtype: OrderedDict[str] = :py:class:`psyclone.psyir.symbols.Symbol` ''' return self._tags @property def symbols(self): ''' :returns: list of symbols. :rtype: list of :py:class:`psyclone.psyir.symbols.Symbol` ''' return list(self._symbols.values()) @property def datasymbols(self): ''' :returns: list of symbols representing data variables. :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol` ''' return [ sym for sym in self._symbols.values() if isinstance(sym, DataSymbol) ] @property def local_datasymbols(self): ''' :returns: list of symbols representing local variables. :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol` ''' return [sym for sym in self.datasymbols if sym.is_local] @property def argument_datasymbols(self): ''' :returns: list of symbols representing arguments. :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol` ''' return [sym for sym in self.datasymbols if sym.is_argument] @property def global_symbols(self): ''' :returns: list of symbols that have 'global' interface (are \ associated with data that exists outside the current scope). :rtype: list of :py:class:`psyclone.psyir.symbols.Symbol` ''' return [sym for sym in self.symbols if sym.is_global] @property def precision_datasymbols(self): ''' :returns: list of all symbols used to define the precision of \ other symbols within the table. :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol` ''' # Accumulate into a set so as to remove any duplicates precision_symbols = set() for sym in self.datasymbols: # Not all types have the 'precision' attribute (e.g. DeferredType) if (hasattr(sym.datatype, "precision") and isinstance(sym.datatype.precision, DataSymbol)): precision_symbols.add(sym.datatype.precision) return list(precision_symbols) @property def containersymbols(self): ''' :returns: a list of the ContainerSymbols present in the Symbol Table. :rtype: list of :py:class:`psyclone.psyir.symbols.ContainerSymbol` ''' return [ sym for sym in self.symbols if isinstance(sym, ContainerSymbol) ] @property def local_typesymbols(self): ''' :returns: the local TypeSymbols present in the Symbol Table. :rtype: list of :py:class:`psyclone.psyir.symbols.TypeSymbol` ''' return [ sym for sym in self.symbols if (isinstance(sym, TypeSymbol) and sym.is_local) ] @property def iteration_indices(self): ''' :returns: list of symbols representing kernel iteration indices. :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol` :raises NotImplementedError: this method is abstract. ''' raise NotImplementedError( "Abstract property. Which symbols are iteration indices is" " API-specific.") @property def data_arguments(self): ''' :returns: list of symbols representing kernel data arguments. :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol` :raises NotImplementedError: this method is abstract. ''' raise NotImplementedError( "Abstract property. Which symbols are data arguments is" " API-specific.") def copy_external_global(self, globalvar, tag=None): ''' Copy the given global variable (and its referenced ContainerSymbol if needed) into the SymbolTable. :param globalvar: the variable to be copied in. :type globalvar: :py:class:`psyclone.psyir.symbols.DataSymbol` :param str tag: a tag identifier for the new copy, by default no tag \ is given. :raises TypeError: if the given variable is not a global variable. :raises KeyError: if the given variable name already exists in the \ symbol table. ''' if not isinstance(globalvar, DataSymbol): raise TypeError( "The globalvar argument of SymbolTable.copy_external_global" " method should be a DataSymbol, but found '{0}'." "".format(type(globalvar).__name__)) if not globalvar.is_global: raise TypeError( "The globalvar argument of SymbolTable.copy_external_" "global method should have a GlobalInterface interface, " "but found '{0}'.".format(type(globalvar.interface).__name__)) external_container_name = globalvar.interface.container_symbol.name # If the Container is not yet in the SymbolTable we need to # create one and add it. if external_container_name not in self: self.add(ContainerSymbol(external_container_name)) container_ref = self.lookup(external_container_name) # Copy the variable into the SymbolTable with the appropriate interface if globalvar.name not in self: new_symbol = globalvar.copy() # Update the interface of this new symbol new_symbol.interface = GlobalInterface(container_ref) self.add(new_symbol, tag) else: # If it already exists it must refer to the same Container and have # the same tag. local_instance = self.lookup(globalvar.name) if not (local_instance.is_global and local_instance.interface.container_symbol.name == external_container_name): raise KeyError("Couldn't copy '{0}' into the SymbolTable. The" " name '{1}' is already used by another symbol." "".format(globalvar, globalvar.name)) if tag: # If the symbol already exists and a tag is provided try: self.lookup_with_tag(tag) except KeyError: # If the tag was not used, it will now be attached # to the symbol. self._tags[tag] = self.lookup(globalvar.name) # The tag should not refer to a different symbol if self.lookup(globalvar.name) != self.lookup_with_tag(tag): raise KeyError( "Couldn't copy '{0}' into the SymbolTable. The" " tag '{1}' is already used by another symbol." "".format(globalvar, tag)) def rename_symbol(self, symbol, name): ''' Rename the given symbol which should belong to this symbol table with the new name provided. :param symbol: the symbol to be renamed. :type symbol: :py:class:`psyclone.psyir.symbols.Symbol` :param str name: the new name. :raises TypeError: if the symbol is not a Symbol. :raises TypeError: if the name is not a str. :raises ValueError: if the given symbol does not belong to this \ symbol table. :raises KeyError: if the given variable name already exists in the \ symbol table. ''' if not isinstance(symbol, Symbol): raise TypeError( "The symbol argument of rename_symbol() must be a Symbol, but" " found: '{0}'.".format(type(symbol).__name__)) if symbol not in self.symbols: raise ValueError( "The symbol argument of rename_symbol() must belong to this " "symbol_table instance, but '{0}' does not.".format(symbol)) if not isinstance(name, str): raise TypeError( "The name argument of rename_symbol() must be a str, but" " found: '{0}'.".format(type(symbol).__name__)) if name in self._symbols: raise KeyError( "The name argument of rename_symbol() must not already exist " "in this symbol_table instance, but '{0}' does.".format(name)) # Delete current dictionary entry del self._symbols[symbol.name] # Rename symbol using protected access as the Symbol class should not # expose a name attribute setter. # pylint: disable=protected-access symbol._name = name # Re-insert modified symbol self.add(symbol) def has_wildcard_imports(self): ''' Searches this symbol table and then up through any parent symbol tables for a ContainerSymbol that has a wildcard import. :returns: True if a wildcard import is found, False otherwise. :rtype: bool ''' current_table = self while current_table: for sym in current_table.containersymbols: if sym.wildcard_import: return True current_table = current_table.parent_symbol_table() return False def view(self): ''' Print a representation of this Symbol Table to stdout. ''' print(str(self)) def __str__(self): return ("Symbol Table:\n" + "\n".join(map(str, self._symbols.values())) + "\n")
class Data(object): """The basic data container in Glue. The data object stores data as a collection of :class:`~glue.core.component.Component` objects. Each component stored in a dataset must have the same shape. Catalog data sets are stored such that each column is a distinct 1-dimensional :class:`~glue.core.component.Component`. There are several ways to extract the actual numerical data stored in a :class:`~glue.core.data.Data` object:: data = Data(x=[1, 2, 3], label='data') xid = data.id['x'] data[xid] data.get_component(xid).data data['x'] # if 'x' is a unique component name Likewise, datasets support :ref:`fancy indexing <numpy:basics.indexing>`:: data[xid, 0:2] data[xid, [True, False, True]] See also: :ref:`data_tutorial` """ def __init__(self, label="", coords=None, **kwargs): """ :param label: label for data :type label: str Extra array-like keywords are extracted into components """ # Coordinate conversion object self.coords = coords or Coordinates() self._shape = () # Components self._components = OrderedDict() self._externally_derivable_components = OrderedDict() self._pixel_aligned_data = OrderedDict() self._pixel_component_ids = ComponentIDList() self._world_component_ids = ComponentIDList() self.id = ComponentIDDict(self) # Metadata self.meta = OrderedDict() # Subsets of the data self._subsets = [] # Hub that the data is attached to self.hub = None self.style = VisualAttributes(parent=self) self._coordinate_links = [] self.data = self self.label = label self.edit_subset = None for lbl, data in sorted(kwargs.items()): self.add_component(data, lbl) self._key_joins = {} # To avoid circular references when saving objects with references to # the data, we make sure that all Data objects have a UUID that can # uniquely identify them. self.uuid = str(uuid.uuid4()) @property def subsets(self): """ Tuple of subsets attached to this dataset """ return tuple(self._subsets) @property def ndim(self): """ Dimensionality of the dataset """ return len(self.shape) @property def shape(self): """ Tuple of array dimensions, like :attr:`numpy.ndarray.shape` """ return self._shape @property def label(self): """ Convenience access to data set's label """ return self._label @label.setter def label(self, value): """ Set the label to value """ self._label = value self.broadcast(attribute='label') @property def size(self): """ Total number of elements in the dataset. """ return np.product(self.shape) @contract(component=Component) def _check_can_add(self, component): if isinstance(component, DerivedComponent): return component._data is self else: if len(self._components) == 0: return True else: if all(comp.shape == () for comp in self._components.values()): return True else: return component.shape == self.shape @contract(cid=ComponentID, returns=np.dtype) def dtype(self, cid): """Lookup the dtype for the data associated with a ComponentID""" # grab a small piece of data ind = tuple([slice(0, 1)] * self.ndim) arr = self[cid, ind] return arr.dtype @contract(component_id=ComponentID) def remove_component(self, component_id): """ Remove a component from a data set :param component_id: the component to remove :type component_id: :class:`~glue.core.component_id.ComponentID` """ # TODO: avoid too many messages when removing a component triggers # the removal of derived components. if component_id in self._components: self._components.pop(component_id) self._removed_derived_that_depend_on(component_id) if self.hub: msg = DataRemoveComponentMessage(self, component_id) self.hub.broadcast(msg) msg = ComponentsChangedMessage(self) self.hub.broadcast(msg) def _removed_derived_that_depend_on(self, component_id): """ Remove internal derived components that can no longer be derived. """ remove = [] for cid in self.derived_components: comp = self.get_component(cid) if component_id in comp.link.get_from_ids(): remove.append(cid) for cid in remove: self.remove_component(cid) @contract(other='isinstance(Data)', cid='cid_like', cid_other='cid_like') def join_on_key(self, other, cid, cid_other): """ Create an *element* mapping to another dataset, by joining on values of ComponentIDs in both datasets. This join allows any subsets defined on `other` to be propagated to self. The different ways to call this method are described in the **Examples** section below. Parameters ---------- other : :class:`~glue.core.data.Data` Data object to join with cid : str or :class:`~glue.core.component_id.ComponentID` or iterable Component(s) in this dataset to use as a key cid_other : str or :class:`~glue.core.component_id.ComponentID` or iterable Component(s) in the other dataset to use as a key Examples -------- There are several ways to use this function, depending on how many components are passed to ``cid`` and ``cid_other``. **Joining on single components** First, one can specify a single component ID for both ``cid`` and ``cid_other``: this is the standard mode, and joins one component from one dataset to the other: >>> d1 = Data(x=[1, 2, 3, 4, 5], k1=[0, 0, 1, 1, 2], label='d1') >>> d2 = Data(y=[2, 4, 5, 8, 4], k2=[1, 3, 1, 2, 3], label='d2') >>> d2.join_on_key(d1, 'k2', 'k1') Selecting all values in ``d1`` where x is greater than 2 returns the last three items as expected: >>> s = d1.new_subset() >>> s.subset_state = d1.id['x'] > 2 >>> s.to_mask() array([False, False, True, True, True], dtype=bool) The linking was done between k1 and k2, and the values of k1 for the last three items are 1 and 2 - this means that the first, third, and fourth item in ``d2`` will then get selected, since k2 has a value of either 1 or 2 for these items. >>> s = d2.new_subset() >>> s.subset_state = d1.id['x'] > 2 >>> s.to_mask() array([ True, False, True, True, False], dtype=bool) **Joining on multiple components** .. note:: This mode is currently slow, and will be optimized significantly in future. Next, one can specify several components for each dataset: in this case, the number of components given should match for both datasets. This causes items in both datasets to be linked when (and only when) the set of keys match between the two datasets: >>> d1 = Data(x=[1, 2, 3, 5, 5], ... y=[0, 0, 1, 1, 2], label='d1') >>> d2 = Data(a=[2, 5, 5, 8, 4], ... b=[1, 3, 2, 2, 3], label='d2') >>> d2.join_on_key(d1, ('a', 'b'), ('x', 'y')) Selecting all items where x is 5 in ``d1`` in which x is a component works as expected and selects the two last items:: >>> s = d1.new_subset() >>> s.subset_state = d1.id['x'] == 5 >>> s.to_mask() array([False, False, False, True, True], dtype=bool) If we apply this selection to ``d2``, only items where a is 5 and b is 2 will be selected: >>> s = d2.new_subset() >>> s.subset_state = d1.id['x'] == 5 >>> s.to_mask() array([False, False, True, False, False], dtype=bool) and in particular, the second item (where a is 5 and b is 3) is not selected. **One-to-many and many-to-one joining** Finally, you can specify one component in one dataset and multiple ones in the other. In the case where one component is specified for this dataset and multiple ones for the other dataset, then when an item is selected in the other dataset, it will cause any item in the present dataset which matches any of the keys in the other data to be selected: >>> d1 = Data(x=[1, 2, 3], label='d1') >>> d2 = Data(a=[1, 1, 2], ... b=[2, 3, 3], label='d2') >>> d1.join_on_key(d2, 'x', ('a', 'b')) In this case, if we select all items in ``d2`` where a is 2, this will select the third item: >>> s = d2.new_subset() >>> s.subset_state = d2.id['a'] == 2 >>> s.to_mask() array([False, False, True], dtype=bool) Since we have joined the datasets using both a and b, we select all items in ``d1`` where x is either the value or a or b (2 or 3) which means we select the second and third item: >>> s = d1.new_subset() >>> s.subset_state = d2.id['a'] == 2 >>> s.to_mask() array([False, True, True], dtype=bool) We can also join the datasets the other way around: >>> d1 = Data(x=[1, 2, 3], label='d1') >>> d2 = Data(a=[1, 1, 2], ... b=[2, 3, 3], label='d2') >>> d2.join_on_key(d1, ('a', 'b'), 'x') In this case, selecting items in ``d1`` where x is 1 selects the first item, as expected: >>> s = d1.new_subset() >>> s.subset_state = d1.id['x'] == 1 >>> s.to_mask() array([ True, False, False], dtype=bool) This then causes any item in ``d2`` where either a or b are 1 to be selected, i.e. the first two items: >>> s = d2.new_subset() >>> s.subset_state = d1.id['x'] == 1 >>> s.to_mask() array([ True, True, False], dtype=bool) """ # To make things easier, we transform all component inputs to a tuple if isinstance(cid, six.string_types) or isinstance(cid, ComponentID): cid = (cid, ) if isinstance(cid_other, six.string_types) or isinstance( cid_other, ComponentID): cid_other = (cid_other, ) if len(cid) > 1 and len(cid_other) > 1 and len(cid) != len(cid_other): raise Exception("Either the number of components in the key join " "sets should match, or one of the component sets " "should contain a single component.") def get_component_id(data, name): if isinstance(name, ComponentID): return name else: cid = data.find_component_id(name) if cid is None: raise ValueError("ComponentID not found in %s: %s" % (data.label, name)) return cid cid = tuple(get_component_id(self, name) for name in cid) cid_other = tuple(get_component_id(other, name) for name in cid_other) self._key_joins[other] = (cid, cid_other) other._key_joins[self] = (cid_other, cid) @contract(component='component_like', label='cid_like') def add_component(self, component, label): """ Add a new component to this data set. :param component: object to add. Can be a Component, array-like object, or ComponentLink :param label: The label. If this is a string, a new :class:`glue.core.component_id.ComponentID` with this label will be created and associated with the Component :type component: :class:`~glue.core.component.Component` or array-like :type label: :class:`str` or :class:`~glue.core.component_id.ComponentID` :raises: TypeError, if label is invalid ValueError if the component has an incompatible shape :returns: The ComponentID associated with the newly-added component """ if isinstance(component, ComponentLink): return self.add_component_link(component, label=label) if not isinstance(component, Component): component = Component.autotyped(component) if isinstance(component, DerivedComponent): if len(self._components) == 0: raise TypeError( "Cannot add a derived component as a first component") component.set_parent(self) if not (self._check_can_add(component)): raise ValueError("The dimensions of component %s are " "incompatible with the dimensions of this data: " "%r vs %r" % (label, component.shape, self.shape)) if isinstance(label, ComponentID): component_id = label if component_id.parent is None: component_id.parent = self else: component_id = ComponentID(label, parent=self) if len(self._components) == 0: # TODO: make sure the following doesn't raise a componentsraised message self._create_pixel_and_world_components(ndim=component.ndim) # In some cases, such as when loading a session, we actually disable the # auto-creation of pixel and world coordinates, so the first component # may be a coordinate component with no shape. Therefore we only set the # shape once a component has a valid shape rather than strictly on the # first component. if self._shape == () and component.shape != (): self._shape = component.shape is_present = component_id in self._components self._components[component_id] = component if self.hub and not is_present: msg = DataAddComponentMessage(self, component_id) self.hub.broadcast(msg) msg = ComponentsChangedMessage(self) self.hub.broadcast(msg) return component_id def _set_externally_derivable_components(self, derivable_components): """ Externally deriable components are components identified by component IDs from other datasets. This method is meant for internal use only and is called by the link manager. The ``derivable_components`` argument should be set to a dictionary where the keys are the derivable component IDs, and the values are DerivedComponent instances which can be used to get the data. """ if len(self._externally_derivable_components) == 0 and len( derivable_components) == 0: return self._externally_derivable_components = derivable_components if self.hub: msg = ExternallyDerivableComponentsChangedMessage(self) self.hub.broadcast(msg) def _set_pixel_aligned_data(self, pixel_aligned_data): """ Pixel-aligned data are datasets that contain pixel component IDs that are equivalent (identically, not transformed) with all pixel component IDs in the present dataset. Note that the other datasets may have more but not fewer dimensions, so this information may not be symmetric between datasets with differing numbers of dimensions. """ # First check if anything has changed, as if not then we should just # leave things as-is and avoid emitting a message. if len(self._pixel_aligned_data) == len(pixel_aligned_data): for data in self._pixel_aligned_data: if data not in pixel_aligned_data or pixel_aligned_data[ data] != self._pixel_aligned_data[data]: break else: return self._pixel_aligned_data = pixel_aligned_data if self.hub: msg = PixelAlignedDataChangedMessage(self) self.hub.broadcast(msg) @property def pixel_aligned_data(self): """ Information about other datasets in the same data collection that have matching or a subset of pixel component IDs. This is returned as a dictionary where each key is a dataset with matching pixel component IDs, and the value is the order in which the pixel component IDs of the other dataset can be found in the current one. """ return self._pixel_aligned_data @contract(link=ComponentLink, label='cid_like|None', returns=DerivedComponent) def add_component_link(self, link, label=None): """ Shortcut method for generating a new :class:`~glue.core.component.DerivedComponent` from a ComponentLink object, and adding it to a data set. Parameters ---------- link : :class:`~glue.core.component_link.ComponentLink` The link to use to generate a new component label : :class:`~glue.core.component_id.ComponentID` or str The ComponentID or label to attach to. Returns ------- component : :class:`~glue.core.component.DerivedComponent` The component that was added """ if label is not None: if not isinstance(label, ComponentID): label = ComponentID(label, parent=self) link.set_to_id(label) if link.get_to_id() is None: raise TypeError("Cannot add component_link: " "has no 'to' ComponentID") for cid in link.get_from_ids(): if cid not in self.components: raise ValueError( "Can only add internal links with add_component_link " "- use DataCollection.add_link to add inter-data links") dc = DerivedComponent(self, link) to_ = link.get_to_id() self.add_component(dc, label=to_) return dc def _create_pixel_and_world_components(self, ndim): for i in range(ndim): comp = CoordinateComponent(self, i) label = pixel_label(i, ndim) cid = PixelComponentID(i, "Pixel Axis %s" % label, parent=self) self.add_component(comp, cid) self._pixel_component_ids.append(cid) if self.coords: for i in range(ndim): comp = CoordinateComponent(self, i, world=True) label = self.coords.axis_label(i) cid = self.add_component(comp, label) self._world_component_ids.append(cid) self._set_up_coordinate_component_links(ndim) def _set_up_coordinate_component_links(self, ndim): def make_toworld_func(i): def pix2world(*args): return self.coords.pixel2world_single_axis(*args[::-1], axis=ndim - 1 - i) return pix2world def make_topixel_func(i): def world2pix(*args): return self.coords.world2pixel_single_axis(*args[::-1], axis=ndim - 1 - i) return world2pix result = [] for i in range(ndim): link = CoordinateComponentLink(self._pixel_component_ids, self._world_component_ids[i], self.coords, i) result.append(link) link = CoordinateComponentLink(self._world_component_ids, self._pixel_component_ids[i], self.coords, i, pixel2world=False) result.append(link) self._coordinate_links = result return result @property def components(self): """All :class:`ComponentIDs <glue.core.component_id.ComponentID>` in the Data :rtype: list """ return list(self._components.keys()) @property def externally_derivable_components(self): return list(self._externally_derivable_components.keys()) @property def visible_components(self): """All :class:`ComponentIDs <glue.core.component_id.ComponentID>` in the Data that aren't coordinate. :rtype: list """ return [ cid for cid, comp in self._components.items() if not isinstance(comp, CoordinateComponent) and cid.parent is self ] @property def coordinate_components(self): """The ComponentIDs associated with a :class:`~glue.core.component.CoordinateComponent` :rtype: list """ return [ c for c in self.component_ids() if isinstance(self._components[c], CoordinateComponent) ] @property def main_components(self): return [ c for c in self.component_ids() if not isinstance(self._components[c], (DerivedComponent, CoordinateComponent)) ] @property def primary_components(self): """The ComponentIDs not associated with a :class:`~glue.core.component.DerivedComponent` :rtype: list """ return [ c for c in self.component_ids() if not isinstance(self._components[c], DerivedComponent) ] @property def derived_components(self): """The ComponentIDs for each :class:`~glue.core.component.DerivedComponent` :rtype: list """ return [ c for c in self.component_ids() if isinstance(self._components[c], DerivedComponent) ] @property def pixel_component_ids(self): """ The :class:`ComponentIDs <glue.core.component_id.ComponentID>` for each pixel coordinate. """ return self._pixel_component_ids @property def world_component_ids(self): """ The :class:`ComponentIDs <glue.core.component_id.ComponentID>` for each world coordinate. """ return self._world_component_ids @contract(label='cid_like', returns='inst($ComponentID)|None') def find_component_id(self, label): """ Retrieve component_ids associated by label name. :param label: ComponentID or string to search for :returns: The associated ComponentID if label is found and unique, else None. First, this checks whether the component ID is present and unique in the primary (non-derived) components of the data, and if not then the derived components are checked. If there is one instance of the label in the primary and one in the derived components, the primary one takes precedence. """ for cid_set in (self.primary_components, self.derived_components, self.coordinate_components, list(self._externally_derivable_components)): result = [] for cid in cid_set: if isinstance(label, ComponentID): if cid is label: result.append(cid) else: if cid.label == label: result.append(cid) if len(result) == 1: return result[0] elif len(result) > 1: return None return None @property def links(self): """ A list of all the links internal to the dataset. """ return self.coordinate_links + self.derived_links @property def coordinate_links(self): """ A list of the ComponentLinks that connect pixel and world. If no coordinate transformation object is present, return an empty list. """ return self._coordinate_links @property def derived_links(self): """ A list of the links present inside all of the DerivedComponent objects in this dataset. """ return [ self.get_component(cid).link for cid in self.derived_components ] @contract(axis=int, returns=ComponentID) def get_pixel_component_id(self, axis): """Return the pixel :class:`glue.core.component_id.ComponentID` associated with a given axis """ return self._pixel_component_ids[axis] @contract(axis=int, returns=ComponentID) def get_world_component_id(self, axis): """Return the world :class:`glue.core.component_id.ComponentID` associated with a given axis """ return self._world_component_ids[axis] @contract(returns='list(inst($ComponentID))') def component_ids(self): """ Equivalent to :attr:`Data.components` """ return ComponentIDList(self._components.keys()) @contract(subset='isinstance(Subset)|None', color='color|None', label='string|None', returns=Subset) def new_subset(self, subset=None, color=None, label=None, **kwargs): """ Create a new subset, and attach to self. .. note:: The preferred way for creating subsets is via :meth:`~glue.core.data_collection.DataCollection.new_subset_group`. Manually-instantiated subsets will **not** be represented properly by the UI :param subset: optional, reference subset or subset state. If provided, the new subset will copy the logic of this subset. :returns: The new subset object """ nsub = len(self.subsets) color = color or settings.SUBSET_COLORS[nsub % len(settings.SUBSET_COLORS)] label = label or "%s.%i" % (self.label, nsub + 1) new_subset = Subset(self, color=color, label=label, **kwargs) if subset is not None: new_subset.subset_state = subset.subset_state.copy() self.add_subset(new_subset) return new_subset @contract(subset='inst($Subset, $SubsetState)') def add_subset(self, subset): """Assign a pre-existing subset to this data object. :param subset: A :class:`~glue.core.subset.Subset` or :class:`~glue.core.subset.SubsetState` object If input is a :class:`~glue.core.subset.SubsetState`, it will be wrapped in a new Subset automatically .. note:: The preferred way for creating subsets is via :meth:`~glue.core.data_collection.DataCollection.new_subset_group`. Manually-instantiated subsets will **not** be represented properly by the UI """ if subset in self.subsets: return # prevents infinite recursion if isinstance(subset, SubsetState): # auto-wrap state in subset state = subset subset = Subset(None) subset.subset_state = state self._subsets.append(subset) if subset.data is not self: subset.do_broadcast(False) subset.data = self subset.label = subset.label # hacky. disambiguates name if needed if self.hub is not None: msg = SubsetCreateMessage(subset) self.hub.broadcast(msg) subset.do_broadcast(True) @contract(hub=Hub) def register_to_hub(self, hub): """ Connect to a hub. This method usually doesn't have to be called directly, as DataCollections manage the registration of data objects """ if not isinstance(hub, Hub): raise TypeError("input is not a Hub object: %s" % type(hub)) self.hub = hub @contract(attribute='string') def broadcast(self, attribute): """ Send a :class:`~glue.core.message.DataUpdateMessage` to the hub :param attribute: Name of an attribute that has changed (or None) :type attribute: str """ if not self.hub: return msg = DataUpdateMessage(self, attribute=attribute) self.hub.broadcast(msg) @contract(old=ComponentID, new=ComponentID) def update_id(self, old, new): """ Reassign a component to a different :class:`glue.core.component_id.ComponentID` Parameters ---------- old : :class:`glue.core.component_id.ComponentID` The old component ID new : :class:`glue.core.component_id.ComponentID` The new component ID """ if new is old: return if new.parent is None: new.parent = self changed = False if old in self._components: # We want to keep the original order, so we can't just do: # self._components[new] = self._components[old] # which will put the new component ID at the end, but instead # we need to do: self._components = OrderedDict( (new, value) if key is old else (key, value) for key, value in self._components.items()) changed = True try: index = self._pixel_component_ids.index(old) self._pixel_component_ids[index] = new changed = True except ValueError: pass try: index = self._world_component_ids.index(old) self._world_component_ids[index] = new changed = True except ValueError: pass if changed and self.hub is not None: # remove old component and broadcast the change # see #508 for discussion of this msg = ComponentReplacedMessage(self, old, new) self.hub.broadcast(msg) def __str__(self): s = "Data Set: %s\n" % self.label s += "Number of dimensions: %i\n" % self.ndim s += "Shape: %s\n" % ' x '.join([str(x) for x in self.shape]) categories = [('Main', self.main_components), ('Derived', self.derived_components), ('Coordinate', self.coordinate_components)] for category, components in categories: if len(components) > 0: s += category + " components:\n" for cid in components: comp = self.get_component(cid) if comp.units is None or comp.units == '': s += " - {0}\n".format(cid) else: s += " - {0} [{1}]\n".format(cid, comp.units) return s[:-1] def __repr__(self): return 'Data (label: %s)' % self.label def __setattr__(self, name, value): if name == "hub" and hasattr(self, 'hub') \ and self.hub is not value and self.hub is not None: raise AttributeError("Data has already been assigned " "to a different hub") object.__setattr__(self, name, value) def __getitem__(self, key): """ Shortcut syntax to access the numerical data in a component. Equivalent to: ``component = data.get_component(component_id).data`` :param key: The component to fetch data from :type key: :class:`~glue.core.component_id.ComponentID` :returns: :class:`~numpy.ndarray` """ key, view = split_component_view(key) if isinstance(key, six.string_types): _k = key key = self.find_component_id(key) if key is None: raise IncompatibleAttribute(_k) if isinstance(key, ComponentLink): return key.compute(self, view) if key in self._components: comp = self._components[key] elif key in self._externally_derivable_components: comp = self._externally_derivable_components[key] else: raise IncompatibleAttribute(key) if view is not None: result = comp[view] else: if comp.categorical: result = comp.codes else: result = comp.data return result def __setitem__(self, key, value): """ Wrapper for data.add_component() """ self.add_component(value, key) @contract(component_id='cid_like|None', returns=Component) def get_component(self, component_id): """Fetch the component corresponding to component_id. :param component_id: the component_id to retrieve """ if component_id is None: raise IncompatibleAttribute() if isinstance(component_id, six.string_types): component_id = self.id[component_id] if component_id in self._components: return self._components[component_id] elif component_id in self._externally_derivable_components: return self._externally_derivable_components[component_id] else: raise IncompatibleAttribute(component_id) def to_dataframe(self, index=None): """ Convert the Data object into a pandas.DataFrame object :param index: Any 'index-like' object that can be passed to the pandas.Series constructor :return: pandas.DataFrame """ h = lambda comp: self.get_component(comp).to_series(index=index) df = pd.DataFrame( dict((comp.label, h(comp)) for comp in self.components)) order = [comp.label for comp in self.components] return df[order] def reorder_components(self, component_ids): """ Reorder the components using a list of component IDs. The new set of component IDs has to match the existing set (though order may differ). """ # We need to be careful because component IDs overload == so we can't # use the normal ways to test whether the component IDs are the same # as self.components - instead we need to explicitly use id if len(component_ids) != len(self.components): raise ValueError("Number of component in component_ids does not " "match existing number of components") if set(id(c) for c in self.components) != set(id(c) for c in component_ids): raise ValueError( "specified component_ids should match existing components") existing = self.components for idx in range(len(component_ids)): if component_ids[idx] is not existing[idx]: break else: # If we get here then the suggested order is the same as the existing one return # PY3: once we drop support for Python 2 we could sort in-place using # the move_to_end method on OrderedDict self._components = OrderedDict( (key, self._components[key]) for key in component_ids) if self.hub: msg = DataReorderComponentMessage(self, list(self._components)) self.hub.broadcast(msg) @contract(mapping="dict(inst($Component, $ComponentID):array_like)") def update_components(self, mapping): """ Change the numerical data associated with some of the Components in this Data object. All changes to component numerical data should use this method, which broadcasts the state change to the appropriate places. :param mapping: A dict mapping Components or ComponenIDs to arrays. This method has the following restrictions: - New compoments must have the same shape as old compoments - Component subclasses cannot be updated. """ for comp, data in mapping.items(): if isinstance(comp, ComponentID): comp = self.get_component(comp) data = np.asarray(data) if data.shape != self.shape: raise ValueError("Cannot change shape of data") comp._data = data # alert hub of the change if self.hub is not None: msg = NumericalDataChangedMessage(self) self.hub.broadcast(msg) for subset in self.subsets: clear_cache(subset.subset_state.to_mask) def update_values_from_data(self, data): """ Replace numerical values in data to match values from another dataset. Notes ----- This method drops components that aren't present in the new data, and adds components that are in the new data that were not in the original data. The matching is done by component label, and components are resized if needed. This means that for components with matching labels in the original and new data, the :class:`~glue.core.component_id.ComponentID` are preserved, and existing plots and selections will be updated to reflect the new values. Note that the coordinates are also copied, but the style is **not** copied. """ old_labels = [cid.label for cid in self.components] new_labels = [cid.label for cid in data.components] if len(old_labels) == len(set(old_labels)): old_labels = set(old_labels) else: raise ValueError("Non-unique component labels in original data") if len(new_labels) == len(set(new_labels)): new_labels = set(new_labels) else: raise ValueError("Non-unique component labels in new data") # Remove components that don't have a match in new data for cname in old_labels - new_labels: cid = self.find_component_id(cname) self.remove_component(cid) # Update shape self._shape = data._shape # Update components that exist in both. Note that we can't just loop # over old_labels & new_labels since we need to make sure we preserve # the order of the components, and sets don't preserve order. for cid in self.components: cname = cid.label if cname in old_labels & new_labels: comp_old = self.get_component(cname) comp_new = data.get_component(cname) comp_old._data = comp_new._data # Add components that didn't exist in original one. As above, we try # and preserve the order of components as much as possible. for cid in data.components: cname = cid.label if cname in new_labels - old_labels: cid = data.find_component_id(cname) comp_new = data.get_component(cname) self.add_component(comp_new, cid.label) # Update data label self.label = data.label # Update data coordinates self.coords = data.coords # alert hub of the change if self.hub is not None: msg = NumericalDataChangedMessage(self) self.hub.broadcast(msg) for subset in self.subsets: clear_cache(subset.subset_state.to_mask)
async def cmd_player_compare(ctx): bot = ctx.bot args = ctx.args config = ctx.config ctx.alt = bot.options.parse_alt(args) lang = bot.options.parse_lang(ctx, args) selected_players, error = bot.options.parse_players(ctx, args) selected_units = bot.options.parse_unit_names(args) if error: return error if args: return bot.errors.unknown_parameters(args) if not selected_players: return bot.errors.no_ally_code_specified(ctx) if not selected_units: return bot.errors.no_unit_selected(ctx) ally_codes = [ x.ally_code for x in selected_players ] players = await bot.client.players(ally_codes=ally_codes, units=selected_units, stats=True) players = { x['allyCode']: x for x in players } msgs = [] for unit in selected_units: units = [] fields = OrderedDict() for player in selected_players: ally_code = player.ally_code jplayer = players[ally_code] jroster = { x['defId']: x for x in jplayer['roster'] } units.append(unit_to_dict(config, jplayer, jroster, unit.base_id, lang)) for someunit in units: for key, val in someunit.items(): if key not in fields: fields[key] = [] fields[key].append(val) lines = [] key = 'Unit still locked' if key in fields: listval = fields.pop(key) pad = 1 lines.append('**`%s`**`:%s%s`' % (key, pad * '\u00a0', ' | '.join(listval))) if fields: lines.append('') first_time = True for key, listval in fields.items(): if key in base_stats: newlistval = [] for item in listval: pad = 0 aval = item if len(item) < 7 and key != 'Players': pad = max(0, 6 - len(item)) aval = '%s%s ' % (pad * '\u00a0', item) newlistval.append(aval) if key == 'Players': lines.append('**Stats**') lines.append(config['separator']) key_string = '**`%s`**' % key if key in [ 'Players', 'Stars' ]: key_string = '' lines.append('`|%s|`%s' % ('|'.join(newlistval), key_string)) else: if first_time: first_time = False lines.append(config['separator']) lines.append('') lines.append('**Skills**') lines.append(config['separator']) ability = listval[0]['name'] tiers = [ x['tier'] for x in listval ] lines.append('`|%s|`**`%s`**' % ('|'.join(tiers), ability)) msgs.append({ 'author': { 'name': unit.name, 'icon_url': unit.get_image(), }, 'description': '\n'.join(lines), }) return msgs
class LoadScopeScheduling(object): """Implement load scheduling across nodes, but grouping test by scope. This distributes the tests collected across all nodes so each test is run just once. All nodes collect and submit the list of tests and when all collections are received it is verified they are identical collections. Then the collection gets divided up in work units, grouped by test scope, and those work units get submitted to nodes. Whenever a node finishes an item, it calls ``.mark_test_complete()`` which will trigger the scheduler to assign more work units if the number of pending tests for the node falls below a low-watermark. When created, ``numnodes`` defines how many nodes are expected to submit a collection. This is used to know when all nodes have finished collection. Attributes: :numnodes: The expected number of nodes taking part. The actual number of nodes will vary during the scheduler's lifetime as nodes are added by the DSession as they are brought up and removed either because of a dead node or normal shutdown. This number is primarily used to know when the initial collection is completed. :collection: The final list of tests collected by all nodes once it is validated to be identical between all the nodes. It is initialised to None until ``.schedule()`` is called. :workqueue: Ordered dictionary that maps all available scopes with their associated tests (nodeid). Nodeids are in turn associated with their completion status. One entry of the workqueue is called a work unit. In turn, a collection of work unit is called a workload. :: workqueue = { '<full>/<path>/<to>/test_module.py': { '<full>/<path>/<to>/test_module.py::test_case1': False, '<full>/<path>/<to>/test_module.py::test_case2': False, (...) }, (...) } :assigned_work: Ordered dictionary that maps worker nodes with their assigned work units. :: assigned_work = { '<worker node A>': { '<full>/<path>/<to>/test_module.py': { '<full>/<path>/<to>/test_module.py::test_case1': False, '<full>/<path>/<to>/test_module.py::test_case2': False, (...) }, (...) }, (...) } :registered_collections: Ordered dictionary that maps worker nodes with their collection of tests gathered during test discovery. :: registered_collections = { '<worker node A>': [ '<full>/<path>/<to>/test_module.py::test_case1', '<full>/<path>/<to>/test_module.py::test_case2', ], (...) } :log: A py.log.Producer instance. :config: Config object, used for handling hooks. """ def __init__(self, config, log=None): self.numnodes = len(parse_spec_config(config)) self.collection = None self.workqueue = OrderedDict() self.assigned_work = OrderedDict() self.registered_collections = OrderedDict() if log is None: self.log = Producer("loadscopesched") else: self.log = log.loadscopesched self.config = config @property def nodes(self): """A list of all active nodes in the scheduler.""" return list(self.assigned_work.keys()) @property def collection_is_completed(self): """Boolean indication initial test collection is complete. This is a boolean indicating all initial participating nodes have finished collection. The required number of initial nodes is defined by ``.numnodes``. """ return len(self.registered_collections) >= self.numnodes @property def tests_finished(self): """Return True if all tests have been executed by the nodes.""" if not self.collection_is_completed: return False if self.workqueue: return False for assigned_unit in self.assigned_work.values(): if self._pending_of(assigned_unit) >= 2: return False return True @property def has_pending(self): """Return True if there are pending test items. This indicates that collection has finished and nodes are still processing test items, so this can be thought of as "the scheduler is active". """ if self.workqueue: return True for assigned_unit in self.assigned_work.values(): if self._pending_of(assigned_unit) > 0: return True return False def add_node(self, node): """Add a new node to the scheduler. From now on the node will be assigned work units to be executed. Called by the ``DSession.worker_workerready`` hook when it successfully bootstraps a new node. """ assert node not in self.assigned_work self.assigned_work[node] = OrderedDict() def remove_node(self, node): """Remove a node from the scheduler. This should be called either when the node crashed or at shutdown time. In the former case any pending items assigned to the node will be re-scheduled. Called by the hooks: - ``DSession.worker_workerfinished``. - ``DSession.worker_errordown``. Return the item being executed while the node crashed or None if the node has no more pending items. """ workload = self.assigned_work.pop(node) if not self._pending_of(workload): return None # The node crashed, identify test that crashed for work_unit in workload.values(): for nodeid, completed in work_unit.items(): if not completed: crashitem = nodeid break else: continue break else: raise RuntimeError( "Unable to identify crashitem on a workload with pending items" ) # Made uncompleted work unit available again self.workqueue.update(workload) for node in self.assigned_work: self._reschedule(node) return crashitem def add_node_collection(self, node, collection): """Add the collected test items from a node. The collection is stored in the ``.registered_collections`` dictionary. Called by the hook: - ``DSession.worker_collectionfinish``. """ # Check that add_node() was called on the node before assert node in self.assigned_work # A new node has been added later, perhaps an original one died. if self.collection_is_completed: # Assert that .schedule() should have been called by now assert self.collection # Check that the new collection matches the official collection if collection != self.collection: other_node = next(iter(self.registered_collections.keys())) msg = report_collection_diff(self.collection, collection, other_node.gateway.id, node.gateway.id) self.log(msg) return self.registered_collections[node] = list(collection) def mark_test_complete(self, node, item_index, duration=0): """Mark test item as completed by node. Called by the hook: - ``DSession.worker_testreport``. """ nodeid = self.registered_collections[node][item_index] scope = self._split_scope(nodeid) self.assigned_work[node][scope][nodeid] = True self._reschedule(node) def _assign_work_unit(self, node): """Assign a work unit to a node.""" assert self.workqueue # Grab a unit of work # scope, work_unit = self.workqueue.popitem(last=False) scope = next(iter(self.workqueue.keys())) if scope == 'test_acl_0052': # waiting other node stop for node in self.nodes: for nodeid, complete in self.assigned_work[node].items(): if not complete: return for node in self.nodes: if node != node: self.remove_node(node) node.shutdown() scope, work_unit = self.workqueue.popitem(last=False) # Keep track of the assigned work assigned_to_node = self.assigned_work.setdefault(node, default=OrderedDict()) assigned_to_node[scope] = work_unit # Ask the node to execute the workload worker_collection = self.registered_collections[node] nodeids_indexes = [ worker_collection.index(nodeid) for nodeid, completed in work_unit.items() if not completed ] node.send_runtest_some(nodeids_indexes) def _split_scope(self, nodeid): """Determine the scope (grouping) of a nodeid. There are usually 3 cases for a nodeid:: example/loadsuite/test/test_beta.py::test_beta0 example/loadsuite/test/test_delta.py::Delta1::test_delta0 example/loadsuite/epsilon/__init__.py::epsilon.epsilon #. Function in a test module. #. Method of a class in a test module. #. Doctest in a function in a package. This function will group tests with the scope determined by splitting the first ``::`` from the right. That is, classes will be grouped in a single work unit, and functions from a test module will be grouped by their module. In the above example, scopes will be:: example/loadsuite/test/test_beta.py example/loadsuite/test/test_delta.py::Delta1 example/loadsuite/epsilon/__init__.py """ return nodeid.rsplit("::", 1)[0] def _pending_of(self, workload): """Return the number of pending tests in a workload.""" pending = sum( list(scope.values()).count(False) for scope in workload.values()) return pending def _reschedule(self, node): """Maybe schedule new items on the node. If there are any globally pending work units left then this will check if the given node should be given any more tests. """ # Do not add more work to a node shutting down if node.shutting_down: return # Check that more work is available if not self.workqueue: node.shutdown() return self.log("Number of units waiting for node:", len(self.workqueue)) # Check that the node is almost depleted of work # 2: Heuristic of minimum tests to enqueue more work if self._pending_of(self.assigned_work[node]) > 2: return # Pop one unit of work and assign it self._assign_work_unit(node) def schedule(self): """Initiate distribution of the test collection. Initiate scheduling of the items across the nodes. If this gets called again later it behaves the same as calling ``._reschedule()`` on all nodes so that newly added nodes will start to be used. If ``.collection_is_completed`` is True, this is called by the hook: - ``DSession.worker_collectionfinish``. """ assert self.collection_is_completed # Initial distribution already happened, reschedule on all nodes if self.collection is not None: for node in self.nodes: self._reschedule(node) return # Check that all nodes collected the same tests if not self._check_nodes_have_same_collection(): self.log("**Different tests collected, aborting run**") return # Collections are identical, create the final list of items self.collection = list(next(iter( self.registered_collections.values()))) if not self.collection: return # Determine chunks of work (scopes) for nodeid in self.collection: scope = self._split_scope(nodeid) work_unit = self.workqueue.setdefault(scope, default=OrderedDict()) work_unit[nodeid] = False # Avoid having more workers than work extra_nodes = len(self.nodes) - len(self.workqueue) if extra_nodes > 0: self.log("Shuting down {0} nodes".format(extra_nodes)) for _ in range(extra_nodes): unused_node, assigned = self.assigned_work.popitem(last=True) self.log("Shuting down unused node {0}".format(unused_node)) unused_node.shutdown() # Assign initial workload for node in self.nodes: self._assign_work_unit(node) # Ensure nodes start with at least two work units if possible (#277) for node in self.nodes: self._reschedule(node) # Initial distribution sent all tests, start node shutdown if not self.workqueue: for node in self.nodes: node.shutdown() def _check_nodes_have_same_collection(self): """Return True if all nodes have collected the same items. If collections differ, this method returns False while logging the collection differences and posting collection errors to pytest_collectreport hook. """ node_collection_items = list(self.registered_collections.items()) first_node, col = node_collection_items[0] same_collection = True for node, collection in node_collection_items[1:]: msg = report_collection_diff(col, collection, first_node.gateway.id, node.gateway.id) if not msg: continue same_collection = False self.log(msg) if self.config is None: continue rep = CollectReport(node.gateway.id, "failed", longrepr=msg, result=[]) self.config.hook.pytest_collectreport(report=rep) return same_collection
class OWDataTable(OWWidget): name = "Data Table" description = "View the dataset in a spreadsheet." icon = "icons/Table.svg" priority = 50 keywords = [] buttons_area_orientation = Qt.Vertical class Inputs: data = Input("Data", Table, multiple=True) class Outputs: selected_data = Output("Selected Data", Table, default=True) annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Table) show_distributions = Setting(False) dist_color_RGB = Setting((220, 220, 220, 255)) show_attribute_labels = Setting(True) select_rows = Setting(True) auto_commit = Setting(True) color_by_class = Setting(True) settingsHandler = DomainContextHandler( match_values=DomainContextHandler.MATCH_VALUES_ALL) selected_rows = Setting([], schema_only=True) selected_cols = Setting([], schema_only=True) def __init__(self): super().__init__() self._inputs = OrderedDict() self.__pending_selected_rows = self.selected_rows self.selected_rows = None self.__pending_selected_cols = self.selected_cols self.selected_cols = None self.dist_color = QColor(*self.dist_color_RGB) info_box = gui.vBox(self.controlArea, "Info") self.info_ex = gui.widgetLabel(info_box, 'No data on input.', ) self.info_ex.setWordWrap(True) self.info_attr = gui.widgetLabel(info_box, ' ') self.info_attr.setWordWrap(True) self.info_class = gui.widgetLabel(info_box, ' ') self.info_class.setWordWrap(True) self.info_meta = gui.widgetLabel(info_box, ' ') self.info_meta.setWordWrap(True) info_box.setMinimumWidth(200) gui.separator(self.controlArea) box = gui.vBox(self.controlArea, "Variables") self.c_show_attribute_labels = gui.checkBox( box, self, "show_attribute_labels", "Show variable labels (if present)", callback=self._on_show_variable_labels_changed) gui.checkBox(box, self, "show_distributions", 'Visualize numeric values', callback=self._on_distribution_color_changed) gui.checkBox(box, self, "color_by_class", 'Color by instance classes', callback=self._on_distribution_color_changed) box = gui.vBox(self.controlArea, "Selection") gui.checkBox(box, self, "select_rows", "Select full rows", callback=self._on_select_rows_changed) gui.rubber(self.controlArea) reset = gui.button( None, self, "Restore Original Order", callback=self.restore_order, tooltip="Show rows in the original order", autoDefault=False) self.buttonsArea.layout().insertWidget(0, reset) gui.auto_send(self.buttonsArea, self, "auto_commit") # GUI with tabs self.tabs = gui.tabWidget(self.mainArea) self.tabs.currentChanged.connect(self._on_current_tab_changed) def copy_to_clipboard(self): self.copy() @staticmethod def sizeHint(): return QSize(800, 500) @Inputs.data def set_dataset(self, data, tid=None): """Set the input dataset.""" self.closeContext() if data is not None: datasetname = getattr(data, "name", "Data") if tid in self._inputs: # update existing input slot slot = self._inputs[tid] view = slot.view # reset the (header) view state. view.setModel(None) view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) assert self.tabs.indexOf(view) != -1 self.tabs.setTabText(self.tabs.indexOf(view), datasetname) else: view = QTableView() view.setSortingEnabled(True) view.setHorizontalScrollMode(QTableView.ScrollPerPixel) if self.select_rows: view.setSelectionBehavior(QTableView.SelectRows) header = view.horizontalHeader() header.setSectionsMovable(True) header.setSectionsClickable(True) header.setSortIndicatorShown(True) header.setSortIndicator(-1, Qt.AscendingOrder) # QHeaderView does not 'reset' the model sort column, # because there is no guaranty (requirement) that the # models understand the -1 sort column. def sort_reset(index, order): if view.model() is not None and index == -1: view.model().sort(index, order) header.sortIndicatorChanged.connect(sort_reset) self.tabs.addTab(view, datasetname) view.dataset = data self.tabs.setCurrentWidget(view) self._setup_table_view(view, data) slot = TableSlot(tid, data, table_summary(data), view) view._input_slot = slot # pylint: disable=protected-access self._inputs[tid] = slot self.tabs.setCurrentIndex(self.tabs.indexOf(view)) self.set_info(slot.summary) if isinstance(slot.summary.len, concurrent.futures.Future): def update(_): QMetaObject.invokeMethod( self, "_update_info", Qt.QueuedConnection) slot.summary.len.add_done_callback(update) elif tid in self._inputs: slot = self._inputs.pop(tid) view = slot.view view.hide() view.deleteLater() self.tabs.removeTab(self.tabs.indexOf(view)) current = self.tabs.currentWidget() if current is not None: # pylint: disable=protected-access self.set_info(current._input_slot.summary) self.tabs.tabBar().setVisible(self.tabs.count() > 1) self.openContext(data) if data and self.__pending_selected_rows is not None: self.selected_rows = self.__pending_selected_rows self.__pending_selected_rows = None else: self.selected_rows = [] if data and self.__pending_selected_cols is not None: self.selected_cols = self.__pending_selected_cols self.__pending_selected_cols = None else: self.selected_cols = [] self.set_selection() self.unconditional_commit() def _setup_table_view(self, view, data): """Setup the `view` (QTableView) with `data` (Orange.data.Table) """ if data is None: view.setModel(None) return datamodel = RichTableModel(data) rowcount = data.approx_len() if self.color_by_class and data.domain.has_discrete_class: color_schema = [ QColor(*c) for c in data.domain.class_var.colors] else: color_schema = None if self.show_distributions: view.setItemDelegate( gui.TableBarItem( self, color=self.dist_color, color_schema=color_schema) ) else: view.setItemDelegate(QStyledItemDelegate(self)) # Enable/disable view sorting based on data's type view.setSortingEnabled(is_sortable(data)) header = view.horizontalHeader() header.setSectionsClickable(is_sortable(data)) header.setSortIndicatorShown(is_sortable(data)) view.setModel(datamodel) vheader = view.verticalHeader() option = view.viewOptions() size = view.style().sizeFromContents( QStyle.CT_ItemViewItem, option, QSize(20, 20), view) vheader.setDefaultSectionSize(size.height() + 2) vheader.setMinimumSectionSize(5) vheader.setSectionResizeMode(QHeaderView.Fixed) # Limit the number of rows displayed in the QTableView # (workaround for QTBUG-18490 / QTBUG-28631) maxrows = (2 ** 31 - 1) // (vheader.defaultSectionSize() + 2) if rowcount > maxrows: sliceproxy = TableSliceProxy( parent=view, rowSlice=slice(0, maxrows)) sliceproxy.setSourceModel(datamodel) # First reset the view (without this the header view retains # it's state - at this point invalid/broken) view.setModel(None) view.setModel(sliceproxy) assert view.model().rowCount() <= maxrows assert vheader.sectionSize(0) > 1 or datamodel.rowCount() == 0 # update the header (attribute names) self._update_variable_labels(view) selmodel = BlockSelectionModel( view.model(), parent=view, selectBlocks=not self.select_rows) view.setSelectionModel(selmodel) view.selectionModel().selectionChanged.connect(self.update_selection) #noinspection PyBroadException def set_corner_text(self, table, text): """Set table corner text.""" # As this is an ugly hack, do everything in # try - except blocks, as it may stop working in newer Qt. # pylint: disable=broad-except if not hasattr(table, "btn") and not hasattr(table, "btnfailed"): try: btn = table.findChild(QAbstractButton) class Efc(QObject): @staticmethod def eventFilter(o, e): if (isinstance(o, QAbstractButton) and e.type() == QEvent.Paint): # paint by hand (borrowed from QTableCornerButton) btn = o opt = QStyleOptionHeader() opt.initFrom(btn) state = QStyle.State_None if btn.isEnabled(): state |= QStyle.State_Enabled if btn.isActiveWindow(): state |= QStyle.State_Active if btn.isDown(): state |= QStyle.State_Sunken opt.state = state opt.rect = btn.rect() opt.text = btn.text() opt.position = QStyleOptionHeader.OnlyOneSection painter = QStylePainter(btn) painter.drawControl(QStyle.CE_Header, opt) return True # eat event return False table.efc = Efc() # disconnect default handler for clicks and connect a new one, which supports # both selection and deselection of all data btn.clicked.disconnect() btn.installEventFilter(table.efc) btn.clicked.connect(self._on_select_all) table.btn = btn if sys.platform == "darwin": btn.setAttribute(Qt.WA_MacSmallSize) except Exception: table.btnfailed = True if hasattr(table, "btn"): try: btn = table.btn btn.setText(text) opt = QStyleOptionHeader() opt.text = btn.text() s = btn.style().sizeFromContents( QStyle.CT_HeaderSection, opt, QSize(), btn).expandedTo(QApplication.globalStrut()) if s.isValid(): table.verticalHeader().setMinimumWidth(s.width()) except Exception: pass def _on_select_all(self, _): # pylint: disable=protected-access data_info = self.tabs.currentWidget()._input_slot.summary if len(self.selected_rows) == data_info.len \ and len(self.selected_cols) == len(data_info.domain): self.tabs.currentWidget().clearSelection() else: self.tabs.currentWidget().selectAll() def _on_current_tab_changed(self, index): """Update the info box on current tab change""" view = self.tabs.widget(index) if view is not None and view.model() is not None: # pylint: disable=protected-access self.set_info(view._input_slot.summary) else: self.set_info(None) def _update_variable_labels(self, view): "Update the variable labels visibility for `view`" model = view.model() if isinstance(model, TableSliceProxy): model = model.sourceModel() if self.show_attribute_labels: model.setRichHeaderFlags( RichTableModel.Labels | RichTableModel.Name) labelnames = set() domain = model.source.domain for a in itertools.chain(domain.metas, domain.variables): labelnames.update(a.attributes.keys()) labelnames = sorted( [label for label in labelnames if not label.startswith("_")]) self.set_corner_text(view, "\n".join([""] + labelnames)) else: model.setRichHeaderFlags(RichTableModel.Name) self.set_corner_text(view, "") def _on_show_variable_labels_changed(self): """The variable labels (var.attribues) visibility was changed.""" for slot in self._inputs.values(): self._update_variable_labels(slot.view) def _on_distribution_color_changed(self): for ti in range(self.tabs.count()): widget = self.tabs.widget(ti) model = widget.model() while isinstance(model, QAbstractProxyModel): model = model.sourceModel() data = model.source class_var = data.domain.class_var if self.color_by_class and class_var and class_var.is_discrete: color_schema = [QColor(*c) for c in class_var.colors] else: color_schema = None if self.show_distributions: delegate = gui.TableBarItem(self, color=self.dist_color, color_schema=color_schema) else: delegate = QStyledItemDelegate(self) widget.setItemDelegate(delegate) tab = self.tabs.currentWidget() if tab: tab.reset() def _on_select_rows_changed(self): for slot in self._inputs.values(): selection_model = slot.view.selectionModel() selection_model.setSelectBlocks(not self.select_rows) if self.select_rows: slot.view.setSelectionBehavior(QTableView.SelectRows) # Expand the current selection to full row selection. selection_model.select( selection_model.selection(), QItemSelectionModel.Select | QItemSelectionModel.Rows ) else: slot.view.setSelectionBehavior(QTableView.SelectItems) def restore_order(self): """Restore the original data order of the current view.""" table = self.tabs.currentWidget() if table is not None: table.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) def set_info(self, summary): if summary is None: self.info_ex.setText("No data on input.") self.info_attr.setText("") self.info_class.setText("") self.info_meta.setText("") else: info_len, info_attr, info_class, info_meta = \ format_summary(summary) self.info_ex.setText(info_len) self.info_attr.setText(info_attr) self.info_class.setText(info_class) self.info_meta.setText(info_meta) @Slot() def _update_info(self): current = self.tabs.currentWidget() if current is not None and current.model() is not None: # pylint: disable=protected-access self.set_info(current._input_slot.summary) def update_selection(self, *_): self.commit() def set_selection(self): if self.selected_rows and self.selected_cols: view = self.tabs.currentWidget() model = view.model() if model.rowCount() <= self.selected_rows[-1] or \ model.columnCount() <= self.selected_cols[-1]: return selection = QItemSelection() rowranges = list(ranges(self.selected_rows)) colranges = list(ranges(self.selected_cols)) for rowstart, rowend in rowranges: for colstart, colend in colranges: selection.append( QItemSelectionRange( view.model().index(rowstart, colstart), view.model().index(rowend - 1, colend - 1) ) ) view.selectionModel().select( selection, QItemSelectionModel.ClearAndSelect) @staticmethod def get_selection(view): """ Return the selected row and column indices of the selection in view. """ selmodel = view.selectionModel() selection = selmodel.selection() model = view.model() # map through the proxies into input table. while isinstance(model, QAbstractProxyModel): selection = model.mapSelectionToSource(selection) model = model.sourceModel() assert isinstance(selmodel, BlockSelectionModel) assert isinstance(model, TableModel) row_spans, col_spans = selection_blocks(selection) rows = list(itertools.chain.from_iterable(itertools.starmap(range, row_spans))) cols = list(itertools.chain.from_iterable(itertools.starmap(range, col_spans))) rows = numpy.array(rows, dtype=numpy.intp) # map the rows through the applied sorting (if any) rows = model.mapToSourceRows(rows) rows.sort() rows = rows.tolist() return rows, cols @staticmethod def _get_model(view): model = view.model() while isinstance(model, QAbstractProxyModel): model = model.sourceModel() return model def commit(self): """ Commit/send the current selected row/column selection. """ selected_data = table = rowsel = None view = self.tabs.currentWidget() if view and view.model() is not None: model = self._get_model(view) table = model.source # The input data table # Selections of individual instances are not implemented # for SqlTables if isinstance(table, SqlTable): self.Outputs.selected_data.send(selected_data) self.Outputs.annotated_data.send(None) return rowsel, colsel = self.get_selection(view) self.selected_rows, self.selected_cols = rowsel, colsel def select(data, rows, domain): """ Select the data subset with specified rows and domain subsets. If either rows or domain is None they mean select all. """ if rows is not None and domain is not None: return data.from_table(domain, data, rows) elif rows is not None: return data.from_table(data.domain, rows) elif domain is not None: return data.from_table(domain, data) else: return data domain = table.domain if len(colsel) < len(domain) + len(domain.metas): # only a subset of the columns is selected allvars = domain.class_vars + domain.metas + domain.attributes columns = [(c, model.headerData(c, Qt.Horizontal, TableModel.DomainRole)) for c in colsel] assert all(role is not None for _, role in columns) def select_vars(role): """select variables for role (TableModel.DomainRole)""" return [allvars[c] for c, r in columns if r == role] attrs = select_vars(TableModel.Attribute) if attrs and issparse(table.X): # for sparse data you can only select all attributes attrs = table.domain.attributes class_vars = select_vars(TableModel.ClassVar) metas = select_vars(TableModel.Meta) domain = Orange.data.Domain(attrs, class_vars, metas) # Avoid a copy if all/none rows are selected. if not rowsel: selected_data = None elif len(rowsel) == len(table): selected_data = select(table, None, domain) else: selected_data = select(table, rowsel, domain) self.Outputs.selected_data.send(selected_data) self.Outputs.annotated_data.send(create_annotated_table(table, rowsel)) def copy(self): """ Copy current table selection to the clipboard. """ view = self.tabs.currentWidget() if view is not None: mime = table_selection_to_mime_data(view) QApplication.clipboard().setMimeData( mime, QClipboard.Clipboard ) def send_report(self): view = self.tabs.currentWidget() if not view or not view.model(): return model = self._get_model(view) self.report_data_brief(model.source) self.report_table(view)
class BaseFlow(JAMLCompatible, ExitStack, metaclass=FlowType): """An abstract Flow object in Jina. .. note:: :class:`BaseFlow` does not provide `train`, `index`, `search` interfaces. Please use :class:`Flow` or :class:`AsyncFlow`. Explanation on ``optimize_level``: As an example, the following Flow will generate 6 Peas, .. highlight:: python .. code-block:: python f = Flow(optimize_level=FlowOptimizeLevel.NONE).add(uses='forward', parallel=3) The optimized version, i.e. :code:`Flow(optimize_level=FlowOptimizeLevel.FULL)` will generate 4 Peas, but it will force the :class:`GatewayPea` to take BIND role, as the head and tail routers are removed. :param kwargs: other keyword arguments that will be shared by all Pods in this Flow :param args: Namespace args :param env: environment variables shared by all Pods """ _cls_client = Client #: the type of the Client, can be changed to other class def __init__(self, args: Optional['argparse.Namespace'] = None, env: Optional[Dict] = None, **kwargs): """Initialize a Flow object""" super().__init__() self._version = '1' #: YAML version number, this will be later overridden if YAML config says the other way self._pod_nodes = OrderedDict() # type: Dict[str, 'BasePod'] self._inspect_pods = {} # type: Dict[str, str] self._build_level = FlowBuildLevel.EMPTY self._last_changed_pod = [ 'gateway' ] #: default first pod is gateway, will add when build() self._update_args(args, **kwargs) self._env = env if isinstance(self.args, argparse.Namespace): self.logger = JinaLogger(self.__class__.__name__, **vars(self.args)) else: self.logger = JinaLogger(self.__class__.__name__) def _update_args(self, args, **kwargs): from ..parsers.flow import set_flow_parser from ..helper import ArgNamespace _flow_parser = set_flow_parser() if args is None: args = ArgNamespace.kwargs2namespace(kwargs, _flow_parser) self.args = args self._common_kwargs = kwargs self._kwargs = ArgNamespace.get_non_defaults_args( args, _flow_parser) #: for yaml dump @property def yaml_spec(self): """ get the YAML representation of the instance # noqa: DAR401 # noqa: DAR201 """ return JAML.dump(self) @staticmethod def _parse_endpoints(op_flow, pod_name, endpoint, connect_to_last_pod=False) -> Set: # parsing needs if isinstance(endpoint, str): endpoint = [endpoint] elif not endpoint: if op_flow._last_changed_pod and connect_to_last_pod: endpoint = [op_flow.last_pod] else: endpoint = [] if isinstance(endpoint, (list, tuple)): for idx, s in enumerate(endpoint): if s == pod_name: raise FlowTopologyError( 'the income/output of a pod can not be itself') else: raise ValueError(f'endpoint={endpoint} is not parsable') # if an endpoint is being inspected, then replace it with inspected Pod endpoint = set(op_flow._inspect_pods.get(ep, ep) for ep in endpoint) return endpoint @property def last_pod(self): """Last pod # noqa: DAR401 # noqa: DAR201 """ return self._last_changed_pod[-1] @last_pod.setter def last_pod(self, name: str): """ Set a Pod as the last Pod in the Flow, useful when modifying the Flow. # noqa: DAR401 :param name: the name of the existing Pod """ if name not in self._pod_nodes: raise FlowMissingPodError(f'{name} can not be found in this Flow') if self._last_changed_pod and name == self.last_pod: pass else: self._last_changed_pod.append(name) # graph is now changed so we need to # reset the build level to the lowest self._build_level = FlowBuildLevel.EMPTY def _add_gateway(self, needs, **kwargs): pod_name = 'gateway' kwargs.update( dict( name=pod_name, ctrl_with_ipc=True, # otherwise ctrl port would be conflicted read_only=True, runtime_cls='GRPCRuntime', pod_role=PodRoleType.GATEWAY, identity=self.args.identity)) kwargs.update(self._common_kwargs) args = ArgNamespace.kwargs2namespace(kwargs, set_gateway_parser()) self._pod_nodes[pod_name] = BasePod(args, needs) def needs(self, needs: Union[Tuple[str], List[str]], name: str = 'joiner', *args, **kwargs) -> 'BaseFlow': """ Add a blocker to the Flow, wait until all peas defined in **needs** completed. # noqa: DAR401 :param needs: list of service names to wait :param name: the name of this joiner, by default is ``joiner`` :param *args: *args for .add :param **kwargs: **kwargs for .add :return: the modified Flow """ if len(needs) <= 1: raise FlowTopologyError( 'no need to wait for a single service, need len(needs) > 1') return self.add(name=name, needs=needs, pod_role=PodRoleType.JOIN, *args, **kwargs) def needs_all(self, name: str = 'joiner', *args, **kwargs) -> 'BaseFlow': """ Collect all hanging Pods so far and add a blocker to the Flow; wait until all handing peas completed. :param name: the name of this joiner (default is ``joiner``) :param *args: *args for .add or .needs :param **kwargs: **kwargs for .add or .needs :return: the modified Flow """ needs = _hanging_pods(self) if len(needs) == 1: return self.add(name=name, needs=needs, *args, **kwargs) return self.needs(name=name, needs=needs, *args, **kwargs) def add(self, needs: Union[str, Tuple[str], List[str]] = None, copy_flow: bool = True, pod_role: 'PodRoleType' = PodRoleType.POD, **kwargs) -> 'BaseFlow': """ Add a Pod to the current Flow object and return the new modified Flow object. The attribute of the Pod can be later changed with :py:meth:`set` or deleted with :py:meth:`remove` Note there are shortcut versions of this method. Recommend to use :py:meth:`add_encoder`, :py:meth:`add_preprocessor`, :py:meth:`add_router`, :py:meth:`add_indexer` whenever possible. # noqa: DAR401 :param needs: the name of the Pod(s) that this Pod receives data from. One can also use 'pod.Gateway' to indicate the connection with the gateway. :param pod_role: the role of the Pod, used for visualization and route planning :param copy_flow: when set to true, then always copy the current Flow and do the modification on top of it then return, otherwise, do in-line modification :param **kwargs: other keyword-value arguments that the Pod CLI supports :return: a (new) Flow object with modification """ op_flow = copy.deepcopy(self) if copy_flow else self # pod naming logic pod_name = kwargs.get('name') if pod_name in op_flow._pod_nodes: new_name = f'{pod_name}{len(op_flow._pod_nodes)}' self.logger.debug( f'"{pod_name}" is used in this Flow already! renamed it to "{new_name}"' ) pod_name = new_name if not pod_name: pod_name = f'pod{len(op_flow._pod_nodes)}' if not pod_name.isidentifier(): # hyphen - can not be used in the name raise ValueError( f'name: {pod_name} is invalid, please follow the python variable name conventions' ) # needs logic needs = op_flow._parse_endpoints(op_flow, pod_name, needs, connect_to_last_pod=True) # set the kwargs inherit from `Flow(kwargs1=..., kwargs2=)` for key, value in op_flow._common_kwargs.items(): if key not in kwargs: kwargs[key] = value # check if host is set to remote:port if 'host' in kwargs: m = re.match(_regex_port, kwargs['host']) if kwargs.get( 'host', __default_host__ ) != __default_host__ and m and 'port_expose' not in kwargs: kwargs['port_expose'] = m.group(2) kwargs['host'] = m.group(1) # update kwargs of this Pod kwargs.update( dict(name=pod_name, pod_role=pod_role, num_part=len(needs))) parser = set_pod_parser() if pod_role == PodRoleType.GATEWAY: parser = set_gateway_parser() args = ArgNamespace.kwargs2namespace(kwargs, parser) op_flow._pod_nodes[pod_name] = BasePod(args, needs=needs) op_flow.last_pod = pod_name return op_flow def inspect(self, name: str = 'inspect', *args, **kwargs) -> 'BaseFlow': """Add an inspection on the last changed Pod in the Flow Internally, it adds two Pods to the Flow. But don't worry, the overhead is minimized and you can remove them by simply using `Flow(inspect=FlowInspectType.REMOVE)` before using the Flow. .. highlight:: bash .. code-block:: bash Flow -- PUB-SUB -- BasePod(_pass) -- Flow | -- PUB-SUB -- InspectPod (Hanging) In this way, :class:`InspectPod` looks like a simple ``_pass`` from outside and does not introduce side-effects (e.g. changing the socket type) to the original Flow. The original incoming and outgoing socket types are preserved. This function is very handy for introducing an Evaluator into the Flow. .. seealso:: :meth:`gather_inspect` :param name: name of the Pod :param *args: *args for .add() :param **kwargs: **kwargs for .add() :return: the new instance of the Flow """ _last_pod = self.last_pod op_flow = self.add(name=name, needs=_last_pod, pod_role=PodRoleType.INSPECT, *args, **kwargs) # now remove uses and add an auxiliary Pod if 'uses' in kwargs: kwargs.pop('uses') op_flow = op_flow.add(name=f'_aux_{name}', needs=_last_pod, pod_role=PodRoleType.INSPECT_AUX_PASS, *args, **kwargs) # register any future connection to _last_pod by the auxiliary Pod op_flow._inspect_pods[_last_pod] = op_flow.last_pod return op_flow def gather_inspect(self, name: str = 'gather_inspect', uses='_merge_eval', include_last_pod: bool = True, *args, **kwargs) -> 'BaseFlow': """ Gather all inspect Pods output into one Pod. When the Flow has no inspect Pod then the Flow itself is returned. .. note:: If ``--no-inspect`` is **not** given, then :meth:`gather_inspect` is auto called before :meth:`build`. So in general you don't need to manually call :meth:`gather_inspect`. :param name: the name of the gather Pod :param uses: the config of the executor, by default is ``_pass`` :param include_last_pod: if to include the last modified Pod in the Flow :param *args: *args for .add() :param **kwargs: **kwargs for .add() :return: the modified Flow or the copy of it .. seealso:: :meth:`inspect` """ needs = [ k for k, v in self._pod_nodes.items() if v.role == PodRoleType.INSPECT ] if needs: if include_last_pod: needs.append(self.last_pod) return self.add(name=name, uses=uses, needs=needs, pod_role=PodRoleType.JOIN_INSPECT, *args, **kwargs) else: # no inspect node is in the graph, return the current graph return self def build(self, copy_flow: bool = False) -> 'BaseFlow': """ Build the current Flow and make it ready to use .. note:: No need to manually call it since 0.0.8. When using Flow with the context manager, or using :meth:`start`, :meth:`build` will be invoked. :param copy_flow: when set to true, then always copy the current Flow and do the modification on top of it then return, otherwise, do in-line modification :return: the current Flow (by default) .. note:: ``copy_flow=True`` is recommended if you are building the same Flow multiple times in a row. e.g. .. highlight:: python .. code-block:: python f = Flow() with f: f.index() with f.build(copy_flow=True) as fl: fl.search() # noqa: DAR401 """ op_flow = copy.deepcopy(self) if copy_flow else self _pod_edges = set() if op_flow.args.inspect == FlowInspectType.COLLECT: op_flow.gather_inspect(copy_flow=False) if 'gateway' not in op_flow._pod_nodes: op_flow._add_gateway(needs={op_flow.last_pod}) # construct a map with a key a start node and values an array of its end nodes _outgoing_map = defaultdict(list) # if set no_inspect then all inspect related nodes are removed if op_flow.args.inspect == FlowInspectType.REMOVE: op_flow._pod_nodes = { k: v for k, v in op_flow._pod_nodes.items() if not v.role.is_inspect } reverse_inspect_map = { v: k for k, v in op_flow._inspect_pods.items() } for end, pod in op_flow._pod_nodes.items(): # if an endpoint is being inspected, then replace it with inspected Pod # but not those inspect related node if op_flow.args.inspect.is_keep: pod.needs = set(ep if pod.role.is_inspect else op_flow. _inspect_pods.get(ep, ep) for ep in pod.needs) else: pod.needs = set( reverse_inspect_map.get(ep, ep) for ep in pod.needs) for start in pod.needs: if start not in op_flow._pod_nodes: raise FlowMissingPodError( f'{start} is not in this flow, misspelled name?') _outgoing_map[start].append(end) _pod_edges.add((start, end)) op_flow = _build_flow(op_flow, _outgoing_map) op_flow = _optimize_flow(op_flow, _outgoing_map, _pod_edges) hanging_pods = _hanging_pods(op_flow) if hanging_pods: self.logger.warning( f'{hanging_pods} are hanging in this flow with no pod receiving from them, ' f'you may want to double check if it is intentional or some mistake' ) op_flow._build_level = FlowBuildLevel.GRAPH self._update_client() return op_flow def __call__(self, *args, **kwargs): """Builds the Flow :param *args: *args for build :param **kwargs: **kwargs for build :return: the built Flow """ return self.build(*args, **kwargs) def __enter__(self): return self.start() def __exit__(self, exc_type, exc_val, exc_tb): super().__exit__(exc_type, exc_val, exc_tb) # unset all envs to avoid any side-effect if self._env: for k in self._env.keys(): os.unsetenv(k) self._pod_nodes.pop('gateway') self._build_level = FlowBuildLevel.EMPTY self.logger.success( f'flow is closed and all resources are released, current build level is {self._build_level}' ) self.logger.close() def start(self): """Start to run all Pods in this Flow. Remember to close the Flow with :meth:`close`. Note that this method has a timeout of ``timeout_ready`` set in CLI, which is inherited all the way from :class:`jina.peapods.peas.BasePea` # noqa: DAR401 :return: this instance """ if self._build_level.value < FlowBuildLevel.GRAPH.value: self.build(copy_flow=False) # set env only before the Pod get started if self._env: for k, v in self._env.items(): os.environ[k] = str(v) for k, v in self: v.args.noblock_on_start = True self.enter_context(v) for k, v in self: try: v.wait_start_success() except Exception as ex: self.logger.error( f'{k}:{v!r} can not be started due to {ex!r}, Flow is aborted' ) self.close() raise self.logger.info( f'{self.num_pods} Pods (i.e. {self.num_peas} Peas) are running in this Flow' ) self._show_success_message() return self @property def num_pods(self) -> int: """Get the number of Pods in this Flow # noqa: DAR201""" return len(self._pod_nodes) @property def num_peas(self) -> int: """Get the number of peas (parallel count) in this Flow # noqa: DAR201""" return sum(v.num_peas for v in self._pod_nodes.values()) def __eq__(self, other: 'BaseFlow') -> bool: """ Compare the topology of a Flow with another Flow. Identification is defined by whether two flows share the same set of edges. :param other: the second Flow object :return: result of equality check """ if self._build_level.value < FlowBuildLevel.GRAPH.value: a = self.build() else: a = self if other._build_level.value < FlowBuildLevel.GRAPH.value: b = other.build() else: b = other return a._pod_nodes == b._pod_nodes @build_required(FlowBuildLevel.GRAPH) def _get_client(self, **kwargs) -> 'Client': kwargs.update(self._common_kwargs) if 'port_expose' not in kwargs: kwargs['port_expose'] = self.port_expose if 'host' not in kwargs: kwargs['host'] = self.host args = ArgNamespace.kwargs2namespace(kwargs, set_client_cli_parser()) return self._cls_client(args) @property def _mermaid_str(self): mermaid_graph = [ "%%{init: {'theme': 'base', " "'themeVariables': { 'primaryColor': '#32C8CD', " "'edgeLabelBackground':'#fff', 'clusterBkg': '#FFCC66'}}}%%", 'graph LR' ] start_repl = {} end_repl = {} for node, v in self._pod_nodes.items(): if not v.is_singleton and v.role != PodRoleType.GATEWAY: mermaid_graph.append( f'subgraph sub_{node} ["{node} ({v.args.parallel})"]') if v.is_head_router: head_router = node + '_HEAD' end_repl[node] = (head_router, '((fa:fa-random))') if v.is_tail_router: tail_router = node + '_TAIL' start_repl[node] = (tail_router, '((fa:fa-random))') p_r = '((%s))' p_e = '[[%s]]' for j in range(v.args.parallel): r = node + (f'_{j}' if v.args.parallel > 1 else '') if v.is_head_router: mermaid_graph.append( f'\t{head_router}{p_r % "head"}:::pea-->{r}{p_e % r}:::pea' ) if v.is_tail_router: mermaid_graph.append( f'\t{r}{p_e % r}:::pea-->{tail_router}{p_r % "tail"}:::pea' ) mermaid_graph.append('end') for node, v in self._pod_nodes.items(): ed_str = str(v.head_args.socket_in).split('_')[0] for need in sorted(v.needs): edge_str = '' if need in self._pod_nodes: st_str = str( self._pod_nodes[need].tail_args.socket_out).split( '_')[0] edge_str = f'|{st_str}-{ed_str}|' _s = start_repl.get(need, (need, f'({need})')) _e = end_repl.get(node, (node, f'({node})')) _s_role = self._pod_nodes[need].role _e_role = self._pod_nodes[node].role line_st = '-->' if _s_role in {PodRoleType.INSPECT, PodRoleType.JOIN_INSPECT}: _s = start_repl.get(need, (need, f'{{{{{need}}}}}')) if _e_role == PodRoleType.GATEWAY: _e = ('gateway_END', f'({node})') elif _e_role in { PodRoleType.INSPECT, PodRoleType.JOIN_INSPECT }: _e = end_repl.get(node, (node, f'{{{{{node}}}}}')) if _s_role == PodRoleType.INSPECT or _e_role == PodRoleType.INSPECT: line_st = '-.->' mermaid_graph.append( f'{_s[0]}{_s[1]}:::{str(_s_role)} {line_st} {edge_str}{_e[0]}{_e[1]}:::{str(_e_role)}' ) mermaid_graph.append( f'classDef {str(PodRoleType.POD)} fill:#32C8CD,stroke:#009999') mermaid_graph.append( f'classDef {str(PodRoleType.INSPECT)} fill:#ff6666,color:#fff') mermaid_graph.append( f'classDef {str(PodRoleType.JOIN_INSPECT)} fill:#ff6666,color:#fff' ) mermaid_graph.append( f'classDef {str(PodRoleType.GATEWAY)} fill:#6E7278,color:#fff') mermaid_graph.append( f'classDef {str(PodRoleType.INSPECT_AUX_PASS)} fill:#fff,color:#000,stroke-dasharray: 5 5' ) mermaid_graph.append('classDef pea fill:#009999,stroke:#1E6E73') return '\n'.join(mermaid_graph) def plot(self, output: str = None, vertical_layout: bool = False, inline_display: bool = False, build: bool = True, copy_flow: bool = False) -> 'BaseFlow': """ Visualize the Flow up to the current point If a file name is provided it will create a jpg image with that name, otherwise it will display the URL for mermaid. If called within IPython notebook, it will be rendered inline, otherwise an image will be created. Example, .. highlight:: python .. code-block:: python flow = Flow().add(name='pod_a').plot('flow.svg') :param output: a filename specifying the name of the image to be created, the suffix svg/jpg determines the file type of the output image :param vertical_layout: top-down or left-right layout :param inline_display: show image directly inside the Jupyter Notebook :param build: build the Flow first before plotting, gateway connection can be better showed :param copy_flow: when set to true, then always copy the current Flow and do the modification on top of it then return, otherwise, do in-line modification :return: the Flow """ # deepcopy causes the below error while reusing a Flow in Jupyter # 'Pickling an AuthenticationString object is disallowed for security reasons' op_flow = copy.deepcopy(self) if copy_flow else self if build: op_flow.build(False) mermaid_str = op_flow._mermaid_str if vertical_layout: mermaid_str = mermaid_str.replace('graph LR', 'graph TD') image_type = 'svg' if output and output.endswith('jpg'): image_type = 'jpg' url = op_flow._mermaid_to_url(mermaid_str, image_type) showed = False if inline_display: try: from IPython.display import display, Image display(Image(url=url)) showed = True except: # no need to panic users pass if output: download_mermaid_url(url, output) elif not showed: op_flow.logger.info(f'flow visualization: {url}') return self def _ipython_display_(self): """Displays the object in IPython as a side effect""" self.plot(inline_display=True, build=(self._build_level != FlowBuildLevel.GRAPH)) def _mermaid_to_url(self, mermaid_str: str, img_type: str) -> str: """ Render the current Flow as URL points to a SVG. It needs internet connection :param mermaid_str: the mermaid representation :param img_type: image type (svg/jpg) :return: the url points to a SVG """ if img_type == 'jpg': img_type = 'img' encoded_str = base64.b64encode(bytes(mermaid_str, 'utf-8')).decode('utf-8') return f'https://mermaid.ink/{img_type}/{encoded_str}' @build_required(FlowBuildLevel.GRAPH) def to_swarm_yaml(self, path: TextIO): """ Generate the docker swarm YAML compose file :param path: the output yaml path """ swarm_yml = {'version': '3.4', 'services': {}} for k, v in self._pod_nodes.items(): if v.role == PodRoleType.GATEWAY: cmd = 'jina gateway' else: cmd = 'jina pod' swarm_yml['services'][k] = { 'command': f'{cmd} {" ".join(ArgNamespace.kwargs2list(vars(v.args)))}', 'deploy': { 'parallel': 1 } } JAML.dump(swarm_yml, path) @property @build_required(FlowBuildLevel.GRAPH) def port_expose(self) -> int: """Return the exposed port of the gateway # noqa: DAR201""" return self._pod_nodes['gateway'].port_expose @property @build_required(FlowBuildLevel.GRAPH) def host(self) -> str: """Return the local address of the gateway # noqa: DAR201""" return self._pod_nodes['gateway'].host @property @build_required(FlowBuildLevel.GRAPH) def address_private(self) -> str: """Return the private IP address of the gateway for connecting from other machine in the same network # noqa: DAR201""" return get_internal_ip() @property @build_required(FlowBuildLevel.GRAPH) def address_public(self) -> str: """Return the public IP address of the gateway for connecting from other machine in the public network # noqa: DAR201""" return get_public_ip() def __iter__(self): return self._pod_nodes.items().__iter__() def _show_success_message(self): if self._pod_nodes['gateway'].args.restful: header = 'http://' protocol = 'REST' else: header = 'tcp://' protocol = 'gRPC' address_table = [ f'\t🖥️ Local access:\t' + colored(f'{header}{self.host}:{self.port_expose}', 'cyan', attrs='underline'), f'\t🔒 Private network:\t' + colored(f'{header}{self.address_private}:{self.port_expose}', 'cyan', attrs='underline') ] if self.address_public: address_table.append( f'\t🌐 Public address:\t' + colored(f'{header}{self.address_public}:{self.port_expose}', 'cyan', attrs='underline')) self.logger.success( f'🎉 Flow is ready to use, accepting {colored(protocol + " request", attrs="bold")}' ) self.logger.info('\n' + '\n'.join(address_table)) def block(self): """Block the process until user hits KeyboardInterrupt """ try: threading.Event().wait() except KeyboardInterrupt: pass def use_grpc_gateway(self, port: int = None): """Change to use gRPC gateway for IO :param port: the port to change""" self._common_kwargs['restful'] = False if port: self._common_kwargs['port_expose'] = port def use_rest_gateway(self, port: int = None): """Change to use REST gateway for IO :param port: the port to change""" self._common_kwargs['restful'] = True if port: self._common_kwargs['port_expose'] = port def __getitem__(self, item): if isinstance(item, str): return self._pod_nodes[item] elif isinstance(item, int): return list(self._pod_nodes.values())[item] else: raise TypeError(f'{typename(item)} is not supported') def _update_client(self): if self._pod_nodes['gateway'].args.restful: self._cls_client = WebSocketClient @property def workspace_id(self) -> Dict[str, str]: """Get all Pods' ``workspace_id`` values in a dict # noqa: DAR201""" return { k: p.args.workspace_id for k, p in self if hasattr(p.args, 'workspace_id') } @workspace_id.setter def workspace_id(self, value: str): """Set all Pods' ``workspace_id`` to ``value`` :param value: a hexadecimal UUID string """ uuid.UUID(value) for k, p in self: if hasattr(p.args, 'workspace_id'): p.args.workspace_id = value for k, v in p.peas_args.items(): if v and isinstance(v, argparse.Namespace): v.workspace_id = value if v and isinstance(v, List): for i in v: i.workspace_id = value @property def identity(self) -> Dict[str, str]: """Get all Pods' ``identity`` values in a dict # noqa: DAR201 """ return {k: p.args.identity for k, p in self} @identity.setter def identity(self, value: str): """Set all Pods' ``identity`` to ``value`` :param value: a hexadecimal UUID string """ uuid.UUID(value) self.args.identity = value # Re-initiating logger with new identity self.logger = JinaLogger(self.__class__.__name__, **vars(self.args)) for _, p in self: p.args.identity = value # for backward support join = needs
def read_json(json_file): """Read Json dataset package files Load each json and get the appropriate encoding for the dataset Reload the json using the encoding to ensure correct character sets """ json_object = OrderedDict() json_file_encoding = None json_file = str(json_file) + ".json" try: file_obj = open_fr(json_file) json_object = json.load(file_obj) if "encoding" in json_object: json_file_encoding = json_object['encoding'] file_obj.close() except ValueError: return None # Reload json using encoding if available try: if json_file_encoding: file_obj = open_fr(json_file, encoding=json_file_encoding) else: file_obj = open_fr(json_file) json_object = json.load(file_obj) file_obj.close() except ValueError: return None if isinstance(json_object, dict) and "resources" in json_object.keys(): # Note::formats described by frictionless data may need to change tabular_exts = {"csv", "tab"} vector_exts = {"shp", "kmz"} raster_exts = {"tif", "tiff", "bil", "hdr", "h5", "hdf5", "hr", "image"} for resource_item in json_object["resources"]: if "format" not in resource_item: if "format" in json_object: resource_item["format"] = json_object["format"] else: resource_item["format"] = "tabular" if "extensions" in resource_item: exts = set(resource_item["extensions"]) if exts <= tabular_exts: resource_item["format"] = "tabular" elif exts <= vector_exts: resource_item["format"] = "vector" elif exts <= raster_exts: resource_item["format"] = "raster" if "url" in resource_item: if "urls" in json_object: json_object["urls"][resource_item["name"]] = resource_item["url"] json_object["tables"] = OrderedDict() temp_tables = {} table_names = [item["name"] for item in json_object["resources"]] temp_tables["tables"] = OrderedDict(zip(table_names, json_object["resources"])) for table_name, table_spec in temp_tables["tables"].items(): json_object["tables"][table_name] = myTables[temp_tables["tables"][table_name] ["format"]](**table_spec) json_object.pop("resources", None) return TEMPLATES["default"](**json_object) return None
class KafkaBrokerClient(ReconnectingClientFactory): """The low-level client which handles transport to a single Kafka broker. The KafkaBrokerClient object is responsible for maintaining a connection to a single Kafka broker, reconnecting as needed, over which is sends requests and receives responses. Callers can register as 'subscribers' which will cause them to be notified of changes in the state of the connection to the Kafka broker. Callers make requests with :py:method:`makeRequest` """ # What class protocol instances do we produce? protocol = KafkaProtocol # Reduce log spam from twisted noisy = False def __init__(self, host, port=DefaultKafkaPort, clientId=CLIENT_ID, subscribers=None, maxDelay=MAX_RECONNECT_DELAY_SECONDS, maxRetries=None, reactor=None): """Create a KafkaBrokerClient for a given host/port. Create a new object to manage the connection to a single Kafka broker. KafkaBrokerClient will reconnect as needed to keep the connection to the broker up and ready to service requests. Requests are retried when the connection fails before the client receives the response. Requests can be cancelled at any time. Args: host (str): hostname or IP address of Kafka broker port (int): port number of Kafka broker on `host`. Defaulted: 9092 clientId (str): Identifying string for log messages. NOTE: not the ClientId in the RequestMessage PDUs going over the wire. subscribers (list of callbacks): Initial list of callbacks to be called when the connection changes state. maxDelay (seconds): The maximum amount of time between reconnect attempts when a connection has failed. maxRetries: The maximum number of times a reconnect attempt will be made. reactor: the twisted reactor to use when making connections or scheduling iDelayedCall calls. Used primarily for testing. """ # Set the broker host & port self.host = host self.port = port # No connector until we try to connect self.connector = None # Set our clientId self.clientId = clientId # If the caller set maxRetries, we will retry that many # times to reconnect, otherwise we retry forever self.maxRetries = maxRetries # Set max delay between reconnect attempts self.maxDelay = maxDelay # clock/reactor for testing... self.clock = reactor # The protocol object for the current connection self.proto = None # ordered dict of _Requests, keyed by requestId self.requests = OrderedDict() # deferred which fires when the close() completes self.dDown = None # Deferred list for any on-going notification self.notifydList = None # list of subscribers to our connection status: # when the connection goes up or down, we call the callback # with ourself and True/False for Connection-Up/Down if subscribers is None: self.connSubscribers = [] else: self.connSubscribers = subscribers def __repr__(self): """return a string representing this KafkaBrokerClient.""" return ('<KafkaBrokerClient {0}:{1}:{2}'.format( self.host, self.port, self.clientId)) def makeRequest(self, requestId, request, expectResponse=True): """ Send a request to our broker via our self.proto KafkaProtocol object. Return a deferred which will fire when the reply matching the requestId comes back from the server, or, if expectResponse is False, then return None instead. If we are not currently connected, then we buffer the request to send when the connection comes back up. """ if requestId in self.requests: # Id is duplicate to 'in-flight' request. Reject it, as we # won't be able to properly deliver the response(s) # Note that this won't protect against a client calling us # twice with the same ID, but first with expectResponse=False # But that's pathological, and the only defense is to track # all requestIds sent regardless of whether we expect to see # a response, which is effectively a memory leak... raise DuplicateRequestError( 'Reuse of requestId:{}'.format(requestId)) # If we've been told to shutdown (close() called) then fail request if self.dDown: return fail(ClientError('makeRequest() called after close()')) # Ok, we are going to save/send it, create a _Request object to track canceller = partial( self.cancelRequest, requestId, CancelledError("Request:{} was cancelled".format(requestId))) tReq = _Request(requestId, request, expectResponse, canceller) # add it to our requests dict self.requests[requestId] = tReq # Add an errback to the tReq.d to remove it from our requests dict # if something goes wrong... tReq.d.addErrback(self._handleRequestFailure, requestId) # Do we have a connection over which to send the request? if self.proto: # Send the request self._sendRequest(tReq) # Have we not even started trying to connect yet? Do so now elif not self.connector: self._connect() return tReq.d def addSubscriber(self, cb): """Add a callback to be called when the connection changes state.""" self.connSubscribers.append(cb) def delSubscriber(self, cb): """Remove a previously added 'subscriber' callback.""" if cb in self.connSubscribers: self.connSubscribers.remove(cb) def close(self): """Close the brokerclient's connection, cancel any pending requests.""" log.debug('%r: close proto:%r connector:%r', self, self.proto, self.connector) # Give our proto a 'heads up'... if self.proto is not None: self.proto.closing = True # Don't try to reconnect, and if we have an outstanding 'callLater', # cancel it. Also, if we are in the middle of connecting, stop # and call our clientConnectionFailed method with UserError self.stopTrying() # Ok, stopTrying() call above took care of the 'connecting' state, # now handle 'connected' state connector, self.connector = self.connector, None if connector and connector.state != "disconnected": # Create a deferred to return self.dDown = Deferred() connector.disconnect() else: # Fake a cleanly closing connection self.dDown = succeed(None) # Cancel any requests for tReq in self.requests.values(): # can't use itervalues() may del() tReq.d.cancel() return self.dDown def buildProtocol(self, addr): """Create a KafkaProtocol object, store it in self.proto, return it.""" # Schedule notification of subscribers self._get_clock().callLater(0, self._notify, True) # Build the protocol self.proto = ReconnectingClientFactory.buildProtocol(self, addr) # point it at us for notifications of arrival of messages self.proto.factory = self return self.proto def clientConnectionLost(self, connector, reason): """Handle notification from the lower layers of connection loss. If we are shutting down, and twisted sends us the expected type of error, eat the error. Otherwise, log it and pass it along. Also, schedule notification of our subscribers at the next pass through the reactor. """ if self.dDown and reason.check(ConnectionDone): # We initiated the close, this is an expected close/lost log.debug('%r: Connection Closed:%r:%r', self, connector, reason) notifyReason = None # Not a failure else: log.debug('%r: clientConnectionLost:%r:%r', self, connector, reason) notifyReason = reason # Reset our proto so we don't try to send to a down connection self.proto = None # Schedule notification of subscribers self._get_clock().callLater(0, self._notify, False, notifyReason) # Call our superclass's method to handle reconnecting ReconnectingClientFactory.clientConnectionLost(self, connector, reason) def clientConnectionFailed(self, connector, reason): """Handle notification from the lower layers of connection failure. If we are shutting down, and twisted sends us the expected type of error, eat the error. Otherwise, log it and pass it along. Also, schedule notification of our subscribers at the next pass through the reactor. """ if self.dDown and reason.check(UserError): # We initiated the close, this is an expected connectionFailed, # given we were trying to connect when close() was called log.debug('%r: clientConnectionFailed:%r:%r', self, connector, reason) notifyReason = None # Not a failure else: log.error('%r: clientConnectionFailed:%r:%r', self, connector, reason) notifyReason = reason # Reset our proto so we don't try to send to a down connection # Needed? I'm not sure we should even _have_ a proto at this point... self.proto = None # Schedule notification of subscribers self._get_clock().callLater(0, self._notify, False, notifyReason) # Call our superclass's method to handle reconnecting return ReconnectingClientFactory.clientConnectionFailed( self, connector, reason) def handleResponse(self, response): """Handle the response string received by KafkaProtocol. Ok, we've received the response from the broker. Find the requestId in the message, lookup & fire the deferred with the response. """ requestId = KafkaCodec.get_response_correlation_id(response) # Protect against responses coming back we didn't expect tReq = self.requests.pop(requestId, None) if tReq is None: # This could happen if we've sent it, are waiting on the response # when it's cancelled, causing us to remove it from self.requests log.warning('Unexpected response:%r, %r', requestId, response) else: tReq.d.callback(response) # # Private Methods # # def _sendRequest(self, tReq): """Send a single request over our protocol to the Kafka broker.""" try: tReq.sent = True self.proto.sendString(tReq.data) except Exception as e: log.exception('%r: request id: %d send failed:', self, tReq.id) del self.requests[tReq.id] tReq.d.errback(e) else: if not tReq.expect: # Once we've sent a request for which we don't expect a reply, # we're done, remove it from requests, and fire the deferred # with 'None', since there is no reply to be expected del self.requests[tReq.id] tReq.d.callback(None) def _sendQueued(self): """Connection just came up, send the unsent requests.""" for tReq in self.requests.values(): # can't use itervalues() may del() if not tReq.sent: self._sendRequest(tReq) def cancelRequest(self, requestId, reason=CancelledError(), _=None): """Cancel a request: remove it from requests, & errback the deferred. NOTE: Attempts to cancel a request which is no longer tracked (expectResponse == False and already sent, or response already received) will raise KeyError """ tReq = self.requests.pop(requestId) tReq.d.errback(reason) def _handlePending(self, reason): """Connection went down: handle in-flight & unsent as configured. Note: for now, we just 'requeue' all the in-flight by setting their 'sent' variable to False and let '_sendQueued()' handle resending when the connection comes back. In the future, we may want to extend this so we can errback() to our client's any in-flight (and possibly queued) so they can deal with it at the application level. """ for tReq in self.requests.itervalues(): tReq.sent = False return reason def _handleRequestFailure(self, failure, requestId): """Remove a failed request from our bookkeeping dict. Not an error if already removed (canceller removes). """ self.requests.pop(requestId, None) return failure def _get_clock(self): """Reactor to use for connecting, callLater, etc [for testing].""" if self.clock is None: from twisted.internet import reactor self.clock = reactor return self.clock def _connect(self): """Initiate a connection to the Kafka Broker.""" log.debug('%r: _connect', self) # We can't connect, we're not disconnected! if self.connector: raise ClientError('_connect called but not disconnected') # Needed to enable retries after a disconnect self.resetDelay() self.connector = self._get_clock().connectTCP(self.host, self.port, self) log.debug('%r: _connect got connector: %r', self, self.connector) def _notify(self, connected, reason=None, subs=None): """Notify the caller of :py:method:`close` of completion. Also notify any subscribers of the state change of the connection. """ if connected: self._sendQueued() else: if self.dDown and not self.dDown.called: self.dDown.callback(reason) # If the connection just went down, we need to handle any # outstanding requests. self._handlePending(reason) # Notify if requested. We call all of the callbacks, but don't # wait for any returned deferreds to fire here. Instead we add # them to a deferredList which we check for and wait on before # calling any callbacks for subsequent events. # This should keep any state-changes done by these callbacks in the # proper order. Note however that the ordering of the individual # callbacks in each (connect/disconnect) list isn't guaranteed, and # they can all be progressing in parallel if they yield or otherwise # deal with deferreds if self.notifydList: # We already have a notify list in progress, so just call back here # when the deferred list fires, with the _current_ list of subs subs = list(self.connSubscribers) self.notifydList.addCallback( lambda _: self._notify(connected, reason=reason, subs=subs)) return # Ok, no notifications currently in progress. Notify all the # subscribers, keep track of any deferreds, so we can make sure all # the subs have had a chance to completely process this event before # we send them any new ones. dList = [] if subs is None: subs = list(self.connSubscribers) for cb in subs: dList.append(maybeDeferred(cb, self, connected, reason)) self.notifydList = DeferredList(dList) def clearNotifydList(_): """Reset the notifydList once we've notified our subscribers.""" self.notifydList = None self.notifydList.addCallback(clearNotifydList)
def __new__(mcs, name, bases, attrs): attrs["_meta"] = opts = TableOptions(attrs.get("Meta", None), name) # extract declared columns cols, remainder = [], {} for attr_name, attr in attrs.items(): if isinstance(attr, columns.Column): attr._explicit = True cols.append((attr_name, attr)) else: remainder[attr_name] = attr attrs = remainder cols.sort(key=lambda x: x[1].creation_counter) # If this class is subclassing other tables, add their fields as # well. Note that we loop over the bases in *reverse* - this is # necessary to preserve the correct order of columns. parent_columns = [] for base in reversed(bases): if hasattr(base, "base_columns"): parent_columns = list( base.base_columns.items()) + parent_columns # Start with the parent columns base_columns = OrderedDict(parent_columns) # Possibly add some generated columns based on a model if opts.model: extra = OrderedDict() # honor Table.Meta.fields, fallback to model._meta.fields if opts.fields is not None: # Each item in opts.fields is the name of a model field or a # normal attribute on the model for field_name in opts.fields: field = Accessor(field_name).get_field(opts.model) extra[field_name] = columns.library.column_for_field(field) else: for field in opts.model._meta.fields: extra[field.name] = columns.library.column_for_field(field) # update base_columns with extra columns for key, col in extra.items(): # skip current col because the parent was explicitly defined, # and the current column is not. if key in base_columns and base_columns[key]._explicit is True: continue base_columns[key] = col # Explicit columns override both parent and generated columns base_columns.update(OrderedDict(cols)) # Apply any explicit exclude setting for exclusion in opts.exclude: if exclusion in base_columns: base_columns.pop(exclusion) # Remove any columns from our remainder, else columns from our parent class will remain for attr_name in remainder: if attr_name in base_columns: base_columns.pop(attr_name) # Set localize on columns for col_name in base_columns.keys(): localize_column = None if col_name in opts.localize: localize_column = True # unlocalize gets higher precedence if col_name in opts.unlocalize: localize_column = False if localize_column is not None: base_columns[col_name].localize = localize_column attrs["base_columns"] = base_columns return super(DeclarativeColumnsMetaclass, mcs).__new__(mcs, name, bases, attrs)
class FitSettingsDialog(QtWidgets.QDialog): """ A dialog that is used to configure the fits in a FitContainer. """ sigFitsUpdated = QtCore.Signal(dict) def __init__(self, fit_container): """ Create a FitSettingsDialog for a matching FitContainer. @param fit_container FitContainer: the FitContainer that this dialog should manipulate """ super().__init__() self.fc = fit_container self.title = '{0} fit settings'.format(self.fc.name) # set up window self.setModal(False) self.setWindowTitle(self.title) # variables self.all_functions = self.fc.fit_logic.fit_list[self.fc.dimension] self.tabs = {} self.parameters = {} self.parameterUse = {} self.currentFits = OrderedDict() self.fitWidgets = OrderedDict() self.currentFitWidgets = OrderedDict() # widgets and layouts self._dialogLayout = QtWidgets.QVBoxLayout() self._btnLayout = QtWidgets.QHBoxLayout() self._tabWidget = QtWidgets.QTabWidget() self._firstPage = QtWidgets.QWidget() self._firstPageLayout = QtWidgets.QVBoxLayout() self._scrollArea = QtWidgets.QScrollArea() self._scrollWidget = QtWidgets.QWidget() self._scrLayout = QtWidgets.QVBoxLayout() self._dbox = QtWidgets.QDialogButtonBox( QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Apply | QtWidgets.QDialogButtonBox.Cancel, QtCore.Qt.Horizontal ) self._addFitButton = QtWidgets.QPushButton('Add fit') self._saveFitButton = QtWidgets.QPushButton('Save fits') self._loadFitButton = QtWidgets.QPushButton('Load fits') # layout construction self._scrollWidget.setLayout(self._scrLayout) self._scrollArea.setWidget(self._scrollWidget) self._scrollArea.setWidgetResizable(True) self._btnLayout.addWidget(self._addFitButton) self._btnLayout.addWidget(self._saveFitButton) self._btnLayout.addWidget(self._loadFitButton) self._firstPageLayout.addLayout(self._btnLayout) self._firstPageLayout.addWidget(self._scrollArea) self._firstPage.setLayout(self._firstPageLayout) self._tabWidget.addTab(self._firstPage, 'Fit functions') self._dialogLayout.addWidget(self._tabWidget) self._dialogLayout.addWidget(self._dbox) self.setLayout(self._dialogLayout) # connections self._dbox.accepted.connect(self.accept) self._dbox.rejected.connect(self.reject) self._dbox.clicked.connect(self.buttonClicked) self.accepted.connect(self.applySettings) self.rejected.connect(self.resetSettings) self._addFitButton.clicked.connect(self.addFitButtonClicked) self._loadFitButton.clicked.connect(self.loadFitButtonClicked) self._saveFitButton.clicked.connect(self.saveFitButtonClicked) self.sigFitsUpdated.connect(self.fc.set_fit_functions) self.fc.sigNewFitParameters.connect(self.updateParameters) # load user defined fits from fit container if len(self.fc.fit_list) > 0: self.loadFits(self.fc.fit_list) @QtCore.Slot(QtWidgets.QAbstractButton) def buttonClicked(self, button): """ Slot for signals from dialog button box. @param button QAbstractButton: designates which button was clicked. """ if self._dbox.buttonRole(button) == QtWidgets.QDialogButtonBox.ApplyRole: self.applySettings() @QtCore.Slot() def addFitButtonClicked(self): """ The 'Add Fit' button was clicked. Display a name input dialog and add the fit. """ res = QtWidgets.QInputDialog.getText( self, 'New fit', 'Enter a name for this fit:', ) if res[1]: self.addFit(res[0]) @QtCore.Slot() def saveFitButtonClicked(self): """ The 'Save Fits' button was clicked. Display file chooser and save fits to file. """ res = QtWidgets.QFileDialog.getSaveFileName( self, 'Save fit collection for {0}'.format(self.title), '/path', 'Fit files (*.fit *.yml)' ) self.fc.fit_logic.save_fits(res[0], {self.fc.dimension: self.fc.fit_list}) @QtCore.Slot() def loadFitButtonClicked(self): """ The 'Load Fits' button was clicked. Display file chooser and load fits from file. """ res = QtWidgets.QFileDialog.getOpenFileName( self, 'Load fit collection for {0}'.format(self.title), '/path', 'Fit files (*.fit *.yml)' ) fits = self.fc.fit_logic.load_fits(res[0]) self.loadFits(fits[self.fc.dimension]) def loadFits(self, user_fits): """ Take a fit config dictionary and create widgets for the fits inside. @param user_fits dict: configured fits dictionary """ self.removeAllFits() self.applySettings() for name, fit in user_fits.items(): self.addFit(name, fit=fit['fit_name'], estimator=fit['est_name']) # add new tab for new fit model, params = self.all_functions[fit['fit_name']]['make_model']() self.tabs[name] = FitParametersWidget(params) self._tabWidget.addTab(self.tabs[name], name) self.updateParameters(name, fit['parameters']) # build fit list and send update signals self.applySettings() def addFit(self, name, fit=None, estimator=None): """ Add a new fit to the dialog. @param name str: configured name for fit @param fit str: name of the fit function for this fit @param estimator str: name of the estimator function for this fit """ if len(name) < 1: return if name in self.fitWidgets: logging.error('{0}: Fit {1} already exists.'.format(self.title, name)) return fcw = FitConfigWidget(name, self.all_functions, fit, estimator) self.currentFitWidgets[name] = fcw self._scrLayout.addWidget(fcw) fcw.sigRemoveFit.connect(self.removeFit) fcw.show() @QtCore.Slot(str) def removeFit(self, name): """ Remove a fit from the dialog. Hides the FitonfigWidet and disables the parameter tab. @param name str: name of fit to remove """ widget = self.currentFitWidgets.pop(name) widget.hide() self._scrLayout.removeWidget(widget) tab = self.tabs[name] tab.setEnabled(False) def removeAllFits(self): """ Remove all configured fits from dialog. """ for name in self.currentFits.keys(): self.removeFit(name) def getFits(self): """ Return all configured fits from this dialog. """ return self.currentFits def applySettings(self): """ Apply all settings that the user has made in the dialog and send out update signals. This copies all input widget values to thei coresponding internal data structures, creates an removes widgets and parameter tabs. """ # remove all settings tabs and config widgets that the user removed for name, widget in self.fitWidgets.items(): if name not in self.currentFitWidgets: tab = self.tabs.pop(name) index = self._tabWidget.indexOf(tab) if index >= 0: self._tabWidget.removeTab(index) self._scrLayout.removeWidget(widget) self.fitWidgets = OrderedDict() # add tabs for new fits, replace tabs for changed fits for name, widget in self.currentFitWidgets.items(): oldfit = widget.fit oldest = widget.estimator widget.applySettings() if name in self.tabs and widget.fit != oldfit: # remove old tab, add new tab, preferably in the same place, for changed fit tab = self.tabs.pop(name) index = self._tabWidget.indexOf(tab) if index >= 0: self._tabWidget.removeTab(index) model, params = self.all_functions[widget.fit]['make_model']() self.tabs[name] = FitParametersWidget(params) if index >= 0: self._tabWidget.insertTab(index, self.tabs[name], name) else: self._tabWidget.addTab(self.tabs[name], name) elif name not in self.tabs: # add new tab for new fir model, params = self.all_functions[widget.fit]['make_model']() self.tabs[name] = FitParametersWidget(params) self._tabWidget.addTab(self.tabs[name], name) # put all widgets here self.fitWidgets[name] = widget # reset update tabs and put new values in dict self.parameters = {} self.parameterUse = {} for name, tab in self.tabs.items(): self.parameters[name], self.parameterUse[name] = tab.applyFitParameters() self.buildCurrentFits() self.sigFitsUpdated.emit(self.currentFits) def buildCurrentFits(self): """ Update dictionary of the configured fits for FitContainer or other componenrs. """ # arrange all of this information in a convenient form self.currentFits = OrderedDict() for name, widget in self.fitWidgets.items(): try: self.currentFits[name] = { 'fit_name': widget.fit, 'est_name': widget.estimator, 'make_fit': self.all_functions[widget.fit]['make_fit'], 'make_model': self.all_functions[widget.fit]['make_model'], 'estimator': self.all_functions[widget.fit][widget.estimator], 'parameters': self.parameters[name], 'use_settings': self.parameterUse[name] } except KeyError: continue def resetSettings(self): """ Reset all input widgets to their stored values, discarding changes by the user. """ for name, widget in self.currentFitWidgets.items(): if name not in self.fitWidgets: self._scrLayout.removeWidget(widget) self.currentFitWidgets = OrderedDict() for name, widget in self.fitWidgets.items(): widget.resetSettings() widget.show() self.currentFitWidgets[name] = widget # reset tabs, do not update dicts for name, tab in self.tabs.items(): tab.setEnabled(True) tab.resetFitParameters() def getParameters(self, fit_name): """ Return Parameters object for a given fit. @param fit_name str: name of the fit @return Parameters: lmfit parameters container """ return self.parameters[fit_name] def updateParameters(self, fit_name, parameters): """ Update parameters of a given fit. @param fit_name str: name of fit @param parameters Parameters: lmfit Parameters container This function updates all parameters for a fit that the dialog has stored and ingores any other parameters in he parameter container. """ if fit_name in self.parameters: for name, param in parameters.items(): if name in self.parameters[fit_name]: self.parameters[fit_name][name] = param self.tabs[fit_name].updateFitParameters(parameters)
def evaluate(self, results, metric='mIoU', logger=None, efficient_test=False, **kwargs): """Evaluate the dataset. Args: results (list): Testing results of the dataset. metric (str | list[str]): Metrics to be evaluated. 'mIoU', 'mDice' and 'mFscore' are supported. logger (logging.Logger | None | str): Logger used for printing related information during evaluation. Default: None. Returns: dict[str, float]: Default metrics. """ if isinstance(metric, str): metric = [metric] allowed_metrics = ['mIoU', 'mDice', 'mFscore'] if not set(metric).issubset(set(allowed_metrics)): raise KeyError('metric {} is not supported'.format(metric)) eval_results = {} gt_seg_maps = self.get_gt_seg_maps(efficient_test) if self.CLASSES is None: num_classes = len( reduce(np.union1d, [np.unique(_) for _ in gt_seg_maps])) else: num_classes = len(self.CLASSES) ret_metrics = eval_metrics( results, gt_seg_maps, num_classes, self.ignore_index, metric, label_map=self.label_map, reduce_zero_label=self.reduce_zero_label) if self.CLASSES is None: class_names = tuple(range(num_classes)) else: class_names = self.CLASSES # summary table ret_metrics_summary = OrderedDict({ ret_metric: np.round(np.nanmean(ret_metric_value) * 100, 2) for ret_metric, ret_metric_value in ret_metrics.items() }) # each class table ret_metrics.pop('aAcc', None) ret_metrics_class = OrderedDict({ ret_metric: np.round(ret_metric_value * 100, 2) for ret_metric, ret_metric_value in ret_metrics.items() }) ret_metrics_class.update({'Class': class_names}) ret_metrics_class.move_to_end('Class', last=False) # for logger class_table_data = PrettyTable() for key, val in ret_metrics_class.items(): class_table_data.add_column(key, val) summary_table_data = PrettyTable() for key, val in ret_metrics_summary.items(): if key == 'aAcc': summary_table_data.add_column(key, [val]) else: summary_table_data.add_column('m' + key, [val]) print_log('per class results:', logger) print_log('\n' + class_table_data.get_string(), logger=logger) print_log('Summary:', logger) print_log('\n' + summary_table_data.get_string(), logger=logger) # each metric dict for key, value in ret_metrics_summary.items(): if key == 'aAcc': eval_results[key] = value / 100.0 else: eval_results['m' + key] = value / 100.0 ret_metrics_class.pop('Class', None) for key, value in ret_metrics_class.items(): eval_results.update({ key + '.' + str(name): value[idx] / 100.0 for idx, name in enumerate(class_names) }) if mmcv.is_list_of(results, str): for file_name in results: os.remove(file_name) return eval_results
class Bibliography: def __init__(self, entries: dict or list): if isinstance(entries, dict): self._data: OrderedDict = OrderedDict( {k: Entry.entry_factory(v) for k, v in entries.items()}) elif isinstance(entries, list): self._data: OrderedDict = OrderedDict( {e['ID']: Entry.entry_factory(e) for e in entries}) else: raise WrongEntryTypeError( f"Expected list or dict, got {type(entries)}") self._handle_duplicates() def __len__(self): return len(self._data) def __add__(self, other): if not isinstance(other, Bibliography): raise NotBibliographyError() for k, v in other._data.items(): if k in self._data: self._data[k].update(v) else: self._data[k] = v self.sort() self._handle_duplicates() return self def __sub__(self, other): if not isinstance(other, Bibliography): raise NotBibliographyError() for k, v in other._data.items(): if self._data[k] == v: self._data.pop(k, None) self.sort() self._handle_duplicates() return self def __iter__(self): raise NotImplementedError("Use entries or entries_dict") def __contains__(self, item): return any((entry == item) for entry in self.entries) @property def entries(self) -> list: return [e for e in self._data.values()] @property def entries_dict(self): return self._data @property def authors_years_dict(self) -> dict: return {k: entry.author_year for k, entry in self._data.items()} @property def authors_years(self) -> list: return [entry for entry in self.authors_years_dict.values()] @property def unique_authors_years(self) -> dict: unique_authors_years = {} for k, v in self.authors_years_dict.items(): if v in unique_authors_years: unique_authors_years[v].append(k) else: unique_authors_years[v] = [k] return unique_authors_years def sort(self): self._data = OrderedDict( sorted(self._data.items(), key=lambda key_val: key_val[1])) def _handle_duplicates(self): self.sort() # delete duplicates duplicates = [] last_entry = object() for k, entry in self._data.items(): if entry == last_entry: duplicates.append(k) last_entry = entry for k in duplicates: self._data.pop(k, None) # set numbers/letters for pseudo-duplicates for k, v in self.unique_authors_years.items(): if 1 < len(v): for i, id_ in enumerate(v): self._data[id_]['letter_number'] = i + 1 else: id_ = v[0] self._data[id_]['letter_number'] = None
def make_editor_session(pad, path, is_attachment=None, alt=PRIMARY_ALT, datamodel=None): """Creates an editor session for the given path object.""" if alt != PRIMARY_ALT and not pad.db.config.is_valid_alternative(alt): raise BadEdit("Attempted to edit an invalid alternative (%s)" % alt) raw_data = pad.db.load_raw_data(path, cls=OrderedDict, alt=alt, fallback=False) raw_data_fallback = None if alt != PRIMARY_ALT: raw_data_fallback = pad.db.load_raw_data(path, cls=OrderedDict) all_data = OrderedDict() all_data.update(raw_data_fallback or ()) all_data.update(raw_data or ()) else: all_data = raw_data id = posixpath.basename(path) if not is_valid_id(id): raise BadEdit("Invalid ID") record = None exists = raw_data is not None or raw_data_fallback is not None if raw_data is None: raw_data = OrderedDict() if is_attachment is None: if not exists: is_attachment = False else: is_attachment = bool(all_data.get("_attachment_for")) elif bool(all_data.get("_attachment_for")) != is_attachment: raise BadEdit("The attachment flag passed is conflicting with the " "record's attachment flag.") if exists: # XXX: what about changing the datamodel after the fact? if datamodel is not None: raise BadEdit("When editing an existing record, a datamodel " "must not be provided.") datamodel = pad.db.get_datamodel_for_raw_data(all_data, pad) else: if datamodel is None: datamodel = pad.db.get_implied_datamodel(path, is_attachment, pad) elif isinstance(datamodel, str): datamodel = pad.db.datamodels[datamodel] if exists: record = pad.instance_from_data(dict(all_data), datamodel) for key in implied_keys: raw_data.pop(key, None) if raw_data_fallback: raw_data_fallback.pop(key, None) return EditorSession( pad, id, str(path), raw_data, raw_data_fallback, datamodel, record, exists, is_attachment, alt, )
class EventListener(object): """``Session``, ``Call`` and ``Job`` tracking through a default set of event handlers. Tracks various session entities by wrapping received event data in local ``models`` APIs and/or data structures. Serves as a higher level API on top of the underlying event loop. """ def __init__( self, event_loop, call_tracking_header='variable_call_uuid', max_limit=float('inf'), ): """ :param str call_tracking_header: Name of the freeswitch variable (including the 'variable_' prefix) to use for associating sessions into tracked calls (see `_handle_create`). It is common to set this to an Xheader variable if attempting to track calls "through" an intermediary device (i.e. the first hop receiving requests) such as a B2BUA. NOTE: in order for this association mechanism to work the intermediary device must be configured to forward the Xheaders it receives. """ self.event_loop = event_loop self.sessions = OrderedDict() self.log = utils.get_logger(utils.pstr(self)) # store last 1k of each type of failed session self.failed_sessions = OrderedDict() self.bg_jobs = OrderedDict() self.calls = OrderedDict() # maps aleg uuids to Sessions instances self.hangup_causes = Counter() # record of causes by category self.sessions_per_app = Counter() self.max_limit = max_limit self.call_tracking_header = call_tracking_header # state reset self.reset() # add default handlers for evname, cbtype, cb in get_callbacks(self, only='handler'): self.event_loop.add_handler(evname, cb) def register_job(self, future, **kwargs): '''Register for a job to be handled when the appropriate event arrives. Once an event corresponding to the job is received, the bgjob event handler will 'consume' it and invoke its callback. Parameters ---------- event : ESL.ESLevent as returned from an ESLConnection.bgapi call kwargs : dict same signatures as for Job.__init__ Returns ------- bj : an instance of Job (a background job) ''' bj = Job(future, **kwargs) self.bg_jobs[bj.uuid] = bj return bj def count_jobs(self): return len(self.bg_jobs) def count_sessions(self): return len(self.sessions) def count_calls(self): '''Count the number of active calls hosted by the slave process ''' return len(self.calls) def count_failed(self): '''Return the failed session count ''' return sum(self.hangup_causes.values() ) - self.hangup_causes['NORMAL_CLEARING'] def reset(self): '''Clear all internal stats and counters ''' self.log.debug('resetting all stats...') self.hangup_causes.clear() self.failed_jobs = Counter() self.total_answered_sessions = 0 @handler('CHANNEL_HANGUP') @handler('CHANNEL_PARK') @handler('CALL_UPDATE') def lookup_sess(self, e): """The most basic handler template which looks up the locally tracked session corresponding to event `e` and updates it with event data """ sess = self.sessions.get(e.get('Unique-ID'), False) if sess: sess.update(e) return True, sess return False, None def lookup_sess_and_job(self, e): """Look up and return the session and any corresponding background job. """ consumed, sess = self.lookup_sess(e) if consumed: return True, sess, sess.bg_job return False, None, None @handler('LOG') def _handle_log(self, e): self.log.info(e.get('Body')) return True, None @handler('SERVER_DISCONNECTED') def _handle_disconnect(self, e): """Log disconnects. """ self.log.warning("Received DISCONNECT from server '{}'".format( self.host)) return True, None @handler('BACKGROUND_JOB') def _handle_bj(self, e): '''Handle bjs and report failures. If a job is found in the local cache then update the instance with event data. This handler returns 'None' on error (i.e. failed bj) which must be handled by any callbacks. ''' error = False consumed = False resp = None sess = None ok = '+OK ' err = '-ERR' job_uuid = e.get('Job-UUID') body = e.get('Body') # always report errors even for jobs which we aren't tracking if err in body: resp = body.strip(err).strip() error = True self.log.debug("job '{}' failed with:\n{}".format( job_uuid, str(body))) elif ok in body: resp = body.strip(ok + '\n') job = self.bg_jobs.get(job_uuid, None) job_just_created = False if not job: job = Job(event=e) job_just_created = True else: job.events.update(e) # attempt to lookup an associated session sess = self.sessions.get(job.sess_uuid or resp, None) if error: # if the job returned an error, report it and remove the job self.log.error("Job '{}' corresponding to session '{}'" " failed with:\n{}".format(job_uuid, job.sess_uuid, str(body))) job.fail(resp) # fail the job if not job_just_created: # always pop failed jobs self.bg_jobs.pop(job_uuid) # append the id for later lookup and discard? self.failed_jobs[resp] += 1 consumed = True else: # OK case if sess: # special case: the bg job event returns a known originated # session's (i.e. pre-registered) uuid in its body if job.sess_uuid: assert str(job.sess_uuid) == str(resp), \ ("""Session uuid '{}' <-> BgJob uuid '{}' mismatch!? """.format(job.sess_uuid, resp)) # reference this job in the corresponding session # self.sessions[resp].bg_job = job sess.bg_job = job self.log.debug("Job '{}' was sucessful".format(job_uuid)) consumed = True else: self.log.warn( "No session corresponding to bj '{}'".format(job_uuid)) # run the job's callback job(resp) return consumed, sess, job @handler('CHANNEL_CREATE') @handler('CHANNEL_ORIGINATE') def _handle_initial_event(self, e): '''Handle channel create events by building local `Session` and `Call` objects for state tracking. ''' uuid = e.get('Unique-ID') # Record the newly activated session # TODO: pass con as weakref? con = self.event_loop._con # short circuit if we have already allocated a session since FS is # indeterminate about which event create|originate will arrive first sess = self.sessions.get(uuid) if sess: return True, sess # allocate a session model sess = Session(e, event_loop=self.event_loop, uuid=uuid, con=con) direction = sess['Call-Direction'] self.log.debug("{} session created with uuid '{}'".format( direction, uuid)) sess.cid = self.event_loop.get_id(e, 'default') # Use our specified "call identification variable" to try and associate # sessions into calls. By default the 'variable_call_uuid' channel # variable is used for tracking locally bridged calls call_uuid = e.get(self.call_tracking_header) # could be 'None' if not call_uuid: self.log.warn( "Unable to associate {} session '{}' with a call using " "variable '{}'".format(direction, sess.uuid, self.call_tracking_header)) call_uuid = uuid # associate sessions into a call # (i.e. set the relevant sessions to reference each other) if call_uuid in self.calls: call = self.calls[call_uuid] self.log.debug("session '{}' is bridged to call '{}'".format( uuid, call.uuid)) # append this session to the call's set call.append(sess) else: # this sess is not yet tracked so use its id as the 'call' id call = Call(call_uuid, sess) self.calls[call_uuid] = call self.log.debug("call created for session '{}'".format(call_uuid)) sess.call = call self.sessions[uuid] = sess self.sessions_per_app[sess.cid] += 1 return True, sess @handler('CHANNEL_ANSWER') def _handle_answer(self, e): '''Handle answer events Returns ------- sess : session instance corresponding to uuid ''' uuid = e.get('Unique-ID') sess = self.sessions.get(uuid, None) if sess: self.log.debug("answered {} session '{}'".format( e.get('Call-Direction'), uuid)) sess.answered = True self.total_answered_sessions += 1 sess.update(e) return True, sess else: self.log.warn('Skipping answer of {}'.format(uuid)) return False, None @handler('CHANNEL_DESTROY') # @handler('CHANNEL_HANGUP_COMPLETE') # XXX: a race between these two... def _handle_destroy(self, e): '''Handle channel destroy events. Returns ------- sess : session instance corresponding to uuid job : corresponding bj for a session if exists, ow None ''' uuid = e.get('Unique-ID') sess = self.sessions.pop(uuid, None) direction = sess['Call-Direction'] if sess else 'unknown' if not sess: return False, None sess.update(e) sess.hungup = True cause = e.get('Hangup-Cause') self.hangup_causes[cause] += 1 # count session causes self.sessions_per_app[sess.cid] -= 1 # if possible lookup the relevant call call_uuid = e.get(self.call_tracking_header) if not call_uuid: self.log.warn( "handling HANGUP for {} session '{}' which can not be " "associated with an active call using {}?".format( direction, sess.uuid, self.call_tracking_header)) call_uuid = uuid # XXX seems like sometimes FS changes the `call_uuid` # between create and hangup oddly enough call = self.calls.get(call_uuid, sess.call) if call: if sess in call.sessions: self.log.debug("hungup {} session '{}' for Call '{}'".format( direction, uuid, call.uuid)) call.sessions.remove(sess) else: # session was somehow tracked by the wrong call self.log.err("session '{}' mismatched with call '{}'?".format( sess.uuid, call.uuid)) # all sessions hungup if len(call.sessions) == 0: self.log.debug( "all sessions for call '{}' were hung up".format( call_uuid)) # remove call from our set call = self.calls.pop(call.uuid, None) if not call: self.log.warn( "Call with id '{}' containing Session '{}' was " "already removed".format(call.uuid, sess.uuid)) else: # we should never get hangups for calls we never saw created self.log.err("no call found for '{}'".format(call_uuid)) # pop any corresponding job job = sess.bg_job # may have been popped by the partner self.bg_jobs.pop(job.uuid if job else None, None) sess.bg_job = None # deref job - avoid mem leaks if not sess.answered or cause != 'NORMAL_CLEARING': self.log.debug("'{}' was not successful??".format(sess.uuid)) self.failed_sessions.setdefault(cause, deque(maxlen=1000)).append(sess) self.log.debug("hungup Session '{}'".format(uuid)) # hangups are always consumed return True, sess, job @property def host(self): return self.event_loop.host @property def port(self): return self.event_loop.port def is_alive(self): return self.event_loop.is_alive() def is_running(self): return self.event_loop.is_running() def connect(self, **kwargs): return self.event_loop.connect(**kwargs) def connected(self): return self.event_loop.connected() def start(self): return self.event_loop.start() def disconnect(self, **kwargs): return self.event_loop.disconnect(**kwargs) def unsubscribe(self, evname): return self.event_loop.unsubscribe(evname)
class SkillInterface(SkillCore): """ """ #--------Class functions-------- def __init__(self, children_processor=Sequential()): super(SkillInterface, self).__init__() # Params self._remaps = OrderedDict() # Connections self._parent = None self._children = [] self._children_processor = children_processor # Caches self._was_simulated = False self._max_cache = 2 self._params_cache = [] self._input_cache = [] self._remaps_cache = defaultdict(list) def __call__(self, *children): """ @brief Add a set of children skills @return self for nested children declarations """ for c in children: self.addChild(c) return self @property def parent(self): """ @brief Returns the parent skill """ return self._parent def getLightCopy(self): """ @brief Makes a light copy (only description, params and state) @return A light copy """ p = self.__class__(self._children_processor) p._children_processor = deepcopy(self._children_processor) p._type = copy(self._type) p._label = copy(self._label) p._params = deepcopy(self._params) p._remaps = deepcopy(self._remaps) p._description = deepcopy(self._description) p._pre_conditions = deepcopy(self._pre_conditions) p._hold_conditions = deepcopy(self._hold_conditions) p._post_conditions = deepcopy(self._post_conditions) if self._state != State.Uninitialized: p._state = copy(self._state) p._wmi = self._wmi return p def _setState(self, state): self._state = state self._state_change.set() def _clearRemaps(self): """ @brief Clear remaps """ for r1, r2 in reversed(self._remaps.items()): self.remap(r2, r1) self._remaps = OrderedDict() self._remaps_cache = defaultdict(list) def _copyRemaps(self, skill): """ @brief Copy the remaps of another skill. Called automatically when the skill is added as a child """ for r1, r2 in skill._remaps.iteritems(): self.remap(r1, r2) def _revertRemaps(self): """ @brief Revert remaps. Just used in revertInput """ remapid = len(self._params_cache) try: if self._remaps_cache[remapid]: for remap in self._remaps_cache[remapid]: log.warn("REMAP", "Revert {} {}".format(remap[0], remap[1])) print self._remaps_cache self._remaps.pop(remap[0]) self.remap(remap[1], remap[0], False) del self._remaps_cache[remapid] except BaseException: print self.printInfo(True) print self._remaps_cache raise def getParamsNoRemaps(self): """ @brief Get the skill's parameters without key remappings """ ph = params.ParamHandler() ph.reset(self._params.getCopy()) for r1, r2 in reversed(self._remaps.items()): ph.remap(r2, r1) return ph def get_remap(self, key): """ @brief Return the key remapping if existing or otherwise the key unchanged """ return self._remaps[key] if key in self._remaps else key def remap(self, initial_key, target_key, record=True): """ @brief Remap a parameter with initial_key to a new target_key All skill's children are remapped too. """ # Ignore harmful remappings if initial_key in self._remaps: if self._remaps[initial_key] == target_key: # Redundant #log.warn(self.type, "Ignoring redundant remap {}->{}".format(initial_key, target_key)) return else: # Already remapped log.warn( self.type, "Key {} already remapped to {}. Can t remap to {}".format( initial_key, self._remaps[initial_key], target_key)) return #log.warn("Remap", "{} {} {}. Existing: {}".format(self.type, initial_key, target_key, self._remaps)) # The current 2->3 is related to an existing remap 1->2 # if initial_key in self._remaps.values(): # log.info("ChainRemap", "{}: {}->{}->{}".format(self.type, self._remaps.keys()[self._remaps.values().index(initial_key)], initial_key, target_key)) for c in self._children: c.remap(initial_key, target_key) if target_key in self._remaps: # Asking remap 1->2 but exists a remap 2->3. Records a remap 1->3 target_key = self.get_remap(target_key) if self.params.hasParam(target_key): log.error( "Remap", "{}: Invalid remapping {}->{}, target key is already present.". format(self.type, initial_key, target_key)) return if self.params.hasParam(initial_key): #log.warn("{}".format(self.type), "REMAPPING: {} {}".format(initial_key, target_key)) # Remaps self._params.remap(initial_key, target_key) for c in self._pre_conditions: c.remap(initial_key, target_key) for c in self._hold_conditions: c.remap(initial_key, target_key) for c in self._post_conditions: c.remap(initial_key, target_key) # Records if record: self._remaps[initial_key] = target_key remapid = len(self._params_cache) if remapid > 0: # If I have params cached, I cache also the remap to revert it afterwards self._remaps_cache[remapid].append((initial_key, target_key)) def init(self, wmi): self._wmi = wmi self.createDescription() self.modifyDescription(self) self._setState(State.Idle) def inSubtreeOf(self, skill): if self in skill._children: return True for c in skill._children: if self.inSubtreeOf(c): return True return False def hold(self): for c in self._hold_conditions: if not c.setTrue(self._params, self._wmi): log.error(c.getDescription(), "Hold failed.") return False return True def revertHold(self): for c in reversed(self._hold_conditions): #print c.getDescription() if not c.revert(self._params, self._wmi): log.error(c.getDescription(), "Revert hold failed.") return False self._was_simulated = False return True def simulate(self): self._was_simulated = True #print "Simulate: {}".format(self.printInfo(True)) for c in self._post_conditions: if not c.setTrue(self._params, self._wmi): log.error(c.getDescription(), "Simulation failed.") return State.Failure return State.Success def revertSimulation(self): if not self._was_simulated: log.warn("revert", "No simulation was made, can't revert.") return False for c in reversed(self._post_conditions): if not c.revert(self._params, self._wmi): log.error(c.getDescription(), "Revert failed.") return False self._was_simulated = False return True def visit(self, visitor): return visitor.process(self) def visitPreempt(self, visitor): return visitor.processPreempt(self) def getParent(self): return self._parent def hasChildren(self): return len(self._children) > 0 def addChild(self, p, latch=False): p._parent = self self._children.append(p) p._copyRemaps(self) if latch and len(self._children) > 1: for c in self._children[-2]._post_conditions: for key in c.getKeys(): if not p.params.hasParam(key): p.params[key] = deepcopy( self._children[-2]._params._params[key]) p._pre_conditions += deepcopy(self._children[-2]._post_conditions) return self def last(self): return self._children[-1] def popChild(self): child = self._children.pop() child._parent = None def specifyParam(self, key, values): """ Specify a parameter and update the input cache """ super(SkillInterface, self).specifyParam(key, values) if self._input_cache: if key in self._input_cache[-1]: self._input_cache[-1][key].setValues(values) def specifyParamsDefault(self, input_params): """ Set the parameters and makes them default (they will no more be overwritten by setInput) """ self._params_cache.append(self._params.getCopy()) self._input_cache.append(input_params.getCopy()) super(SkillInterface, self).specifyParamsDefault(input_params) def specifyParams(self, input_params, keep_default=True): """ Set the parameters. Params already specified are preserved """ self._params_cache.append(self._params.getCopy()) self._input_cache.append(input_params.getCopy()) super(SkillInterface, self).specifyParams(input_params, keep_default) # Erase the oldest cache if the max lenght is reached if len(self._params_cache) > self._max_cache: self._popCache() def _popCache(self): self._params_cache.pop(0) for i in range(self._max_cache + 1): if i not in self._remaps_cache: continue del self._remaps_cache[i] if i + 1 in self._remaps_cache: self._remaps_cache[i] = self._remaps_cache[i + 1] def revertInput(self): if not self._params_cache: log.warn("revertInput", "No cache available, can't revert input.") return None self._revertRemaps() self._params.reset(deepcopy(self._params_cache.pop())) return deepcopy(self._input_cache.pop()) def start(self, params=None): self._children_processor.reset() return SkillCore.start(self, params) def tick(self): res = self.execute() # print "{} {}".format(self, res) if res is not None: self._setState(res) if not self.onEnd(): self._setState(State.Failure) res = State.Failure return res #--------User functions-------- def resetDescription(self, other=None): if other: self._params.reset(self._description._params.merge(other._params)) else: self._params = deepcopy(self._description._params) self._pre_conditions = list() self._hold_conditions = list() self._post_conditions = list() for p in self._description._pre_conditions: self.addPreCondition(deepcopy(p)) for p in self._description._hold_conditions: self.addHoldCondition(deepcopy(p)) for p in self._description._post_conditions: self.addPostCondition(deepcopy(p)) def mergeDescription(self, other): self.resetDescription(other) for c in other._pre_conditions: if not c in self._pre_conditions: self.addPreCondition(c) for c in other._hold_conditions: if not c in self._hold_conditions: self.addHoldCondition(c) for c in other._post_conditions: if not c in self._post_conditions: self.addPostCondition(c) def addParam(self, key, value, param_type, options=[], description=""): """ @brief Check for remaps before adding a parameter. See SkillDescription for more details """ if key in self._remaps: key = self._remaps[key] SkillDescription.addParam(self, key, value, param_type, options, description) def addPreCondition(self, condition, modify_description=False): if modify_description: self._description.addPreCondition(deepcopy(condition)) self._pre_conditions.append(condition) for r1, r2 in self._remaps.iteritems(): self._pre_conditions[-1].remap(r1, r2) def addHoldCondition(self, condition, modify_description=False): if modify_description: self._description.addHoldCondition(deepcopy(condition)) self._hold_conditions.append(condition) for r1, r2 in self._remaps.iteritems(): self._hold_conditions[-1].remap(r1, r2) def addPostCondition(self, condition, modify_description=False): if modify_description: self._description.addPostCondition(deepcopy(condition)) self._post_conditions.append(condition) for r1, r2 in self._remaps.iteritems(): self._post_conditions[-1].remap(r1, r2) def processChildren(self, visitor): """ TODO: this function has to be removed and embedded into the tick """ self._setState( self._children_processor.processChildren(self._children, visitor)) return self._state def setProcessor(self, processor_type): """ Set the children processor """ self._children_processor = processor_type #--------Virtual functions-------- def hasInstance(self): """ Wrapper skills override this """ return True def execute(self): """ Optional """ self._setProgress("End", 1) return None def wrapper_expand(self): pass
class DebugVariableViewer(object): """ Class that implements a generic viewer that display a list of variable values This class has to be inherited to effectively display variable values """ def __init__(self, window, items=None): """ Constructor @param window: Reference to the Debug Variable Panel @param items: List of DebugVariableItem displayed by Viewer """ self.ParentWindow = window items = [] if items is None else items self.ItemsDict = OrderedDict([(item.GetVariable(), item) for item in items]) self.Items = self.ItemsDict.viewvalues() # Variable storing current highlight displayed in Viewer self.Highlight = HIGHLIGHT_NONE # List of buttons self.Buttons = [] self.InitHighlightPensBrushes() def __del__(self): """ Destructor """ # Remove reference to Debug Variable Panel self.ParentWindow = None def InitHighlightPensBrushes(self): """ Init global pens and brushes """ if not HIGHLIGHT: HIGHLIGHT['DROP_PEN'] = wx.Pen(wx.Colour(0, 128, 255)) HIGHLIGHT['DROP_BRUSH'] = wx.Brush(wx.Colour(0, 128, 255, 128)) HIGHLIGHT['RESIZE_PEN'] = wx.Pen(wx.Colour(200, 200, 200)) HIGHLIGHT['RESIZE_BRUSH'] = wx.Brush(wx.Colour(200, 200, 200)) def GetIndex(self): """ Return position of Viewer in Debug Variable Panel @return: Position of Viewer """ return self.ParentWindow.GetViewerIndex(self) def GetItem(self, variable): """ Return item storing values of a variable @param variable: Variable path @return: Item storing values of this variable """ return self.ItemsDict.get(variable, None) def GetItems(self): """ Return items displayed by Viewer @return: List of items displayed in Viewer """ return self.ItemsDict.values() def AddItem(self, item): """ Add an item to the list of items displayed by Viewer @param item: Item to add to the list """ self.ItemsDict[item.GetVariable()] = item def RemoveItem(self, item): """ Remove an item from the list of items displayed by Viewer @param item: Item to remove from the list """ self.ItemsDict.pop(item.GetVariable(), None) def ClearItems(self): """ Clear list of items displayed by Viewer """ # Unsubscribe every items of the list for item in self.Items: self.ParentWindow.RemoveDataConsumer(item) # Clear list self.ItemsDict.clear() def ItemsIsEmpty(self): """ Return if list of items displayed by Viewer is empty @return: True if list is empty """ return len(self.Items) == 0 def SubscribeAllDataConsumers(self): """ Function that unsubscribe and remove every item that store values of a variable that doesn't exist in PLC anymore """ for item in self.ItemsDict.values()[:]: iec_path = item.GetVariable() # Check that variablepath exist in PLC if self.ParentWindow.GetDataType(iec_path) is None: # If not, unsubscribe and remove it self.ParentWindow.RemoveDataConsumer(item) self.RemoveItem(item) else: # If it exist, resubscribe and refresh data type self.ParentWindow.AddDataConsumer(iec_path.upper(), item, True) item.RefreshVariableType() def ResetItemsData(self): """ Reset data stored in every items displayed in Viewer """ for item in self.Items: item.ResetData() def GetItemsMinCommonTick(self): """ Return the minimum tick common to all iems displayed in Viewer @return: Minimum common tick between items """ return reduce(max, [ item.GetData()[0, 0] for item in self.Items if len(item.GetData()) > 0 ], 0) def RefreshViewer(self): """ Method that refresh the content displayed by Viewer Need to be overridden by inherited classes """ pass def SetHighlight(self, highlight): """ Set Highlight type displayed in Viewer @return: True if highlight has changed """ # Return immediately if highlight don't change if self.Highlight == highlight: return False self.Highlight = highlight return True def GetButtons(self): """ Return list of buttons defined in Viewer @return: List of buttons """ return self.Buttons def IsOverButton(self, x, y): """ Return if point is over one button of Viewer @param x: X coordinate of point @param y: Y coordinate of point @return: button where point is over """ for button in self.GetButtons(): if button.HitTest(x, y): return button return None def HandleButton(self, x, y): """ Search for the button under point and if found execute associated callback @param x: X coordinate of point @param y: Y coordinate of point @return: True if a button was found and callback executed """ button = self.IsOverButton(x, y) if button is None: return False button.ProcessCallback() return True def ShowButtons(self, show): """ Set display state of buttons in Viewer @param show: Display state (True if buttons must be displayed) """ # Change display of every buttons for button in self.Buttons: button.Show(show) # Refresh button positions self.RefreshButtonsPosition() self.RefreshViewer() def RefreshButtonsPosition(self): """ Function that refresh buttons position in Viewer """ # Get Viewer size width, _height = self.GetSize() # Buttons are align right so we calculate buttons positions in # reverse order buttons = self.Buttons[:] buttons.reverse() # Position offset on x coordinate x_offset = 0 for button in buttons: # Buttons are stacked right, removing those that are not active if button.IsEnabled(): # Update button position according to button width and offset # on x coordinate w, _h = button.GetSize() button.SetPosition(width - 5 - w - x_offset, 5) # Update offset on x coordinate x_offset += w + 2 def DrawCommonElements(self, dc, buttons=None): """ Function that draw common graphics for every Viewers @param dc: wx.DC object corresponding to Device context where drawing common graphics @param buttons: List of buttons to draw if different from default (default None) """ # Get Viewer size width, height = self.GetSize() # Set dc styling for drop before or drop after highlight dc.SetPen(HIGHLIGHT['DROP_PEN']) dc.SetBrush(HIGHLIGHT['DROP_BRUSH']) # Draw line at upper side of Viewer if highlight is drop before if self.Highlight == HIGHLIGHT_BEFORE: dc.DrawLine(0, 1, width - 1, 1) # Draw line at lower side of Viewer if highlight is drop before elif self.Highlight == HIGHLIGHT_AFTER: dc.DrawLine(0, height - 1, width - 1, height - 1) # If no specific buttons are defined, get default buttons if buttons is None: buttons = self.Buttons # Draw buttons for button in buttons: button.Draw(dc) # If graph dragging is processing if self.ParentWindow.IsDragging(): destBBox = self.ParentWindow.GetDraggingAxesClippingRegion(self) srcPos = self.ParentWindow.GetDraggingAxesPosition(self) if destBBox.width > 0 and destBBox.height > 0: srcPanel = self.ParentWindow.DraggingAxesPanel srcBBox = srcPanel.GetAxesBoundingBox() srcX = srcBBox.x - (srcPos.x if destBBox.x == 0 else 0) srcY = srcBBox.y - (srcPos.y if destBBox.y == 0 else 0) srcBmp = _convert_agg_to_wx_bitmap(srcPanel.get_renderer(), None) srcDC = wx.MemoryDC() srcDC.SelectObject(srcBmp) dc.Blit(destBBox.x, destBBox.y, int(destBBox.width), int(destBBox.height), srcDC, srcX, srcY) def OnEnter(self, event): """ Function called when entering Viewer @param event: wx.MouseEvent """ # Display buttons self.ShowButtons(True) event.Skip() def OnLeave(self, event): """ Function called when leaving Viewer @param event: wx.MouseEvent """ # Hide buttons self.ShowButtons(False) event.Skip() def OnCloseButton(self): """ Function called when Close button is pressed """ wx.CallAfter(self.ParentWindow.DeleteValue, self) def OnForceButton(self): """ Function called when Force button is pressed """ self.ForceValue(self.ItemsDict.values()[0]) def OnReleaseButton(self): """ Function called when Release button is pressed """ self.ReleaseValue(self.ItemsDict.values()[0]) def OnMouseDragging(self, x, y): """ Function called when mouse is dragged over Viewer @param x: X coordinate of mouse pointer @param y: Y coordinate of mouse pointer """ xw, yw = self.GetPosition() # Refresh highlight in Debug Variable Panel (highlight can be displayed # in another Viewer self.ParentWindow.RefreshHighlight(x + xw, y + yw) def RefreshHighlight(self, x, y): """ Function called by Debug Variable Panel asking Viewer to refresh highlight according to mouse position @param x: X coordinate of mouse pointer @param y: Y coordinate of mouse pointer """ # Get Viewer size _width, height = self.GetSize() # Mouse is in the first half of Viewer if y < height // 2: # If Viewer is the upper one, draw drop before highlight if self.ParentWindow.IsViewerFirst(self): self.SetHighlight(HIGHLIGHT_BEFORE) # Else draw drop after highlight in previous Viewer else: self.SetHighlight(HIGHLIGHT_NONE) self.ParentWindow.HighlightPreviousViewer(self) # Mouse is in the second half of Viewer, draw drop after highlight else: self.SetHighlight(HIGHLIGHT_AFTER) def OnEraseBackground(self, event): """ Function called when Viewer background is going to be erase @param event: wx.EraseEvent """ # Prevent flicker on Windows pass def OnResize(self, event): """ Function called when Viewer size changed @param event: wx.ResizeEvent """ # Refresh button positions self.RefreshButtonsPosition() self.ParentWindow.ForceRefresh() event.Skip() def ForceValue(self, item): """ Force value of item given @param item: Item to force value """ # Check variable data type iec_path = item.GetVariable() iec_type = self.ParentWindow.GetDataType(iec_path) # Return immediately if not found if iec_type is None: return # Open a dialog to enter varaible forced value dialog = ForceVariableDialog(self, iec_type, str(item.GetValue())) if dialog.ShowModal() == wx.ID_OK: self.ParentWindow.ForceDataValue(iec_path.upper(), dialog.GetValue()) def ReleaseValue(self, item): """ Release value of item given @param item: Item to release value """ self.ParentWindow.ReleaseDataValue(item.GetVariable().upper())