Exemplo n.º 1
0
    def _generate_image_metadata(self):
        # First - read old image config, we'll update it instead of
        # generating one from scratch
        metadata = OrderedDict(self.old_image_config)
        # Update image creation date
        metadata['created'] = self.date

        # Remove unnecessary or old fields
        metadata.pop("container", None)

        # Remove squashed layers from history
        metadata['history'] = metadata['history'][:len(self.layers_to_move)]
        # Remove diff_ids for squashed layers
        metadata['rootfs']['diff_ids'] = metadata['rootfs'][
            'diff_ids'][:len(self.layer_paths_to_move)]

        history = {'comment': '', 'created': self.date}

        if self.layer_paths_to_squash:
            # Add diff_ids for the squashed layer
            metadata['rootfs']['diff_ids'].append(
                "sha256:%s" % self.diff_ids[-1])
        else:
            history['empty_layer'] = True

        # Add new entry for squashed layer to history
        metadata['history'].append(history)

        if self.squash_id:
            # Update image id, should be one layer below squashed layer
            metadata['config']['Image'] = self.squash_id
        else:
            metadata['config']['Image'] = ""

        return metadata
Exemplo n.º 2
0
class LRUCache(object):

    def __init__(self, capacity):
        """
        :type capacity: int
        """
        self.dictionary = OrderedDict()
        self.capacity = capacity

    def get(self, key):
        """
        :rtype: int
        """
        if key in self.dictionary:
            temp = self.dictionary.pop(key)
            self.dictionary[key] = temp
            return temp
        else: return -1

    def set(self, key, value):
        """
        :type key: int
        :type value: int
        :rtype: nothing
        """
        if key in self.dictionary:
            self.dictionary.pop(key)()
            self.dictionary[key] = value
        else:
            if len(self.dictionary) >= self.capacity:
                self.dictionary.popitem(last=False)
                self.dictionary[key] = value
            else:
                self.dictionary[key] = value
Exemplo n.º 3
0
class Cache(object):

    def __init__(self):
        self.reset_caches()

    def reset_caches(self):
        self._category_cache = OrderedDict()
        self._search_cache = OrderedDict()

    def search_cache(self, search):
        old = self._search_cache.pop(search, None)
        if old is None or old[0] <= self.db.last_modified():
            matches = self.search_for_books(search) or []
            self._search_cache[search] = (utcnow(), frozenset(matches))
            if len(self._search_cache) > 50:
                self._search_cache.popitem(last=False)
        else:
            self._search_cache[search] = old
        return self._search_cache[search][1]

    def categories_cache(self, restrict_to=frozenset([])):
        base_restriction = self.search_cache('')
        if restrict_to:
            restrict_to = frozenset(restrict_to).intersection(base_restriction)
        else:
            restrict_to = base_restriction
        old = self._category_cache.pop(frozenset(restrict_to), None)
        if old is None or old[0] <= self.db.last_modified():
            categories = self.db.get_categories(ids=restrict_to)
            self._category_cache[restrict_to] = (utcnow(), categories)
            if len(self._category_cache) > 20:
                self._category_cache.popitem(last=False)
        else:
            self._category_cache[frozenset(restrict_to)] = old
        return self._category_cache[restrict_to][1]
Exemplo n.º 4
0
    def trace(self, key, readby=False, strict=False):
        """
        Return the sequence of operations required to compute the node ``key``.
        If ``readby = True``, then return the sequence of operations that will
        depend on ``key``, instead. With ``strict = True``, drop ``key`` from the
        result.
        """
        if key not in self:
            return []

        # OrderedDicts, besides preserving the scheduling order, also prevent
        # scheduling the same node more than once
        found = OrderedDict()
        queue = OrderedDict([(key, self[key])])
        while queue:
            k, v = queue.popitem(last=False)
            reads = self.extract(k, readby=readby)
            if set(reads).issubset(set(found.values())):
                # All dependencies satisfied, schedulable
                found[k] = v
            else:
                # Tensors belong to other traces, so they can be scheduled straight away
                tensors = [i for i in reads if i.is_Tensor]
                found = OrderedDict(list(found.items()) + [(i.lhs, i) for i in tensors])
                # Postpone the rest until all dependening nodes got scheduled
                scalars = [i for i in reads if i.is_Scalar]
                queue = OrderedDict([(i.lhs, i) for i in scalars] +
                                    [(k, v)] + list(queue.items()))
        if strict is True:
            found.pop(key)
        return tuple(found.values())
Exemplo n.º 5
0
def find_unique_url_multi_dict(file):
    current_dict = OrderedDict()
    dicts = [current_dict]

    with open(file) as f:
        line_no = 0
        for url in f:
            line_no += 1
            if len(current_dict) > 100:
                current_dict = OrderedDict()
                dicts.append(current_dict)
            if url in current_dict:
                current_dict.pop(url)
            else:
                current_dict[url] = line_no

    while dicts:
        d = dicts.pop(0)
        for k, v in d.iteritems():
            not_found = True
            for d1 in dicts:
                if k not in d1:
                    continue
                else:
                    not_found = False
                    break
            if not_found:
                return k, v
def smallest_subarray_covering_set(A, Q):
    """Find smallest range (i,j) s.t. all q in Q are in A[i..j]."""

    # handle 0-length covering
    if not Q:
        return 0, -1

    # start with no best and an empty candidate covering range
    Q = set(Q)  # want O(1) membership test
    min_size = None
    min_covering_range = None
    cand_locs = OrderedDict()

    # stream elements of A, maintaining the shortest covering range
    # that ends at the current element
    for j, elem in enumerate(A):

        # skip elements that can't contribute to a covering
        if elem not in Q:
            continue

        # extend the candidate range with the current elem,
        # removing any previous instance of the same elem
        cand_locs.pop(elem, None)
        cand_locs[elem] = j  # will be added in final position

        # if the new candidate is legal and better, make it the current best
        if len(cand_locs) == len(Q):
            i = cand_locs.itervalues().next()  # get front position, O(1) time
            if min_size is None or j - i < min_size:
                min_size = j - i
                min_covering_range = i, j

    return min_covering_range
Exemplo n.º 7
0
    def __new__(mcs, name, bases, attrs):
        # Collect fields from current class.
        current_fields = []
        for key, value in list(attrs.items()):
            if isinstance(value, Field):
                current_fields.append((key, value))
                attrs.pop(key)
        current_fields.sort(key=lambda x: x[1].creation_counter)
        attrs["declared_fields"] = OrderedDict(current_fields)

        new_class = super(DeclarativeFieldsMetaclass, mcs).__new__(mcs, name, bases, attrs)

        # Walk through the MRO.
        declared_fields = OrderedDict()
        for base in reversed(new_class.__mro__):
            # Collect fields from base class.
            if hasattr(base, "declared_fields"):
                declared_fields.update(base.declared_fields)

            # Field shadowing.
            for attr in base.__dict__.keys():
                if attr in declared_fields:
                    declared_fields.pop(attr)

        new_class.base_fields = declared_fields
        new_class.declared_fields = declared_fields

        return new_class
Exemplo n.º 8
0
  def _topological_sort(self, goal_info_by_goal):
    dependees_by_goal = OrderedDict()

    def add_dependee(goal, dependee=None):
      dependees = dependees_by_goal.get(goal)
      if dependees is None:
        dependees = set()
        dependees_by_goal[goal] = dependees
      if dependee:
        dependees.add(dependee)

    for goal, goal_info in goal_info_by_goal.items():
      add_dependee(goal)
      for dependency in goal_info.goal_dependencies:
        add_dependee(dependency, goal)

    satisfied = set()
    while dependees_by_goal:
      count = len(dependees_by_goal)
      for goal, dependees in dependees_by_goal.items():
        unsatisfied = len(dependees - satisfied)
        if unsatisfied == 0:
          satisfied.add(goal)
          dependees_by_goal.pop(goal)
          yield goal_info_by_goal[goal]
          break
      if len(dependees_by_goal) == count:
        for dependees in dependees_by_goal.values():
          dependees.difference_update(satisfied)
        # TODO(John Sirois): Do a better job here and actually collect and print cycle paths
        # between Goals/Tasks.  The developer can most directly address that data.
        raise self.GoalCycleError('Cycle detected in goal dependencies:\n\t{0}'
                                  .format('\n\t'.join('{0} <- {1}'.format(goal, list(dependees))
                                                      for goal, dependees
                                                      in dependees_by_goal.items())))
Exemplo n.º 9
0
class Lineup(object):
    def __init__(self, lineup):
        self.lineup = lineup
        self.healthy_pokemon = OrderedDict()
        self.fainted_pokemon = OrderedDict()

        for pokemon in lineup:
            if pokemon.health > 0:
                self.healthy_pokemon[pokemon] = True
            else:
                self.fainted_pokemon[pokemon] = True

    def fainted(self, pokemon):
        if pokemon in self.healthy_pokemon:
            self.healthy_pokemon.pop(pokemon)

        self.fainted_pokemon[pokemon] = True

    def pokemon(self):
        return self.lineup

    def get_starters(self, size):
        starters = set()
        for pokemon in self.healthy_pokemon.keys():
            starters.add(pokemon)
            if len(starters) == size:
                break
        return starters
Exemplo n.º 10
0
class CompoundIndex:

    def __init__(self, field=[]):
        self.insidedoc = OrderedDict()

        if field:
            if isinstance(field, list):
                self.addListIndex(field)
            else:
                if field.startswith('_'):
                    self.addDescIndex(field)
                else:
                    self.addIndex(field)

    def addDescIndex(self, field):
        self.insidedoc.__setitem__(field, -1)

    def addIndex(self, field):
        self.insidedoc.__setitem__(field, 1)

    def addListIndex(self, list):
         for x in list:
            if x.startswith('_'):
                self.insidedoc.__setitem__(x.split('_', 1)[1], -1)
            else:
                self.insidedoc.__setitem__(x, 1)

    def removeIndex(self, field):
        self.insidedoc.pop(field)

    def __str__(self):
        return json.dumps(self.insidedoc)
Exemplo n.º 11
0
    def __new__(mcs, name, bases, attrs):
        # Collect transitions from current class.
        current_transitions = []
        for key, value in list(attrs.items()):
            if isinstance(value, Transition):
                current_transitions.append((key, value))
                attrs.pop(key)
        current_transitions.sort(key=lambda x: x[1].creation_counter)
        attrs['declared_transitions'] = OrderedDict(current_transitions)

        new_class = (super(DeclarativeTransitionsMetaclass, mcs).__new__(mcs, name, bases, attrs))

        # Walk through the MRO.
        declared_transitions = OrderedDict()
        for base in reversed(new_class.__mro__):
            # Collect transitions from base class.
            if hasattr(base, 'declared_transitions'):
                declared_transitions.update(base.declared_transitions)

            # Field shadowing.
            for attr, value in base.__dict__.items():
                if value is None and attr in declared_transitions:
                    declared_transitions.pop(attr)

        new_class.declared_transitions = declared_transitions

        return new_class
Exemplo n.º 12
0
def run(state, include=None, exclude=None):
    """
    Run data validations for state.

    State is required. Optionally filter validations using include/exclude flags.
    """
    if include and exclude:
        sys.exit("ERROR: You can not use both include and exclude flags!")

    state_mod = load_module(state, ['validate'])
    # Load all validations in order found
    validations = OrderedDict()
    for name in dir(state_mod.validate):
        if name.startswith('validate_'):
            func = getattr(state_mod.validate, name)
            validations[name] = func

    # Filter validations based in include/exclude flags
    if include:
        to_run = split_args(include)
        for val in validations:
            if val not in to_run:
                validations.pop(val)
    if exclude:
        to_skip = split_args(exclude)
        for val in validations:
            if val in to_skip:
                validations.pop(val)

    # Run remaining validations
    run_validation(state, list(validations.values())) 
Exemplo n.º 13
0
    def parse_strand_chart_data (self):
        """ Make a data structure suitable for HighCharts for the strand alignment plot """
        self.bismark_strand_samples = list()
        self.bismark_directional_mode = 0
        series = OrderedDict()
        series['OB'] = list()
        series['CTOB'] = list()
        series['CTOT'] = list()
        series['OT'] = list()
        for sn in sorted(self.bismark_data.keys()):
            self.bismark_strand_samples.append(sn)
            series['OB'].append(int(self.bismark_data[sn].get('aln_strand_ob', 0)))
            series['CTOB'].append(int(self.bismark_data[sn].get('aln_strand_ctob', 0)))
            series['CTOT'].append(int(self.bismark_data[sn].get('aln_strand_ctot', 0)))
            series['OT'].append(int(self.bismark_data[sn].get('aln_strand_ot', 0)))
            if 'aln_strand_directional' in self.bismark_data[sn]:
                self.bismark_directional_mode += 1

        if self.bismark_directional_mode == len(self.bismark_strand_samples):
            series.pop('CTOB', None)
            series.pop('CTOT', None)

        self.bismark_strand_plot_series = list()
        for cat in series:
            self.bismark_strand_plot_series.append({
                'name': cat,
                'data': series[cat]
            })
Exemplo n.º 14
0
class MissingPonies(object):
    def __init__(self, owned_ponies, inventory_ponies, all_ponies):
        owned_ponies_id = set(owned_ponies.keys()) | set(inventory_ponies.keys())
        all_ponies_id = set(all_ponies.keys())
        not_owned = all_ponies_id - owned_ponies_id
        self._ponies = OrderedDict([(ID, name)
                                    for ID, name in all_ponies.items()
                                    if ID in not_owned])

    def items(self):
        return self._ponies.items()

    def values(self):
        return self._ponies.values()

    def keys(self):
        return self._ponies.keys()

    def __contains__(self, item):
        return item in self._ponies

    def __isub__(self, pony_id):
        self._ponies.pop(pony_id, None)
        return self

    def remove(self, pony_id):
        self -= pony_id
Exemplo n.º 15
0
	def parse_recipe(self, response, title, picture):
		hxs = HtmlXPathSelector(response)

		section = None
		sections = OrderedDict()

		for node in hxs.select(xp_ingredients):
			text = html_to_text(node.extract()).strip()

			if not text:
				continue

			if node.select('strong'):
				section = text
				continue

			sections.setdefault(section, []).append(text)

		ingredients = sections.pop(None, None) or sections.pop(sections.keys()[-1])
		extra_ingredients = [x for y in sections.values() for x in y]

		yield CocktailItem(
			title=title,
			picture=picture,
			url=response.url,
			ingredients=ingredients,
			extra_ingredients=extra_ingredients
		)
Exemplo n.º 16
0
 def bismark_strand_chart (self):
     """ Make the strand alignment plot """
     
     # Specify the order of the different possible categories
     keys = OrderedDict()
     keys['strand_ob']   = { 'name': 'Original bottom strand' }
     keys['strand_ctob'] = { 'name': 'Complementary to original bottom strand' }
     keys['strand_ctot'] = { 'name': 'Complementary to original top strand' }
     keys['strand_ot']   = { 'name': 'Original top strand' }
     
     # See if we have any directional samples
     directional = 0
     d_mode = ''
     for sn in self.bismark_data['alignment'].values():
         if 'strand_directional' in sn.keys():
             directional += 1
     if directional == len(self.bismark_data['alignment']):
         keys.pop('strand_ctob', None)
         keys.pop('strand_ctot', None)
         d_mode = '<p>All samples were run with <code>--directional</code> mode; alignments to complementary strands (CTOT, CTOB) were ignored.</p>'
     elif directional > 0:
         d_mode = '<p>{} samples were run with <code>--directional</code> mode; alignments to complementary strands (CTOT, CTOB) were ignored.</p>'.format(directional)
     
     # Config for the plot
     config = {
         'id': 'bismark_strand_alignment',
         'title': 'Alignment to Individual Bisulfite Strands',
         'ylab': '% Reads',
         'cpswitch_c_active': False,
         'cpswitch_counts_label': 'Number of Reads'
     }
     
     return d_mode + plots.bargraph.plot(self.bismark_data['alignment'], keys, config)
Exemplo n.º 17
0
    def get_rack_units(self, face=RACK_FACE_FRONT, exclude=None, remove_redundant=False):
        """
        Return a list of rack units as dictionaries. Example: {'device': None, 'face': 0, 'id': 48, 'name': 'U48'}
        Each key 'device' is either a Device or None. By default, multi-U devices are repeated for each U they occupy.

        :param face: Rack face (front or rear)
        :param exclude: PK of a Device to exclude (optional); helpful when relocating a Device within a Rack
        :param remove_redundant: If True, rack units occupied by a device already listed will be omitted
        """

        elevation = OrderedDict()
        for u in reversed(range(1, self.u_height + 1)):
            elevation[u] = {'id': u, 'name': 'U{}'.format(u), 'face': face, 'device': None}

        # Add devices to rack units list
        if self.pk:
            for device in Device.objects.select_related('device_type__manufacturer', 'device_role')\
                    .annotate(devicebay_count=Count('device_bays'))\
                    .exclude(pk=exclude)\
                    .filter(rack=self, position__gt=0)\
                    .filter(Q(face=face) | Q(device_type__is_full_depth=True)):
                if remove_redundant:
                    elevation[device.position]['device'] = device
                    for u in range(device.position + 1, device.position + device.device_type.u_height):
                        elevation.pop(u, None)
                else:
                    for u in range(device.position, device.position + device.device_type.u_height):
                        elevation[u]['device'] = device

        return [u for u in elevation.values()]
Exemplo n.º 18
0
 def get_round_results(self, competition, round, valid_session_rounds, scores):
     results = OrderedDict()
     for score in scores:
         session_entry = score.target.session_entry
         if session_entry.session_round.shot_round.id is not round.id:
             continue
         if session_entry.session_round not in valid_session_rounds:
             continue
         categories = self.get_categories_for_entry(competition, session_entry.competition_entry)
         for category in categories:
             if category not in results:
                 results[category] = {}
             if session_entry.competition_entry not in results[category]:
                 results[category][session_entry.competition_entry] = []
             results[category][session_entry.competition_entry].append(score)
     for category, scores in results.items():
         scores = OrderedDict((entry, rounds) for entry, rounds in scores.items() if len(rounds) >= 2)
         if not scores:
             results.pop(category)
             continue
         for entry in scores:
             scores[entry] = sorted(scores[entry], key=lambda s: s.target.session_entry.session_round.session.start)[:2]
         new_scores = [ScoreMock(
             disqualified=any(s.disqualified for s in sub_scores),
             retired=any(s.disqualified for s in sub_scores),
             target=sub_scores[0].target,
             score=sum(s.score for s in sub_scores),
             hits=sum(s.hits for s in sub_scores),
             golds=sum(s.golds for s in sub_scores),
             xs=sum(s.xs for s in sub_scores),
         ) for entry, sub_scores in scores.items()]
         if not self.leaderboard:
             new_scores = filter(lambda s: s.score > 0, new_scores)
         results[category] = self.sort_results(new_scores)
     return results
Exemplo n.º 19
0
class LRUCache(object):

    def __init__(self, capacity):
        """
        :type capacity: int
        """
        self.c = capacity
        self.mapi = OrderedDict()

    def get(self, key):
        """
        :rtype: int
        """
        if key in self.mapi:
            ret = self.mapi.pop(key)
            self.mapi[key] = ret
            return ret
        return -1

    def set(self, key, value):
        """
        :type key: int
        :type value: int
        :rtype: nothing
        """
        if key in self.mapi:
            self.mapi.pop(key)
        self.mapi[key] = value
        if len(self.mapi) > self.c:
            self.mapi.popitem(last=False)
Exemplo n.º 20
0
def iet_make(stree):
    """Create an IET from a ScheduleTree."""
    nsections = 0
    queues = OrderedDict()
    for i in stree.visit():
        if i == stree:
            # We hit this handle at the very end of the visit
            return List(body=queues.pop(i))

        elif i.is_Exprs:
            exprs = [Increment(e) if e.is_Increment else Expression(e) for e in i.exprs]
            body = ExpressionBundle(i.shape, i.ops, i.traffic, body=exprs)

        elif i.is_Conditional:
            body = Conditional(i.guard, queues.pop(i))

        elif i.is_Iteration:
            # Order to ensure deterministic code generation
            uindices = sorted(i.sub_iterators, key=lambda d: d.name)
            # Generate Iteration
            body = Iteration(queues.pop(i), i.dim, i.dim._limits, offsets=i.limits,
                             direction=i.direction, uindices=uindices)

        elif i.is_Section:
            body = Section('section%d' % nsections, body=queues.pop(i))
            nsections += 1

        elif i.is_Halo:
            body = HaloSpot(i.halo_scheme, body=queues.pop(i))

        queues.setdefault(i.parent, []).append(body)

    assert False
Exemplo n.º 21
0
def convert_block(data):
    block_id = data.findtext('key')

    params = OrderedDict(sorted(
        (param.findtext('key'), param.findtext('value'))
        for param in data.findall('param')
    ))
    if block_id == "import":
        params["imports"] = params.pop("import")
    states = OrderedDict()
    x, y = ast.literal_eval(params.pop('_coordinate', '(10, 10)'))
    states['coordinate'] = yaml.ListFlowing([x, y])
    states['rotation'] = int(params.pop('_rotation', '0'))
    enabled = params.pop('_enabled', 'True')
    states['state'] = (
        'enabled' if enabled in ('1', 'True') else
        'bypassed' if enabled == '2' else
        'disabled'
    )

    block = OrderedDict()
    if block_id != 'options':
        block['name'] = params.pop('id')
    block['id'] = block_id
    block['parameters'] = params
    block['states'] = states

    return block
Exemplo n.º 22
0
    def longestConsecutiveSeq(self, num):
        seqDic = OrderedDict()

        for each in num:
            seqDic[each] = each

        maxlength = 0

        for key in num:
            left = key - 1
            right = key + 1
            cnt = 1

            while left in seqDic:
                cnt += 1
                seqDic.pop(left)
                left -= 1

            while right in seqDic:
                cnt += 1
                seqDic.pop(right)
                right += 1

            if cnt > maxlength:
                maxlength = cnt

        return maxlength
Exemplo n.º 23
0
class LRUCache(OrderedDict):
    """
    #实现一个LRU算法的dict,用于记录order_id
    #避免storm平台重发log记录,造成redis数据偏大!
    """
    def __init__(self, capacity):
        """
        @param capacity 字典长度
        """
        self.capacity = capacity
        self.cache = OrderedDict()
    
    def get(self,key):
        if self.cache.has_key(key):
            value = self.cache.pop(key)
            self.cache[key] = value
        else:
            value = None
        return value
    
    def set(self,key,value):
        if self.cache.has_key(key):
            value = self.cache.pop(key)
            self.cache[key] = value
        else:
            if len(self.cache) == self.capacity:
                 #pop出第一个item
                self.cache.popitem(last = False)
                self.cache[key] = value
            else:
                self.cache[key] = value
Exemplo n.º 24
0
    def __new__(mcs, name, bases, attrs):
        # Collect fields from current class.
        current_fields = []
        for key, value in attrs.items():
            if isinstance(value, BaseField):
                current_fields.append((key, value))
        current_fields.sort(key=lambda x: x[1].creation_counter)
        attrs['declared_fields'] = OrderedDict(current_fields)

        new_class = (super(MetaField, mcs)
            .__new__(mcs, name, bases, attrs))

        # Walk through the MRO.
        declared_fields = OrderedDict()
        for base in reversed(new_class.__mro__):
            # Collect fields from base class.
            if hasattr(base, 'declared_fields'):
                declared_fields.update(base.declared_fields)

            # Field shadowing.
            for attr, value in base.__dict__.items():
                if value is None and attr in declared_fields:
                    declared_fields.pop(attr)

        # Store these fields on the class.
        new_class.base_fields = declared_fields
        new_class.declared_fields = declared_fields

        return new_class
Exemplo n.º 25
0
class LimitedMemoryBackend(BasicInterface, dc.api.CacheBackend):

    def __init__(self, argument_dict):
        '''If argument_dict contains a value for max_kbytes this the total memory limit in kByte that is enforced on the
        internal cache dictionary, otherwise it's set to sys.maxint.
        If argument_dict contains a value for max_keys this maximum amount of cache values kept in the
        internal cache dictionary, otherwise it's set to sys.maxlen.
        If necessary values are deleted from the cache in FIFO order.
        '''
        self.logger.debug('LimitedMemoryBackend args {}'.format(pformat(argument_dict)))
        self._max_keys = argument_dict.get('max_keys', sys.maxsize)
        self._max_bytes = argument_dict.get('max_kbytes', sys.maxint / 1024) * 1024
        self._cache = OrderedDict()

    def get(self, key):
        return self._cache.get(key, dc.api.NO_VALUE)

    def print_limit(self, additional_size=0):
        self.logger.info('LimitedMemoryBackend at {}({}) keys -- {}({}) Byte'
                         .format(len(self._cache), self._max_keys,
                                 memory.getsizeof(self._cache) / 8, self._max_bytes))

    def _enforce_limits(self, new_value):
        additional_size = memory.getsizeof(new_value) / 8
        while len(self._cache) > 0 and not (len(self._cache) <= self._max_keys and
                                            (memory.getsizeof(self._cache) + additional_size) / 8 <= self._max_bytes):
            self.logger.debug('shrinking limited memory cache')
            self._cache.popitem(last=False)

    def set(self, key, value):
        self._enforce_limits(value)
        self._cache[key] = value

    def delete(self, key):
        self._cache.pop(key)
Exemplo n.º 26
0
    def to_representation(self, instance):
        """
        Serialize objects -> primitives.
        """
        ret = OrderedDict()
        fields = [field for field in self.fields.values() if not field.write_only]

        # geo structure
        if self.Meta.id_field is not False:
            ret["id"] = ""
        ret["type"] = "Feature"
        ret["geometry"] = {}
        ret["properties"] = OrderedDict()

        for field in fields:
            field_name = field.field_name
            if field.read_only and instance is None:
                continue
            value = field.to_representation(field.get_attribute(instance))
            if self.Meta.id_field is not False and field_name == self.Meta.id_field:
                ret["id"] = value
            elif field_name == self.Meta.geo_field:
                ret["geometry"] = value
            elif not getattr(field, 'write_only', False):
                ret["properties"][field_name] = value

        if self.Meta.id_field is False:
            ret.pop(self.Meta.model._meta.pk.name)

        return ret
Exemplo n.º 27
0
class WorkerQueue(object):
    def __init__(self):
        self.queue = OrderedDict()

    def ready(self, worker):
        self.queue.pop(worker.address, None)
        self.queue[worker.address] = worker

    def purge(self):
        """Look for & kill expired workers."""
        t = time.time()
        expired = []
        for address, worker in self.queue.items():
            if t > worker.expiry:  # Worker expired
                expired.append(address)
        for address in expired:
            self.queue.pop(address, None)

    def size(self):
        return len(self.queue.items())

    def next(self):
        if len(self.queue.items()) == 0:
            return None

        address, worker = self.queue.popitem(False)
        return address
def extract_properties_from_manifest(manifest_src):
    '''return a dictionary with the properties of a task read from a task manifest if it exists'''

    if not any(key.lower() in manifest_src.lower()
               for key in MANIFEST_ALIASES_GROUPED['manifest_version']):
        raise VersionError('Missing Task Manifest Version, must be v0')

    props = ordered_load(manifest_src)

    if not isinstance(props, dict):
        raise TypeError('Docstring should be a yaml dictionary. Got %s instead.' %
                        type(props).__name__)

    props = OrderedDict((MANIFEST_ALIASING.get(p.lower(), p.lower()), v) for p, v in props.items())

    if 'manifest_version' not in props:
        # note that it MUST be present because we check for it in the entire string just above
        # probably this is broken yaml
        raise KeyError('Missing Task Manifest Version, must be present')

    if props['manifest_version'] != 1:
        raise VersionError('Unknown Task Manifest Version %s' % props['manifest_version'])

    props = parse_args_from_manifest(props)

    props.pop('manifest_version')

    return props
Exemplo n.º 29
0
def _diff_trees(lhs, rhs, index, *path):
    def _fq(name):
        return "/".join(reversed((name,) + path))

    # Diff the trees (and keep deterministic order)
    rhs_tree_ids = OrderedDict((o.name, o.id) for o in rhs.tree_ids)
    for o in lhs.tree_ids:
        rhs_id = rhs_tree_ids.pop(o.name, None)  # remove so won't be picked up as added, below
        if rhs_id == o.id:  # no change
            continue
        elif rhs_id is None:  # removed
            yield (_fq(o.name), o.id, None)
            rhs_tree = Object(_id=None, tree_ids=[], blob_ids=[], other_ids=[])
        else:  # changed
            rhs_tree = index[rhs_id]
        for difference in _diff_trees(index[o.id], rhs_tree, index, o.name, *path):
            yield difference
    for name, id in rhs_tree_ids.items():  # added
        yield (_fq(name), None, id)
        lhs_tree = Object(_id=None, tree_ids=[], blob_ids=[], other_ids=[])
        for difference in _diff_trees(lhs_tree, index[id], index, name, *path):
            yield difference
    # Diff the blobs (and keep deterministic order)
    rhs_blob_ids = OrderedDict((o.name, o.id) for o in rhs.blob_ids)
    for o in lhs.blob_ids:
        rhs_id = rhs_blob_ids.pop(o.name, None)
        if rhs_id == o.id:
            continue  # no change
        elif rhs_id is None:
            yield (_fq(o.name), o.id, None)
        else:
            yield (_fq(o.name), o.id, rhs_id)
    for name, id in rhs_blob_ids.items():
        yield (_fq(name), None, id)
Exemplo n.º 30
0
class PointSpace(object):
    """Given a point will assign numeric IDs"""
    def __init__(self):
        self.points = OrderedDict()
        self.point_to_id = {}
        self._updated = True

    def _update(self):
        if self._updated:
            return
        current_id = 0
        for point in self.points:
            self.point_to_id[point] = current_id
            current_id += 1
        self._updated = True

    def add(self, point):
        self.points[point] = True
        self._updated = False

    def delete(self, point):
        self.points.pop(point, None)
        self._updated = False

    def get_id(self, point):
        self._update()
        return self.point_to_id.get(point)
Exemplo n.º 31
0
class Watchdog(BaseCacheClient):
    """Main device for running the watchdog service."""

    parameters = {
        'watch':
        Param('The configuration of things to watch',
              type=listof(dictof(str, anytype))),
        'mailreceiverkey':
        Param(
            'Cache key that updates the receivers for '
            'any mail notifiers we have configured',
            type=str),
        # these would be adevs, but adevs don't allow this flexible
        # configuration with dictionaries
        'notifiers':
        Param('Configures the notifiers for each warning type',
              type=dictof(str, listof(str))),
    }

    parameter_overrides = {
        'prefix': Override(mandatory=False, default='nicos/'),
    }

    def doInit(self, mode):
        BaseCacheClient.doInit(self, mode)
        # cache of all interesting keys with current values
        self._keydict = LCDict()
        # put status constants in key dict to simplify status conditions
        for stval, stname in status.statuses.items():
            self._keydict[stname.upper()] = stval
        # set to true during connect action
        self._process_updates = False
        # current setups
        self._setups = set()
        # mapping entry ids to entrys
        self._entries = {}
        # (mangled) key to update mail receivers
        self._mailreceiverkey = self.mailreceiverkey.replace('/', '_').lower()
        # mapping cache keys to entries that check this key
        self._keymap = {'session_mastersetup': set()}
        if self._mailreceiverkey:
            self._keymap[self._mailreceiverkey] = set()
        # current warnings: mapping entry ids to the string description
        self._warnings = OrderedDict()
        # current count loop pause reasons: mapping like self._warnings
        self._pausecount = OrderedDict()

        # create all notifier devices
        self._all_notifiers = []
        self._notifiers = {'': []}
        for key, devnames in iteritems(self.notifiers):
            self._notifiers[key] = notiflist = []
            for devname in devnames:
                dev = session.getDevice(devname, Notifier)
                notiflist.append(dev)
                self._all_notifiers.append(dev)

        # process entries in the default watchlist
        for entry_dict in self.watch:
            self._add_entry(entry_dict, 'watchdog')

        # start a thread checking for modification of the setup file
        createThread('refresh checker', self._checker)

    def _checker(self):
        setupname = session.explicit_setups[0]
        fn = session._setup_info[setupname]['filenames']
        watchFileContent(fn, self.log)
        self.log.info('setup file changed; restarting watchdog process')
        os.execv(sys.executable, [sys.executable] + sys.argv)

    def _add_entry(self, entryd, source):
        entryd = dict(entryd)  # convert back from readonlydict
        logprefix = 'entry %s from setup %r' % (entryd, source)

        # sanity-check the input
        if not entryd.get('condition'):
            self.log.warning('%s: missing "condition" key', logprefix)
            return
        if not entryd.get('message'):
            self.log.warning('%s: missing "message" key', logprefix)
            return
        if entryd.get('scriptaction') not in (None, 'pausecount', 'stop',
                                              'immediatestop'):
            self.log.warning(
                '%s: scriptaction is invalid, needs '
                "to be one of 'pausecount', 'stop' or "
                "'immediatestop'", logprefix)
            entryd.pop('scriptaction')

        entry = Entry(entryd)
        entry.from_setup = source

        if entry.id in self._entries:
            self.log.error('%s: duplicate entry, ignoring', logprefix)
            return

        if entry.type and entry.type not in self._notifiers:
            log_msg = (
                '%s: the condition type %r is not valid, must be '
                'one of %r' %
                (logprefix, entry.type, ', '.join(map(repr, self._notifiers))))
            if 'default' in self._notifiers:
                self.log.warning(log_msg + '; using default')
                entry.type = 'default'
            else:
                self.log.warning(log_msg + '; ignoring notifiers')
                entry.type = ''

        try:
            cond = Expression(self.log, entry.condition, entry.setup)
            if entry.gracetime:
                cond = DelayedTrigger(self.log, cond, entry.gracetime)
            if entry.precondition:
                precond = Expression(self.log, entry.precondition, entry.setup)
                if entry.precondtime:
                    precond = DelayedTrigger(self.log, precond,
                                             entry.precondtime)
                cond = Precondition(self.log, precond, cond)
            if not entry.enabled:
                cond.enabled = False
            entry.cond_obj = cond

        except Exception:
            self.log.error(
                '%s: could not construct condition, ignoring '
                'this condition', logprefix)
            return

        for key in cond.interesting_keys():
            self._keymap.setdefault(key, set()).add(entry)
        self._entries[entry.id] = entry

    def _remove_entry(self, eid):
        entry = self._entries.pop(eid, None)
        if entry:
            for key in entry.cond_obj.interesting_keys():
                self._keymap[key].discard(entry)

    # cache client API

    def _connect_action(self):
        # inhibit direct processing of updates
        self._process_updates = False
        try:
            BaseCacheClient._connect_action(self)
            # now process all keys we got
            time = currenttime()
            for key in list(self._keydict):
                try:
                    self._process_key(time, key, self._keydict[key])
                except Exception:
                    self.log.warning('error handling first update for key %s',
                                     key,
                                     exc=1)
        finally:
            self._process_updates = True
        self.storeSysInfo('watchdog')
        self._queue.put('watchdog/%s\n' % OP_SUBSCRIBE)
        self._publish_config()

    def _wait_data(self):
        t = currenttime()
        for entry in itervalues(self._entries):
            if entry.cond_obj.tick(t) or entry.cond_obj.is_expired(t):
                self._check_state(entry, t)

    def _handle_msg(self, time, ttlop, ttl, tsop, key, op, value):
        if key.startswith('watchdog/'):
            self._handle_control_msg(key, cache_load(value), time,
                                     op != OP_TELL)
            return

        key = key[len(self._prefix):].replace('/', '_').lower()

        # do we need the key for conditions?
        if key not in self._keymap:
            return
        expired = op == OP_TELLOLD or not value
        value = None if expired else cache_load(value)
        if not expired:
            self._keydict[key] = value
        else:
            self._keydict.pop(key, None)
        time = float(time)

        if self._process_updates:
            self._process_key(time, key, value)

    # internal watchdog API

    def _handle_control_msg(self, key, value, time, expired):
        if key == 'watchdog/enable':
            time = currenttime()
            for (eid, enabled) in value[1]:
                entry = self._entries.get(eid)
                if entry:
                    entry.cond_obj.enabled = enabled
                    entry.cond_obj.update(time, self._keydict)
                    self._check_state(entry, time)
            self.log.info('updated enabled conditions by user request')
        elif key == 'watchdog/reset':
            # reset all condition enables to their initial state
            # (e.g. due to NewExperiment)
            for entry in itervalues(self._entries):
                if entry.enabled != entry.cond_obj.enabled:
                    entry.cond_obj.enabled = entry.enabled
                    entry.cond_obj.update(time, self._keydict)
                    self._check_state(entry, time)
            self.log.info('enable status of all conditions reset')
        self._publish_config()

    def _publish_config(self):
        # publish current condition info in the cache
        self._put_message('configured', None,
                          [c.serialize() for c in itervalues(self._entries)])

    def _process_key(self, time, key, value):
        # check setups?
        if key == 'session_mastersetup' and value:
            self._setups_updated(time, set(value))
            return
        # update notification targets?
        if key == self._mailreceiverkey and value:
            self._update_mailreceivers(value)
            return
        for entry in self._keymap.get(key, ()):
            entry.cond_obj.update(time, self._keydict)
            self._check_state(entry, time)

    def _setups_updated(self, time, new_setups):
        prev_setups, self._setups = self._setups, new_setups
        # check if we need to remove some conditions
        for entry in listvalues(self._entries):
            if entry.from_setup != 'watchdog':
                if entry.from_setup not in self._setups:
                    self._remove_entry(entry.id)
        # check if we need to add some conditions
        session.readSetups()  # refresh setup info
        for new_setup in self._setups - prev_setups:
            info = session._setup_info.get(new_setup)
            if info and info['watch_conditions']:
                self.log.info('adding conditions from setup %s', new_setup)
                for entry_dict in info['watch_conditions']:
                    self._add_entry(entry_dict, new_setup)
        # trigger an update of all conditions
        for entry in itervalues(self._entries):
            entry.cond_obj.new_setups(self._setups)
            entry.cond_obj.update(time, self._keydict)
            self._check_state(entry, time)
        # update everyone els
        self._publish_config()
        self.log.info('new setups list: %s', ', '.join(self._setups))

    def _check_state(self, entry, time):
        """Check if the state of this entry changed and we need to
        emit a warning or a clear it.
        """
        eid = entry.id
        if entry.cond_obj.is_expired(time):
            if eid not in self._warnings or self._warnings[eid][0]:
                self._emit_expired_warning(entry)
        elif entry.cond_obj.triggered:
            if eid not in self._warnings or not self._warnings[eid][0]:
                self._emit_warning(entry)
        else:
            if eid in self._warnings:
                self._clear_warning(entry)
        self.log.debug('new conditions: %s', self._warnings)

    def _emit_warning(self, entry):
        """Emit a warning that this condition is now triggered."""
        self.log.info('got a new warning for %r', entry)
        warning_desc = strftime('%Y-%m-%d %H:%M') + ' -- ' + entry.message
        if entry.action:
            warning_desc += ' -- executing %r' % entry.action
        if entry.scriptaction == 'pausecount':
            self._pausecount[entry.id] = entry.message
            self._update_pausecount_str()
            warning_desc += ' -- counting paused'
        elif entry.scriptaction:
            self._put_message('scriptaction', entry,
                              (entry.scriptaction, entry.message))
        if entry.type:
            for notifier in self._notifiers[entry.type]:
                notifier.send('New warning from NICOS', warning_desc)
        self._put_message('warning', entry, entry.message)
        self._warnings[entry.id] = (True, warning_desc)
        self._update_warnings_str()
        if entry.action:
            self._put_message('action', entry, entry.action)
            self._spawn_action(entry.action)

    def _emit_expired_warning(self, entry):
        """Emit a warning that this condition has missing information."""
        self.log.info('value(s) missing for %r', entry)
        warning_desc = (strftime('%Y-%m-%d %H:%M') +
                        ' -- current value(s) missing, '
                        'cannot check condition %r' % entry.condition)
        # warn that we cannot check the condition anymore
        if entry.type:
            for notifier in self._notifiers[entry.type]:
                notifier.send('New warning from NICOS', warning_desc)
        self._put_message('warning', entry, warning_desc)
        self._warnings[entry.id] = (False, warning_desc)
        self._update_warnings_str()

    def _clear_warning(self, entry):
        """Clear a previously emitted warning for this condition."""
        self.log.info('condition %r normal again', entry)
        was_real_warning = self._warnings.pop(entry.id)[0]
        if entry.scriptaction == 'pausecount':
            self._pausecount.pop(entry.id, None)
            self._update_pausecount_str()
        self._update_warnings_str()

        if was_real_warning:
            self._put_message('resolved', entry, '')
            if entry.okmessage and entry.type:
                msg = strftime('%Y-%m-%d %H:%M -- ')
                msg += '%s\n\nWarning was: %s' % (entry.okmessage,
                                                  entry.message)
                for notifier in self._notifiers[entry.type]:
                    notifier.send('NICOS warning resolved', msg)
            if entry.okaction:
                self._put_message('action', entry, entry.okaction)
                self._spawn_action(entry.okaction)

    # internal helper methods

    def _put_message(self, msgtype, entry, message):
        if entry is not None:
            message = [currenttime(), message, entry.id]
        self._queue.put('watchdog/%s%s%s\n' %
                        (msgtype, OP_TELL, cache_dump(message)))

    def _update_mailreceivers(self, emails):
        self.log.info('updating any Mailer receivers to %s', emails)
        for notifier in self._all_notifiers:
            if isinstance(notifier, Mailer):
                # we're in slave mode, so _setROParam is necessary to set params
                notifier._setROParam('receivers', emails)

    def _update_warnings_str(self):
        self._put_message('warnings', None,
                          '\n'.join(v[1] for v in self._warnings.values()))

    def _update_pausecount_str(self):
        self._put_message('pausecount', None,
                          ', '.join(self._pausecount.values()))

    def _spawn_action(self, action):
        self.log.warning('will execute action %r', action)
        script = path.join(config.nicos_root, 'bin', 'nicos-script')
        createSubprocess([
            sys.executable,
            script,
            '-M',  # start in maintenance mode
            '-S',
            '60',  # abort after 60 seconds
            '-A',
            'watchdog-action',  # appname for the logfiles
            ','.join(self._setups),  # setups to load
            action
        ])  # code to execute
Exemplo n.º 32
0
class DynamoDBBackend(BaseBackend):
    def __init__(self):
        self.tables = OrderedDict()

    def create_table(self, name, **params):
        table = Table(name, **params)
        self.tables[name] = table
        return table

    def delete_table(self, name):
        return self.tables.pop(name, None)

    def update_table_throughput(self, name, new_read_units, new_write_units):
        table = self.tables[name]
        table.read_capacity = new_read_units
        table.write_capacity = new_write_units
        return table

    def put_item(self, table_name, item_attrs):
        table = self.tables.get(table_name)
        if not table:
            return None

        return table.put_item(item_attrs)

    def get_item(self, table_name, hash_key_dict, range_key_dict):
        table = self.tables.get(table_name)
        if not table:
            return None

        hash_key = DynamoType(hash_key_dict)
        range_key = DynamoType(range_key_dict) if range_key_dict else None

        return table.get_item(hash_key, range_key)

    def query(self, table_name, hash_key_dict, range_comparison,
              range_value_dicts):
        table = self.tables.get(table_name)
        if not table:
            return None, None

        hash_key = DynamoType(hash_key_dict)
        range_values = [
            DynamoType(range_value) for range_value in range_value_dicts
        ]

        return table.query(hash_key, range_comparison, range_values)

    def scan(self, table_name, filters):
        table = self.tables.get(table_name)
        if not table:
            return None, None, None

        scan_filters = {}
        for key, (comparison_operator,
                  comparison_values) in filters.iteritems():
            dynamo_types = [DynamoType(value) for value in comparison_values]
            scan_filters[key] = (comparison_operator, dynamo_types)

        return table.scan(scan_filters)

    def delete_item(self, table_name, hash_key_dict, range_key_dict):
        table = self.tables.get(table_name)
        if not table:
            return None

        hash_key = DynamoType(hash_key_dict)
        range_key = DynamoType(range_key_dict) if range_key_dict else None

        return table.delete_item(hash_key, range_key)
Exemplo n.º 33
0
class ConnectionState:
    def __init__(self, *, dispatch, chunker, handlers, syncer, http, loop, **options):
        self.loop = loop
        self.http = http
        self.max_messages = max(options.get('max_messages', 5000), 100)
        self.dispatch = dispatch
        self.chunker = chunker
        self.syncer = syncer
        self.is_bot = None
        self.handlers = handlers
        self.shard_count = None
        self._ready_task = None
        self._fetch_offline = options.get('fetch_offline_members', True)
        self.heartbeat_timeout = options.get('heartbeat_timeout', 60.0)
        self._listeners = []

        activity = options.get('activity', None)
        if activity:
            if not isinstance(activity, _ActivityTag):
                raise TypeError('activity parameter must be one of Game, Streaming, or Activity.')

            activity = activity.to_dict()

        status = options.get('status', None)
        if status:
            if status is Status.offline:
                status = 'invisible'
            else:
                status = str(status)

        self._activity = activity
        self._status = status

        self.clear()

    def clear(self):
        self.user = None
        self._users = weakref.WeakValueDictionary()
        self._emojis = {}
        self._calls = {}
        self._guilds = {}
        self._voice_clients = {}

        # LRU of max size 128
        self._private_channels = OrderedDict()
        # extra dict to look up private channels by user id
        self._private_channels_by_user = {}
        self._messages = deque(maxlen=self.max_messages)

    def process_listeners(self, listener_type, argument, result):
        removed = []
        for i, listener in enumerate(self._listeners):
            if listener.type != listener_type:
                continue

            future = listener.future
            if future.cancelled():
                removed.append(i)
                continue

            try:
                passed = listener.predicate(argument)
            except Exception as exc:
                future.set_exception(exc)
                removed.append(i)
            else:
                if passed:
                    future.set_result(result)
                    removed.append(i)
                    if listener.type == ListenerType.chunk:
                        break

        for index in reversed(removed):
            del self._listeners[index]

    def call_handlers(self, key, *args, **kwargs):
        try:
            func = self.handlers[key]
        except KeyError:
            pass
        else:
            func(*args, **kwargs)

    @property
    def self_id(self):
        u = self.user
        return u.id if u else None

    @property
    def voice_clients(self):
        return list(self._voice_clients.values())

    def _get_voice_client(self, guild_id):
        return self._voice_clients.get(guild_id)

    def _add_voice_client(self, guild_id, voice):
        self._voice_clients[guild_id] = voice

    def _remove_voice_client(self, guild_id):
        self._voice_clients.pop(guild_id, None)

    def _update_references(self, ws):
        for vc in self.voice_clients:
            vc.main_ws = ws

    def store_user(self, data):
        # this way is 300% faster than `dict.setdefault`.
        user_id = int(data['id'])
        try:
            return self._users[user_id]
        except KeyError:
            user = User(state=self, data=data)
            if user.discriminator != '0000':
                self._users[user_id] = user
            return user

    def get_user(self, id):
        return self._users.get(id)

    def store_emoji(self, guild, data):
        emoji_id = int(data['id'])
        self._emojis[emoji_id] = emoji = Emoji(guild=guild, state=self, data=data)
        return emoji

    @property
    def guilds(self):
        return list(self._guilds.values())

    def _get_guild(self, guild_id):
        return self._guilds.get(guild_id)

    def _add_guild(self, guild):
        self._guilds[guild.id] = guild

    def _remove_guild(self, guild):
        self._guilds.pop(guild.id, None)

        for emoji in guild.emojis:
            self._emojis.pop(emoji.id, None)

        del guild

    @property
    def emojis(self):
        return list(self._emojis.values())

    def get_emoji(self, emoji_id):
        return self._emojis.get(emoji_id)

    @property
    def private_channels(self):
        return list(self._private_channels.values())

    def _get_private_channel(self, channel_id):
        try:
            value = self._private_channels[channel_id]
        except KeyError:
            return None
        else:
            self._private_channels.move_to_end(channel_id)
            return value

    def _get_private_channel_by_user(self, user_id):
        return self._private_channels_by_user.get(user_id)

    def _add_private_channel(self, channel):
        channel_id = channel.id
        self._private_channels[channel_id] = channel

        if self.is_bot and len(self._private_channels) > 128:
            _, to_remove = self._private_channels.popitem(last=False)
            if isinstance(to_remove, DMChannel):
                self._private_channels_by_user.pop(to_remove.recipient.id, None)

        if isinstance(channel, DMChannel):
            self._private_channels_by_user[channel.recipient.id] = channel

    def add_dm_channel(self, data):
        channel = DMChannel(me=self.user, state=self, data=data)
        self._add_private_channel(channel)
        return channel

    def _remove_private_channel(self, channel):
        self._private_channels.pop(channel.id, None)
        if isinstance(channel, DMChannel):
            self._private_channels_by_user.pop(channel.recipient.id, None)

    def _get_message(self, msg_id):
        return utils.find(lambda m: m.id == msg_id, reversed(self._messages))

    def _add_guild_from_data(self, guild):
        guild = Guild(data=guild, state=self)
        self._add_guild(guild)
        return guild

    def chunks_needed(self, guild):
        for _ in range(math.ceil(guild._member_count / 1000)):
            yield self.receive_chunk(guild.id)

    def _get_guild_channel(self, data):
        try:
            guild = self._get_guild(int(data['guild_id']))
        except KeyError:
            channel = self.get_channel(int(data['channel_id']))
            guild = None
        else:
            channel = guild and guild.get_channel(int(data['channel_id']))

        return channel, guild

    async def request_offline_members(self, guilds):
        # get all the chunks
        chunks = []
        for guild in guilds:
            chunks.extend(self.chunks_needed(guild))

        # we only want to request ~75 guilds per chunk request.
        splits = [guilds[i:i + 75] for i in range(0, len(guilds), 75)]
        for split in splits:
            await self.chunker(split)

        # wait for the chunks
        if chunks:
            try:
                await utils.sane_wait_for(chunks, timeout=len(chunks) * 30.0, loop=self.loop)
            except asyncio.TimeoutError:
                log.info('Somehow timed out waiting for chunks.')

    async def _delay_ready(self):
        try:
            launch = self._ready_state.launch

            # only real bots wait for GUILD_CREATE streaming
            if self.is_bot:
                while not launch.is_set():
                    # this snippet of code is basically waiting 2 seconds
                    # until the last GUILD_CREATE was sent
                    launch.set()
                    await asyncio.sleep(2, loop=self.loop)

            guilds = next(zip(*self._ready_state.guilds), [])
            if self._fetch_offline:
                await self.request_offline_members(guilds)

            for guild, unavailable in self._ready_state.guilds:
                if unavailable is False:
                    self.dispatch('guild_available', guild)
                else:
                    self.dispatch('guild_join', guild)

            # remove the state
            try:
                del self._ready_state
            except AttributeError:
                pass # already been deleted somehow

            # call GUILD_SYNC after we're done chunking
            if not self.is_bot:
                log.info('Requesting GUILD_SYNC for %s guilds', len(self.guilds))
                await self.syncer([s.id for s in self.guilds])
        except asyncio.CancelledError:
            pass
        else:
            # dispatch the event
            self.call_handlers('ready')
            self.dispatch('ready')
        finally:
            self._ready_task = None

    def parse_ready(self, data):
        if self._ready_task is not None:
            self._ready_task.cancel()

        self._ready_state = ReadyState(launch=asyncio.Event(), guilds=[])
        self.clear()
        self.user = user = ClientUser(state=self, data=data['user'])
        self._users[user.id] = user

        guilds = self._ready_state.guilds
        for guild_data in data['guilds']:
            guild = self._add_guild_from_data(guild_data)
            if (not self.is_bot and not guild.unavailable) or guild.large:
                guilds.append((guild, guild.unavailable))

        for relationship in data.get('relationships', []):
            try:
                r_id = int(relationship['id'])
            except KeyError:
                continue
            else:
                user._relationships[r_id] = Relationship(state=self, data=relationship)

        for pm in data.get('private_channels', []):
            factory, _ = _channel_factory(pm['type'])
            self._add_private_channel(factory(me=user, data=pm, state=self))

        self.dispatch('connect')
        self._ready_task = asyncio.ensure_future(self._delay_ready(), loop=self.loop)

    def parse_resumed(self, data):
        self.dispatch('resumed')

    def parse_message_create(self, data):
        channel, _ = self._get_guild_channel(data)
        message = Message(channel=channel, data=data, state=self)
        self.dispatch('message', message)
        self._messages.append(message)
        if channel and channel._type in (0, 5):
            channel.last_message_id = message.id

    def parse_message_delete(self, data):
        raw = RawMessageDeleteEvent(data)
        found = self._get_message(raw.message_id)
        raw.cached_message = found
        self.dispatch('raw_message_delete', raw)
        if found is not None:
            self.dispatch('message_delete', found)
            self._messages.remove(found)

    def parse_message_delete_bulk(self, data):
        raw = RawBulkMessageDeleteEvent(data)
        found_messages = [message for message in self._messages if message.id in raw.message_ids]
        raw.cached_messages = found_messages
        self.dispatch('raw_bulk_message_delete', raw)
        if found_messages:
            self.dispatch('bulk_message_delete', found_messages)
            for msg in found_messages:
                self._messages.remove(msg)

    def parse_message_update(self, data):
        raw = RawMessageUpdateEvent(data)
        self.dispatch('raw_message_edit', raw)
        message = self._get_message(raw.message_id)
        if message is not None:
            older_message = copy.copy(message)
            if 'call' in data:
                # call state message edit
                message._handle_call(data['call'])
            elif 'content' not in data:
                # embed only edit
                message.embeds = [Embed.from_dict(d) for d in data['embeds']]
            else:
                message._update(channel=message.channel, data=data)

            self.dispatch('message_edit', older_message, message)

    def parse_message_reaction_add(self, data):
        emoji_data = data['emoji']
        emoji_id = utils._get_as_snowflake(emoji_data, 'id')
        emoji = PartialEmoji.with_state(self, animated=emoji_data['animated'], id=emoji_id, name=emoji_data['name'])
        raw = RawReactionActionEvent(data, emoji)
        self.dispatch('raw_reaction_add', raw)

        # rich interface here
        message = self._get_message(raw.message_id)
        if message is not None:
            emoji = self._upgrade_partial_emoji(emoji)
            reaction = message._add_reaction(data, emoji, raw.user_id)
            user = self._get_reaction_user(message.channel, raw.user_id)
            if user:
                self.dispatch('reaction_add', reaction, user)

    def parse_message_reaction_remove_all(self, data):
        raw = RawReactionClearEvent(data)
        self.dispatch('raw_reaction_clear', raw)

        message = self._get_message(raw.message_id)
        if message is not None:
            old_reactions = message.reactions.copy()
            message.reactions.clear()
            self.dispatch('reaction_clear', message, old_reactions)

    def parse_message_reaction_remove(self, data):
        emoji_data = data['emoji']
        emoji_id = utils._get_as_snowflake(emoji_data, 'id')
        emoji = PartialEmoji.with_state(self, animated=emoji_data['animated'], id=emoji_id, name=emoji_data['name'])
        raw = RawReactionActionEvent(data, emoji)
        self.dispatch('raw_reaction_remove', raw)

        message = self._get_message(raw.message_id)
        if message is not None:
            emoji = self._upgrade_partial_emoji(emoji)
            try:
                reaction = message._remove_reaction(data, emoji, raw.user_id)
            except (AttributeError, ValueError): # eventual consistency lol
                pass
            else:
                user = self._get_reaction_user(message.channel, raw.user_id)
                if user:
                    self.dispatch('reaction_remove', reaction, user)

    def parse_presence_update(self, data):
        guild_id = utils._get_as_snowflake(data, 'guild_id')
        guild = self._get_guild(guild_id)
        if guild is None:
            log.warning('PRESENCE_UPDATE referencing an unknown guild ID: %s. Discarding.', guild_id)
            return

        user = data['user']
        member_id = int(user['id'])
        member = guild.get_member(member_id)
        if member is None:
            if 'username' not in user:
                # sometimes we receive 'incomplete' member data post-removal.
                # skip these useless cases.
                return

            member, old_member = Member._from_presence_update(guild=guild, data=data, state=self)
            guild._add_member(member)
        else:
            old_member = Member._copy(member)
            user_update = member._presence_update(data=data, user=user)
            if user_update:
                self.dispatch('user_update', user_update[0], user_update[1])

        self.dispatch('member_update', old_member, member)

    def parse_user_update(self, data):
        self.user._update(data)

    def parse_channel_delete(self, data):
        guild = self._get_guild(utils._get_as_snowflake(data, 'guild_id'))
        channel_id = int(data['id'])
        if guild is not None:
            channel = guild.get_channel(channel_id)
            if channel is not None:
                guild._remove_channel(channel)
                self.dispatch('guild_channel_delete', channel)
        else:
            # the reason we're doing this is so it's also removed from the
            # private channel by user cache as well
            channel = self._get_private_channel(channel_id)
            if channel is not None:
                self._remove_private_channel(channel)
                self.dispatch('private_channel_delete', channel)

    def parse_channel_update(self, data):
        channel_type = try_enum(ChannelType, data.get('type'))
        channel_id = int(data['id'])
        if channel_type is ChannelType.group:
            channel = self._get_private_channel(channel_id)
            old_channel = copy.copy(channel)
            channel._update_group(data)
            self.dispatch('private_channel_update', old_channel, channel)
            return

        guild_id = utils._get_as_snowflake(data, 'guild_id')
        guild = self._get_guild(guild_id)
        if guild is not None:
            channel = guild.get_channel(channel_id)
            if channel is not None:
                old_channel = copy.copy(channel)
                channel._update(guild, data)
                self.dispatch('guild_channel_update', old_channel, channel)
            else:
                log.warning('CHANNEL_UPDATE referencing an unknown channel ID: %s. Discarding.', channel_id)
        else:
            log.warning('CHANNEL_UPDATE referencing an unknown guild ID: %s. Discarding.', guild_id)

    def parse_channel_create(self, data):
        factory, ch_type = _channel_factory(data['type'])
        if factory is None:
            log.warning('CHANNEL_CREATE referencing an unknown channel type %s. Discarding.', data['type'])
            return

        channel = None

        if ch_type in (ChannelType.group, ChannelType.private):
            channel_id = int(data['id'])
            if self._get_private_channel(channel_id) is None:
                channel = factory(me=self.user, data=data, state=self)
                self._add_private_channel(channel)
                self.dispatch('private_channel_create', channel)
        else:
            guild_id = utils._get_as_snowflake(data, 'guild_id')
            guild = self._get_guild(guild_id)
            if guild is not None:
                channel = factory(guild=guild, state=self, data=data)
                guild._add_channel(channel)
                self.dispatch('guild_channel_create', channel)
            else:
                log.warning('CHANNEL_CREATE referencing an unknown guild ID: %s. Discarding.', guild_id)
                return

    def parse_channel_pins_update(self, data):
        channel_id = int(data['channel_id'])
        channel = self.get_channel(channel_id)
        if channel is None:
            log.warning('CHANNEL_PINS_UPDATE referencing an unknown channel ID: %s. Discarding.', channel_id)
            return

        last_pin = utils.parse_time(data['last_pin_timestamp']) if data['last_pin_timestamp'] else None

        try:
            # I have not imported discord.abc in this file
            # the isinstance check is also 2x slower than just checking this attribute
            # so we're just gonna check it since it's easier and faster and lazier
            channel.guild
        except AttributeError:
            self.dispatch('private_channel_pins_update', channel, last_pin)
        else:
            self.dispatch('guild_channel_pins_update', channel, last_pin)

    def parse_channel_recipient_add(self, data):
        channel = self._get_private_channel(int(data['channel_id']))
        user = self.store_user(data['user'])
        channel.recipients.append(user)
        self.dispatch('group_join', channel, user)

    def parse_channel_recipient_remove(self, data):
        channel = self._get_private_channel(int(data['channel_id']))
        user = self.store_user(data['user'])
        try:
            channel.recipients.remove(user)
        except ValueError:
            pass
        else:
            self.dispatch('group_remove', channel, user)

    def parse_guild_member_add(self, data):
        guild = self._get_guild(int(data['guild_id']))
        if guild is None:
            log.warning('GUILD_MEMBER_ADD referencing an unknown guild ID: %s. Discarding.', data['guild_id'])
            return

        member = Member(guild=guild, data=data, state=self)
        guild._add_member(member)
        guild._member_count += 1
        self.dispatch('member_join', member)

    def parse_guild_member_remove(self, data):
        guild = self._get_guild(int(data['guild_id']))
        if guild is not None:
            user_id = int(data['user']['id'])
            member = guild.get_member(user_id)
            if member is not None:
                guild._remove_member(member)
                guild._member_count -= 1
                self.dispatch('member_remove', member)
        else:
            log.warning('GUILD_MEMBER_REMOVE referencing an unknown guild ID: %s. Discarding.', data['guild_id'])

    def parse_guild_member_update(self, data):
        guild = self._get_guild(int(data['guild_id']))
        user = data['user']
        user_id = int(user['id'])
        if guild is None:
            log.warning('GUILD_MEMBER_UPDATE referencing an unknown guild ID: %s. Discarding.', data['guild_id'])
            return

        member = guild.get_member(user_id)
        if member is not None:
            old_member = copy.copy(member)
            member._update(data)
            self.dispatch('member_update', old_member, member)
        else:
            log.warning('GUILD_MEMBER_UPDATE referencing an unknown member ID: %s. Discarding.', user_id)

    def parse_guild_emojis_update(self, data):
        guild = self._get_guild(int(data['guild_id']))
        if guild is None:
            log.warning('GUILD_EMOJIS_UPDATE referencing an unknown guild ID: %s. Discarding.', data['guild_id'])
            return

        before_emojis = guild.emojis
        for emoji in before_emojis:
            self._emojis.pop(emoji.id, None)
        guild.emojis = tuple(map(lambda d: self.store_emoji(guild, d), data['emojis']))
        self.dispatch('guild_emojis_update', guild, before_emojis, guild.emojis)

    def _get_create_guild(self, data):
        if data.get('unavailable') is False:
            # GUILD_CREATE with unavailable in the response
            # usually means that the guild has become available
            # and is therefore in the cache
            guild = self._get_guild(int(data['id']))
            if guild is not None:
                guild.unavailable = False
                guild._from_data(data)
                return guild

        return self._add_guild_from_data(data)

    async def _chunk_and_dispatch(self, guild, unavailable):
        chunks = list(self.chunks_needed(guild))
        await self.chunker(guild)
        if chunks:
            try:
                await utils.sane_wait_for(chunks, timeout=len(chunks), loop=self.loop)
            except asyncio.TimeoutError:
                log.info('Somehow timed out waiting for chunks.')

        if unavailable is False:
            self.dispatch('guild_available', guild)
        else:
            self.dispatch('guild_join', guild)

    def parse_guild_create(self, data):
        unavailable = data.get('unavailable')
        if unavailable is True:
            # joined a guild with unavailable == True so..
            return

        guild = self._get_create_guild(data)

        # check if it requires chunking
        if guild.large:
            if unavailable is False:
                # check if we're waiting for 'useful' READY
                # and if we are, we don't want to dispatch any
                # event such as guild_join or guild_available
                # because we're still in the 'READY' phase. Or
                # so we say.
                try:
                    state = self._ready_state
                    state.launch.clear()
                    state.guilds.append((guild, unavailable))
                except AttributeError:
                    # the _ready_state attribute is only there during
                    # processing of useful READY.
                    pass
                else:
                    return

            # since we're not waiting for 'useful' READY we'll just
            # do the chunk request here if wanted
            if self._fetch_offline:
                asyncio.ensure_future(self._chunk_and_dispatch(guild, unavailable), loop=self.loop)
                return

        # Dispatch available if newly available
        if unavailable is False:
            self.dispatch('guild_available', guild)
        else:
            self.dispatch('guild_join', guild)

    def parse_guild_sync(self, data):
        guild = self._get_guild(int(data['id']))
        guild._sync(data)

    def parse_guild_update(self, data):
        guild = self._get_guild(int(data['id']))
        if guild is not None:
            old_guild = copy.copy(guild)
            guild._from_data(data)
            self.dispatch('guild_update', old_guild, guild)
        else:
            log.warning('GUILD_UPDATE referencing an unknown guild ID: %s. Discarding.', data['id'])

    def parse_guild_delete(self, data):
        guild = self._get_guild(int(data['id']))
        if guild is None:
            log.warning('GUILD_DELETE referencing an unknown guild ID: %s. Discarding.', data['id'])
            return

        if data.get('unavailable', False) and guild is not None:
            # GUILD_DELETE with unavailable being True means that the
            # guild that was available is now currently unavailable
            guild.unavailable = True
            self.dispatch('guild_unavailable', guild)
            return

        # do a cleanup of the messages cache
        self._messages = deque((msg for msg in self._messages if msg.guild != guild), maxlen=self.max_messages)

        self._remove_guild(guild)
        self.dispatch('guild_remove', guild)

    def parse_guild_ban_add(self, data):
        # we make the assumption that GUILD_BAN_ADD is done
        # before GUILD_MEMBER_REMOVE is called
        # hence we don't remove it from cache or do anything
        # strange with it, the main purpose of this event
        # is mainly to dispatch to another event worth listening to for logging
        guild = self._get_guild(int(data['guild_id']))
        if guild is not None:
            try:
                user = User(data=data['user'], state=self)
            except KeyError:
                pass
            else:
                member = guild.get_member(user.id) or user
                self.dispatch('member_ban', guild, member)

    def parse_guild_ban_remove(self, data):
        guild = self._get_guild(int(data['guild_id']))
        if guild is not None:
            if 'user' in data:
                user = self.store_user(data['user'])
                self.dispatch('member_unban', guild, user)

    def parse_guild_role_create(self, data):
        guild = self._get_guild(int(data['guild_id']))
        if guild is None:
            log.warning('GUILD_ROLE_CREATE referencing an unknown guild ID: %s. Discarding.', data['guild_id'])
            return

        role_data = data['role']
        role = Role(guild=guild, data=role_data, state=self)
        guild._add_role(role)
        self.dispatch('guild_role_create', role)

    def parse_guild_role_delete(self, data):
        guild = self._get_guild(int(data['guild_id']))
        if guild is not None:
            role_id = int(data['role_id'])
            try:
                role = guild._remove_role(role_id)
            except KeyError:
                return
            else:
                self.dispatch('guild_role_delete', role)
        else:
            log.warning('GUILD_ROLE_DELETE referencing an unknown guild ID: %s. Discarding.', data['guild_id'])

    def parse_guild_role_update(self, data):
        guild = self._get_guild(int(data['guild_id']))
        if guild is not None:
            role_data = data['role']
            role_id = int(role_data['id'])
            role = guild.get_role(role_id)
            if role is not None:
                old_role = copy.copy(role)
                role._update(role_data)
                self.dispatch('guild_role_update', old_role, role)
        else:
            log.warning('GUILD_ROLE_UPDATE referencing an unknown guild ID: %s. Discarding.', data['guild_id'])

    def parse_guild_members_chunk(self, data):
        guild_id = int(data['guild_id'])
        guild = self._get_guild(guild_id)
        members = data.get('members', [])
        for member in members:
            m = Member(guild=guild, data=member, state=self)
            existing = guild.get_member(m.id)
            if existing is None or existing.joined_at is None:
                guild._add_member(m)

        log.info('Processed a chunk for %s members in guild ID %s.', len(members), guild_id)
        self.process_listeners(ListenerType.chunk, guild, len(members))

    def parse_guild_integrations_update(self, data):
        guild = self._get_guild(int(data['guild_id']))
        if guild is not None:
            self.dispatch('guild_integrations_update', guild)
        else:
            log.warning('GUILD_INTEGRATIONS_UPDATE referencing an unknown guild ID: %s. Discarding.', data['guild_id'])

    def parse_webhooks_update(self, data):
        channel = self.get_channel(int(data['channel_id']))
        if channel is not None:
            self.dispatch('webhooks_update', channel)
        else:
            log.warning('WEBHOOKS_UPDATE referencing an unknown channel ID: %s. Discarding.', data['channel_id'])

    def parse_voice_state_update(self, data):
        guild = self._get_guild(utils._get_as_snowflake(data, 'guild_id'))
        channel_id = utils._get_as_snowflake(data, 'channel_id')
        if guild is not None:
            if int(data['user_id']) == self.user.id:
                voice = self._get_voice_client(guild.id)
                if voice is not None:
                    ch = guild.get_channel(channel_id)
                    if ch is not None:
                        voice.channel = ch

            member, before, after = guild._update_voice_state(data, channel_id)
            if member is not None:
                self.dispatch('voice_state_update', member, before, after)
            else:
                log.warning('VOICE_STATE_UPDATE referencing an unknown member ID: %s. Discarding.', data['user_id'])
        else:
            # in here we're either at private or group calls
            call = self._calls.get(channel_id)
            if call is not None:
                call._update_voice_state(data)

    def parse_voice_server_update(self, data):
        try:
            key_id = int(data['guild_id'])
        except KeyError:
            key_id = int(data['channel_id'])

        vc = self._get_voice_client(key_id)
        if vc is not None:
            asyncio.ensure_future(vc._create_socket(key_id, data))

    def parse_typing_start(self, data):
        channel, guild = self._get_guild_channel(data)
        if channel is not None:
            member = None
            user_id = utils._get_as_snowflake(data, 'user_id')
            if isinstance(channel, DMChannel):
                member = channel.recipient
            elif isinstance(channel, TextChannel) and guild is not None:
                member = guild.get_member(user_id)
            elif isinstance(channel, GroupChannel):
                member = utils.find(lambda x: x.id == user_id, channel.recipients)

            if member is not None:
                timestamp = datetime.datetime.utcfromtimestamp(data.get('timestamp'))
                self.dispatch('typing', channel, member, timestamp)

    def parse_relationship_add(self, data):
        key = int(data['id'])
        old = self.user.get_relationship(key)
        new = Relationship(state=self, data=data)
        self.user._relationships[key] = new
        if old is not None:
            self.dispatch('relationship_update', old, new)
        else:
            self.dispatch('relationship_add', new)

    def parse_relationship_remove(self, data):
        key = int(data['id'])
        try:
            old = self.user._relationships.pop(key)
        except KeyError:
            pass
        else:
            self.dispatch('relationship_remove', old)

    def _get_reaction_user(self, channel, user_id):
        if isinstance(channel, TextChannel):
            return channel.guild.get_member(user_id)
        return self.get_user(user_id)

    def get_reaction_emoji(self, data):
        emoji_id = utils._get_as_snowflake(data, 'id')

        if not emoji_id:
            return data['name']

        try:
            return self._emojis[emoji_id]
        except KeyError:
            return PartialEmoji(animated=data['animated'], id=emoji_id, name=data['name'])

    def _upgrade_partial_emoji(self, emoji):
        emoji_id = emoji.id
        if not emoji_id:
            return emoji.name
        try:
            return self._emojis[emoji_id]
        except KeyError:
            return emoji

    def get_channel(self, id):
        if id is None:
            return None

        pm = self._get_private_channel(id)
        if pm is not None:
            return pm

        for guild in self.guilds:
            channel = guild.get_channel(id)
            if channel is not None:
                return channel

    def create_message(self, *, channel, data):
        return Message(state=self, channel=channel, data=data)

    def receive_chunk(self, guild_id):
        future = self.loop.create_future()
        listener = Listener(ListenerType.chunk, future, lambda s: s.id == guild_id)
        self._listeners.append(listener)
        return future
Exemplo n.º 34
0
def make_table(dt, hide_bar=None):
    """
    Build the HTML needed for a MultiQC table.
    :param data: MultiQC datatable object
    """

    table_id = dt.pconfig.get('id', 'table_{}'.format(''.join(random.sample(letters, 4))))
    table_id = report.save_htmlid(table_id)
    t_headers = OrderedDict()
    t_modal_headers = OrderedDict()
    t_rows = OrderedDict()
    t_rows_empty = OrderedDict()
    dt.raw_vals = defaultdict(lambda: dict())
    empty_cells = dict()
    hidden_cols = 1
    table_title = dt.pconfig.get('table_title')
    if table_title is None:
        table_title = table_id.replace("_", " ").title()

    for idx, k, header in dt.get_headers_in_order():

        rid = header['rid']

        # Build the table header cell
        shared_key = ''
        if header.get('shared_key', None) is not None:
            shared_key = ' data-shared-key={}'.format(header['shared_key'])

        hide = ''
        muted = ''
        checked = ' checked="checked"'
        if header.get('hidden', False) is True:
            hide = 'hidden'
            muted = ' text-muted'
            checked = ''
            hidden_cols += 1

        data_attr = 'data-dmax="{}" data-dmin="{}" data-namespace="{}" {}' \
            .format(header['dmax'], header['dmin'], header['namespace'], shared_key)

        if header.get('namespace'):
            cell_contents = '<span class="mqc_table_tooltip" title="{}: {}">{}</span>' \
                .format(header['namespace'], header['description'], header['title'])
        else:
            cell_contents = '<span class="mqc_table_tooltip" title="{} {}">{}</span>' \
                .format(header['namespace'], header['description'], header['title'])

        t_headers[rid] = '<th id="header_{rid}" class="{rid} {h}" {da}>{c}</th>' \
            .format(rid=rid, h=hide, da=data_attr, c=cell_contents)

        empty_cells[rid] = '<td class="data-coloured {rid} {h}"></td>'.format(rid=rid, h=hide)

        # Build the modal table row
        t_modal_headers[rid] = """
        <tr class="{rid}{muted}" style="background-color: rgba({col}, 0.15);">
          <td class="sorthandle ui-sortable-handle">||</span></td>
          <td style="text-align:center;">
            <input class="mqc_table_col_visible" type="checkbox" {checked} value="{rid}" data-target="#{tid}">
          </td>
          <td>{name}</td>
          <td>{title}</td>
          <td>{desc}</td>
          <td>{col_id}</td>
          <td>{sk}</td>
        </tr>""".format(
            rid=rid,
            muted=muted,
            checked=checked,
            tid=table_id,
            col=header['colour'],
            name=header['namespace'],
            title=header['title'],
            desc=header['description'],
            col_id='<code>{}</code>'.format(k),
            sk=header.get('shared_key', '')
        )

        # Make a colour scale
        if header['scale'] == False:
            c_scale = None
        else:
            c_scale = mqc_colour.mqc_colour_scale(header['scale'], header['dmin'], header['dmax'])

        # Add the data table cells
        for (s_name, samp) in dt.data[idx].items():
            if k in samp:
                val = samp[k]
                kname = '{}_{}'.format(header['namespace'], rid)
                dt.raw_vals[s_name][kname] = val

                # if "is_int" in header.keys():
                #     val = int(val)
                #     print(val)

                if 'modify' in header and callable(header['modify']):
                    val = header['modify'](val)

                try:
                    dmin = header['dmin']
                    dmax = header['dmax']
                    percentage = ((float(val) - dmin) / (dmax - dmin)) * 100
                    percentage = min(percentage, 100)
                    percentage = max(percentage, 0)
                except (ZeroDivisionError, ValueError):
                    percentage = 0

                try:
                    if not header.get("is_int"):
                        valstring = str(header['format'].format(val))
                    else:
                        valstring = str(int(val))

                except ValueError:
                    try:
                        valstring = str(header['format'].format(float(val)))

                    except ValueError:
                        valstring = str(val)

                except:
                    valstring = str(val)


                # This is horrible, but Python locale settings are worse
                if config.thousandsSep_format is None:
                    config.thousandsSep_format = '<span class="mqc_thousandSep"></span>'
                if config.decimalPoint_format is None:
                    config.decimalPoint_format = '.'
                valstring = valstring.replace('.', 'DECIMAL').replace(',', 'THOUSAND')
                valstring = valstring.replace('DECIMAL', config.decimalPoint_format).replace('THOUSAND',
                                                                                             config.thousandsSep_format)

                # Percentage suffixes etc
                if header.get('suffix') == "show_perc":
                    suff_dict = header.get("suffix_dict")
                    valstring += str(suff_dict.get(s_name))
                else:
                    valstring += header.get('suffix', '')

                # Conditional formatting
                cmatches = {cfck: False for cfc in config.table_cond_formatting_colours for cfck in cfc}
                # Find general rules followed by column-specific rules
                for cfk in ['all_columns', rid]:
                    if cfk in config.table_cond_formatting_rules:
                        # Loop through match types
                        for ftype in cmatches.keys():
                            # Loop through array of comparison types
                            for cmp in config.table_cond_formatting_rules[cfk].get(ftype, []):
                                try:
                                    # Each comparison should be a dict with single key: val
                                    if 's_eq' in cmp and str(cmp['s_eq']).lower() == str(val).lower():
                                        cmatches[ftype] = True
                                    if 's_contains' in cmp and str(cmp['s_contains']).lower() in str(val).lower():
                                        cmatches[ftype] = True
                                    if 's_ne' in cmp and str(cmp['s_ne']).lower() != str(val).lower():
                                        cmatches[ftype] = True
                                    if 'eq' in cmp and float(cmp['eq']) == float(val):
                                        cmatches[ftype] = True
                                    if 'ne' in cmp and float(cmp['ne']) != float(val):
                                        cmatches[ftype] = True
                                    if 'gt' in cmp and float(cmp['gt']) < float(val):
                                        cmatches[ftype] = True
                                    if 'lt' in cmp and float(cmp['lt']) > float(val):
                                        cmatches[ftype] = True
                                except:
                                    logger.warn(
                                        "Not able to apply table conditional formatting to '{}' ({})".format(val, cmp))
                # Apply HTML in order of config keys
                bgcol = None
                for cfc in config.table_cond_formatting_colours:
                    for cfck in cfc:  # should always be one, but you never know
                        if cmatches[cfck]:
                            bgcol = cfc[cfck]
                if bgcol is not None:
                    valstring = '<span class="badge" style="background-color:{}">{}</span>'.format(bgcol, valstring)

                # Build HTML
                if not header['scale']:
                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][rid] = '<td class="{rid} {h}">{v}</td>'.format(rid=rid, h=hide, v=valstring)

                # coloring with quartiles
                elif header['scale'] == "quart":

                    col_dict = header['col_dict']
                    bar_dict = header.get('bar_dict')
                    col = ' background-color:{};'.format(col_dict.get(s_name))
                    if bar_dict:
                        bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(bar_dict.get(s_name)+3, col)
                    else:
                        bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col)
                    # bar percentage here

                    val_html = '<span class="val">{}</span>'.format(valstring)
                    wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html)
                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide,
                                                                                                c=wrapper_html)

                else:
                    if c_scale is not None:
                        col = ' background-color:{};'.format(c_scale.get_colour(val))
                    else:
                        col = ''
                    bar_html = '<span class="bar" style="width:{}%;{}"></span>'.format(percentage, col)
                    val_html = '<span class="val">{}</span>'.format(valstring)
                    wrapper_html = '<div class="wrapper">{}{}</div>'.format(bar_html, val_html)

                    if s_name not in t_rows:
                        t_rows[s_name] = dict()
                    t_rows[s_name][rid] = '<td class="data-coloured {rid} {h}">{c}</td>'.format(rid=rid, h=hide,
                                                                                                c=wrapper_html)

                # Is this cell hidden or empty?
                if s_name not in t_rows_empty:
                    t_rows_empty[s_name] = dict()
                t_rows_empty[s_name][rid] = header.get('hidden', False) or str(val).strip() == ''

        # Remove header if we don't have any filled cells for it
        if sum([len(rows) for rows in t_rows.values()]) == 0:
            t_headers.pop(rid, None)
            t_modal_headers.pop(rid, None)
            logger.debug('Removing header {} from general stats table, as no data'.format(k))

    #
    # Put everything together
    #

    # Buttons above the table
    html = ''
    if not config.simple_output:
        # Copy Table Button
        html += """<div class="row">
                    <div class="col-sm-2">
        <button type="button" class="mqc_table_copy_btn btn btn-default btn-sm" data-clipboard-target="#{tid}">
            <span class="glyphicon glyphicon-copy"></span> Copy table
        </button>
        </div>
        """.format(tid=table_id)

        # <div class="progress-bar progress-bar-warning progress-bar-striped" style="width: 20%">
        #                 <span class="sr-only">20% Complete (warning)</span>
        #               </div>
        #               <div class="progress-bar progress-bar-danger" style="width: 10%">
        #                 <span class="sr-only">10% Complete (danger)</span>
        #               </div>

        if not hide_bar:
            html += """ <div class="col-sm-2">
                <div id="quartiles_bar">
                <div class="progress">
                  <div class="progress-bar progress-bar-q1" style="width: 25%">
                    <span class="sr-only">35% Complete (success)</span> Q1
                  </div>
    
                  <div class="progress-bar progress-bar-q2" style="width: 25%">
                    <span class="sr-only">35% Complete (success)</span> Q2
                  </div>
                
                  <div class="progress-bar progress-bar-q3" style="width: 25%">
                    <span class="sr-only">35% Complete (success)</span> Q3
                  </div>
    
                  <div class="progress-bar progress-bar-q4" style="width: 25%">
                    <span class="sr-only">35% Complete (success)</span> Q4
                  </div>
                </div>
                </div>
                         </div>
                         </div>
                """
        else:
            html += """ </div>
                """
        # Configure Columns Button
        # if len(t_headers) > 1:
        #     html += """
        #     <button type="button" class="mqc_table_configModal_btn btn btn-default btn-sm" data-toggle="modal" data-target="#{tid}_configModal">
        #         <span class="glyphicon glyphicon-th"></span> Configure Columns
        #     </button>
        #     """.format(tid=table_id)

        # Sort By Highlight button
        # html += """
        # <button type="button" class="mqc_table_sortHighlight btn btn-default btn-sm" data-target="#{tid}" data-direction="desc" style="display:none;">
        #     <span class="glyphicon glyphicon-sort-by-attributes-alt"></span> Sort by highlight
        # </button>
        # """.format(tid=table_id)

        # Scatter Plot Button
        # if len(t_headers) > 1:
        #     html += """
        #     <button type="button" class="mqc_table_makeScatter btn btn-default btn-sm" data-toggle="modal" data-target="#tableScatterModal" data-table="#{tid}">
        #         <span class="glyphicon glyphicon glyphicon-stats"></span> Plot
        #     </button>
        #     """.format(tid=table_id)

        # "Showing x of y columns" text
        row_visibilities = [all(t_rows_empty[s_name].values()) for s_name in t_rows_empty]
        visible_rows = [x for x in row_visibilities if not x]
        # html += """
        # <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
        # """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows), ncols_vis = (len(t_headers)+1)-hidden_cols, ncols=len(t_headers))

        # Add text
        # html += """
        #         <small id="{tid}_numrows_text" class="mqc_table_numrows_text">Showing <sup id="{tid}_numrows" class="mqc_table_numrows">{nvisrows}</sup>/<sub>{nrows}</sub> rows and <sup id="{tid}_numcols" class="mqc_table_numcols">{ncols_vis}</sup>/<sub>{ncols}</sub> columns.</small>
        #         """.format(tid=table_id, nvisrows=len(visible_rows), nrows=len(t_rows),
        #                    ncols_vis=(len(t_headers) + 1) - hidden_cols, ncols=len(t_headers))

    # Build the table itself
    collapse_class = 'mqc-table-collapse' if len(t_rows) > 10 and config.collapse_tables else ''
    html += """
        <div id="{tid}_container" class="mqc_table_container">
            <div class="table-responsive mqc-table-responsive {cc}">
                <table id="{tid}" class="table table-condensed mqc_table" data-title="{title}">
        """.format(tid=table_id, title=table_title, cc=collapse_class)

    # Build the header row
    col1_header = dt.pconfig.get('col1_header', 'Sample Name')
    html += '<thead><tr><th class="rowheader">{}</th>{}</tr></thead>'.format(col1_header, ''.join(t_headers.values()))

    # Build the table body
    html += '<tbody>'
    t_row_keys = t_rows.keys()
    if dt.pconfig.get('sortRows') is not False:
        t_row_keys = sorted(t_row_keys)
    for s_name in t_row_keys:
        # Hide the row if all cells are empty or hidden
        row_hidden = ' style="display:none"' if all(t_rows_empty[s_name].values()) else ''
        html += '<tr{}>'.format(row_hidden)
        # Sample name row header
        html += '<th class="rowheader" data-original-sn="{sn}">{sn}</th>'.format(sn=s_name)
        for k in t_headers:
            html += t_rows[s_name].get(k, empty_cells[k])
        html += '</tr>'
    html += '</tbody></table></div>'
    if len(t_rows) > 10 and config.collapse_tables:
        html += '<div class="mqc-table-expand"><span class="glyphicon glyphicon-chevron-down" aria-hidden="true"></span></div>'
    html += '</div>'

    # Build the bootstrap modal to customise columns and order
    # if not config.simple_output:
    #     html += """
    # <!-- MultiQC Table Columns Modal -->
    # <div class="modal fade" id="{tid}_configModal" tabindex="-1">
    #   <div class="modal-dialog modal-lg">
    #     <div class="modal-content">
    #       <div class="modal-header">
    #         <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
    #         <h4 class="modal-title">{title}: Columns</h4>
    #       </div>
    #       <div class="modal-body">
    #         <p>Uncheck the tick box to hide columns. Click and drag the handle on the left to change order.</p>
    #         <p>
    #             <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showAll">Show All</button>
    #             <button class="btn btn-default btn-sm mqc_configModal_bulkVisible" data-target="#{tid}" data-action="showNone">Show None</button>
    #         </p>
    #         <table class="table mqc_table mqc_sortable mqc_configModal_table" id="{tid}_configModal_table" data-title="{title}">
    #           <thead>
    #             <tr>
    #               <th class="sorthandle" style="text-align:center;">Sort</th>
    #               <th style="text-align:center;">Visible</th>
    #               <th>Group</th>
    #               <th>Column</th>
    #               <th>Description</th>
    #               <th>ID</th>
    #               <th>Scale</th>
    #             </tr>
    #           </thead>
    #           <tbody>
    #             {trows}
    #           </tbody>
    #         </table>
    #     </div>
    #     <div class="modal-footer"> <button type="button" class="btn btn-default" data-dismiss="modal">Close</button> </div>
    # </div> </div> </div>""".format( tid=table_id, title=table_title, trows=''.join(t_modal_headers.values()) )

    # Save the raw values to a file if requested
    if dt.pconfig.get('save_file') is True:
        fn = dt.pconfig.get('raw_data_fn', 'multiqc_{}'.format(table_id))
        util_functions.write_data_file(dt.raw_vals, fn)
        report.saved_raw_data[fn] = dt.raw_vals

    return html
Exemplo n.º 35
0
    def __new__(cls, name, bases, attrs):
        super_new = super(ResourceBase, cls).__new__
        parents = [b for b in bases if isinstance(b, ResourceBase)]
        if not parents:
            # If this isn't a subclass of RestObject, don't do anything
            # special.
            return super_new(cls, name, bases, attrs)

        attrs['__ordered__'] = [key for key in attrs.keys()
                                if key not in ('__module__', '__qualname__')]
        current_fields = []
        for key, value in attrs.items():
            if isinstance(value, Field):
                if getattr(value, 'attname', None) is None:
                    value.attname = key
                    value.name = key

                current_fields.append((key, value))

        for key,value in current_fields:
            attrs.pop(key)

        current_fields.sort(key=lambda x: x[1].creation_counter)
        attrs['declared_fields'] = OrderedDict(current_fields)

        new_class = super_new(cls, name, bases, attrs)

        # Create the meta class.
        attr_meta = attrs.pop('Meta', None)
        abstract = getattr(attr_meta, 'abstract', False)
        if not attr_meta:
            meta = getattr(new_class, 'Meta', None)
        else:
            meta = attr_meta

        module = attrs.pop('__module__')
        app_config = apps.get_containing_app_config(module)

        # TODO: Verify the purpose and use of this.
        # Wrapped in a condition to avoid throwing an
        # AttributeError when meta is None.
        if meta:
            meta.app_config = app_config

        if getattr(meta, 'app_label', None) is None:

            if app_config is None:

                model_module = sys.modules[new_class.__module__]
                package_components = model_module.__name__.split('.')
                # find the last occurrence of 'models'
                package_components.reverse()
                try:
                    app_label_index = package_components.index(MODELS_MODULE_NAME) + 1
                except ValueError:
                    #app_label_index = 1
                    app_label_index = 0
                try:
                    kwargs = {"app_label": package_components[app_label_index]}
                except IndexError:
                    raise ImproperlyConfigured(
                        'Unable to detect the app label for model "%s." '
                        'Ensure that its module, "%s", is located inside an '
                        'installed app.' % (
                            new_class.__name__, model_module.__name__)
                    )
            else:
                kwargs = {"app_label": app_config.label}

        else:
            kwargs = {}

        if not 'resource_name' in  meta.__dict__.copy():
            meta.resource_name = new_class.__name__

        opts = ResourceOptions(meta, **kwargs)
        new_class._meta = opts

        # Assign manager.
        manager = attrs.pop('objects', None)
        if manager is None:
            manager = ResourceManager()
        manager.object_class = new_class

        # Wrap default or custom managers such that it can only be used on
        # classes and not on instances.
        new_class.objects = ResourceManagerDescriptor(manager)

        # Walk through the MRO.
        declared_fields = OrderedDict()
        for base in reversed(new_class.__mro__):
            # Collect fields from base class.
            if hasattr(base, 'declared_fields'):
                declared_fields.update(base.declared_fields)

            # Field shadowing.
            for attr, value in base.__dict__.items():
                if value is None and attr in declared_fields:
                    declared_fields.pop(attr)

        primary_key = None
        for attr, value in declared_fields.items():
            setattr(new_class, attr, value)
            if value.primary_key:
                if primary_key is not None:
                    raise ImproperlyConfigured('Multiple primary keys.')
                else:
                    primary_key = value
                    opts._pk_attr = attr

        new_class.base_fields = declared_fields
        new_class.declared_fields = declared_fields
        new_class.DoesNotExist = RestServerException

        opts._fields = declared_fields
        opts.concrete_fields = declared_fields
        opts.concrete_model = new_class

        class State:
            db = opts.client
            adding = False

        new_class._state = State()

        if abstract:
            # Abstract resources are not registered, but subclasses should
            # still be non-abstract by default. Resetting the attribute to
            # False, following the behavior in Django models.
            attr_meta.abstract = False
            new_class.Meta = attr_meta
        elif hasattr(opts, 'resources'):
            # If resource registry was provided, use it to register new_class
            opts.resources.register(new_class._meta.app_label, new_class)

        return new_class
Exemplo n.º 36
0
class ContextTree:
    '''
    Code based on work by Mi et al., Context Tree for Adaptive Session-based Recommendation, 2018.

    Parameters
    --------
    history_maxlen: max considered context length

    nb_candidates (only used for adaptive configuration): the number of recent candidates considered for adaptive configuration

    expert: type of expert for each context
    '''
    def __init__(self,
                 history_maxlen=50,
                 nb_candidates=1000,
                 expert='StdExpert',
                 session_key='SessionId',
                 item_key='ItemId',
                 time_key='Time'):
        self.history_maxlen = history_maxlen
        self.nb_candidates = nb_candidates
        self.expert = eval(expert)
        self.item_key = item_key
        self.session_key = session_key
        self.time_key = time_key

        self.root = TreeRoot(self.expert)

        self.histories = History(history_maxlen)

        self.recent_candidates = OrderedDict()

        self.user_to_previous_recoms = {}

    def fit(self, train, items=None):
        '''
        fit training data for static evalution

        Parameters
        --------
        data: pandas.DataFrame
            Training data. It contains the transactions of the sessions. It has one column for session IDs, one for item IDs and one for the timestamp of the events (unix timestamps).
            It must have a header. Column names are arbitrary, but must correspond to the ones you set during the initialization of the network (session_key, item_key, time_key properties).

        '''
        start_time = time.time()
        for index, row in train.iterrows():
            self.fit_one_row(row, True)
            if index % 1000000 == 0:
                print(index, '---- Time: ', time.time() - start_time)

    def fit_one_row(self, row, first_in_session):
        '''
        fit one row for static setting
        '''

        current_session = row[self.session_key]
        current_item = row[self.item_key]

        history = self.histories.get_history(current_session)

        self.root.update(current_item, history)

        history.appendleft(current_item)

        self.root.expand(history)

    def fit_time_order_online(self, row, first_in_session):
        '''
        fit one row for adpative configuration, we data is ordered by time
        nb_candidates is used to keep a pool of recent candidates
        '''

        current_session = row[self.session_key]
        current_item = row[self.item_key]

        history = self.histories.get_history(current_session)

        self.root.update(current_item, history)

        history.appendleft(current_item)

        best_item_and_probas = self.root.get_n_most_probable(
            self.recent_candidates.keys(), history)
        predictions = [proba for rec, proba in best_item_and_probas]
        series = pd.Series(
            data=predictions,
            index=[int(rec) for rec, proba in best_item_and_probas])

        if not first_in_session:
            predictions = preprocessing.normalize([predictions])
            series = pd.Series(
                data=predictions[0],
                index=[int(rec) for rec, proba in best_item_and_probas])

        self.user_to_previous_recoms[current_session] = series

        self.root.expand(history)

        self.recent_candidates.pop(current_item, None)
        self.recent_candidates[current_item] = True
        if len(self.recent_candidates) > self.nb_candidates:
            self.recent_candidates.popitem(last=False)

    def match_context(self, row, items_to_predict, normalize):
        '''
        only used in static evaluation
        update the recommendation given next time with current row as input
        the model (CT) is not updated in static evalution for a fair comparison against other methods
        '''

        current_session = row[self.session_key]
        current_item = row[self.item_key]

        history = self.histories.get_history(current_session)

        history.appendleft(current_item)

        best_item_and_probas = self.root.get_n_most_probable(
            items_to_predict, history)
        predictions = [proba for rec, proba in best_item_and_probas]
        if normalize:
            predictions = preprocessing.normalize([predictions])
            #predictions = [i / sum(predictions) for i in predictions]
            series = pd.Series(
                data=predictions[0],
                index=[int(rec) for rec, proba in best_item_and_probas])
        else:
            series = pd.Series(
                data=predictions,
                index=[int(rec) for rec, proba in best_item_and_probas])
        self.user_to_previous_recoms[current_session] = series

    def predict_next(self,
                     session_id,
                     input_item_id,
                     predict_for_item_ids,
                     timestamp=0,
                     skip=False,
                     mode_type="view"):
        # print(input_item_id)
        '''
        Gives predicton scores for a selected set of items on how likely they be the next item in the session.

        Parameters
        --------
        session_id : int or string
            The session IDs of the event.
        input_item_id : int or string
            The item ID of the event. Must be in the set of item IDs of the training set.
        predict_for_item_ids : 1D array
            IDs of items for which the network should give prediction scores. Every ID must be in the set of item IDs of the training set.

        Returns
        --------
        out : pandas.Series
            Prediction scores for selected items on how likely to be the next item of this session. Indexed by the item IDs.

        '''

        row = {self.item_key: input_item_id, self.session_key: session_id}
        self.match_context(row, predict_for_item_ids, False)

        previous_recoms = self.user_to_previous_recoms.get(session_id)
        #print(previous_recoms[:5])

        return previous_recoms

    def clear(self):

        del self.histories
        del self.recent_candidates
        del self.user_to_previous_recoms

    def support_users(self):
        '''
          whether it is a session-based or session-aware algorithm
          (if return True, method "predict_with_training_data" must be defined as well)

          Parameters
          --------

          Returns
          --------
          True : if it is session-aware
          False : if it is session-based
        '''
        return False
Exemplo n.º 37
0
    def get_tabulation_data(self, table_code):
        endpoint = settings.API_URL + '/1.0/tabulation/%s' % table_code
        r = r_session.get(endpoint)
        status_code = r.status_code

        # make sure we've requested a legit tabulation code
        if status_code == 200:
            tabulation_data = r.json(object_pairs_hook=OrderedDict)
        elif status_code == 404 or status_code == 400:
            error_data = r.json()
            raise_404_with_messages(self.request, error_data)
        else:
            raise Http404

        # factories for organizing metadata on arbitrary sets of tables
        def table_dict_factory():
            return {
                'version_name': 'Standard Table',
                'releases': {
                    'one_yr': '',
                    'three_yr': '',
                    'five_yr': '',
                },
            }

        def table_ordereddict_factory():
            return OrderedDict()

        def table_expanded_factory():
            return {'table_metadata': self.get_table_data(table_code)}

        tables = OrderedDict()
        table_grid = OrderedDict()
        tables_expanded = OrderedDict()
        default_table = defaultdict(table_dict_factory)
        default_table_groups = defaultdict(table_ordereddict_factory)
        default_table_list = defaultdict(table_ordereddict_factory)
        default_expanded_list = defaultdict(table_ordereddict_factory)
        default_expanded_table = defaultdict(table_expanded_factory)

        # take API data and shape into dicts for:
        # * a grid with each table variant and which releases it's available for
        # * a list of each primary table, with its column metadata from the API
        for release in tabulation_data['tables_by_release']:
            # Sort data by length to have all the PR tables at the end.
            # Note that sorted() is guaranteed to be stable, per Python docs,
            # meaning that keys that compare equal (are equal length) will
            # remain in the same relative order. Assuming they were alphabetical
            # before this, this is guaranteed to list tables as
            # tableA, ... , tableI, tableAPR, ... , tableIPR.
            sorted_data = sorted(tabulation_data['tables_by_release'][release],
                                 key=lambda code: len(code))
            for table_code in sorted_data:
                # is this a B or C table?
                parsed = parse_table_id(table_code)
                letter_code = parsed['table_type']
                tables[letter_code] = default_table_groups[letter_code]

                # keep the grids separate, track which releases a table is in
                tables[letter_code] = default_table_list[letter_code]
                tables[letter_code][table_code] = default_table[table_code]
                tables[letter_code][table_code]['releases'][
                    release] = self.RELEASE_TRANSLATE_DICT[release]

                # get the variant names
                if parsed['racial']:
                    if parsed['puerto_rico']:
                        tables[letter_code][table_code][
                            'version_name'] = parsed['race'] + " (Puerto Rico)"
                    else:
                        tables[letter_code][table_code][
                            'version_name'] = parsed['race']
                elif parsed['puerto_rico']:
                    tables[letter_code][table_code][
                        'version_name'] = "Standard Table (Puerto Rico)"

        tabulation_data['table_versions'] = tables.pop(self.table_group, None)
        tabulation_data['related_tables'] = {
            'grid': tables,
            'preview': {},
        }

        for group, group_values in tables.iteritems():
            preview_table = next(group_values.iteritems())[0]
            try:
                tabulation_data['related_tables']['preview'][
                    preview_table] = self.get_table_data(preview_table)
                tabulation_data['related_tables']['preview'][preview_table][
                    'table_type'] = self.TABLE_TYPE_TRANSLATE_DICT[
                        preview_table.upper()[0]]
            except ValueError:
                continue

        return tabulation_data
Exemplo n.º 38
0
class NamedNodeMap(UserDict):
    """Collection of Attr objects."""
    def __init__(self, owner: Node) -> None:
        """Initialize with owner node.

        :arg Node owner: owner node of this object.
        """
        self._owner = owner
        self._dict = OrderedDict()  # type: OrderedDict[str, Attr]

    def __len__(self) -> int:
        return len(self._dict)

    def __contains__(self, item: object) -> bool:
        return item in self._dict

    def __getitem__(self, index: Union[int, str]) -> Optional[Attr]:
        if isinstance(index, int):
            return tuple(self._dict.values())[index]
        return None

    def __setitem__(self, attr: str, item: Attr) -> None:
        self._dict[attr] = item

    def __delitem__(self, attr: str) -> None:
        del self._dict[attr]

    def __iter__(self) -> Iterator[str]:
        for attr in self._dict.keys():
            yield attr

    @property
    def length(self) -> int:
        """Return number of Attrs in this collection."""
        return len(self)

    def getNamedItem(self, name: str) -> Optional[Attr]:
        """Get ``Attr`` object which has ``name``.

        If does not have ``name`` attr, return None.
        """
        return self._dict.get(name, None)

    def setNamedItem(self, item: Attr) -> None:
        """Set ``Attr`` object in this collection."""
        from wdom.web_node import WdomElement
        if not isinstance(item, Attr):
            raise TypeError('item must be an instance of Attr')
        if isinstance(self._owner, WdomElement):
            self._owner.js_exec(
                'setAttribute',
                item.name,  # type: ignore
                item.value)
        self._dict[item.name] = item
        item._owner = self._owner

    def removeNamedItem(self, item: Attr) -> Optional[Attr]:
        """Set ``Attr`` object and return it (if exists)."""
        from wdom.web_node import WdomElement
        if not isinstance(item, Attr):
            raise TypeError('item must be an instance of Attr')
        if isinstance(self._owner, WdomElement):
            self._owner.js_exec('removeAttribute', item.name)
        removed_item = self._dict.pop(item.name, None)
        if removed_item:
            removed_item._owner = self._owner
        return removed_item

    def item(self, index: int) -> Optional[Attr]:
        """Return ``index``-th attr node."""
        if 0 <= index < len(self):
            return self._dict[tuple(self._dict.keys())[index]]
        return None

    def toString(self) -> str:
        """Return string representation of collections."""
        return ' '.join(attr.html for attr in self._dict.values())
Exemplo n.º 39
0
class TVDepsGraph(object):
    cachedir = None
    cache_indices = None

    def __init__(self, state, clone=False):
        self.tasks = OrderedDict()
        self.data = OrderedDict()
        self.resources = OrderedDict()
        self.allocators = OrderedDict()
        self._evaluated = False
        self.state = state

        if clone: return

        from .TVAction import TVAction, TVTask
        from .TVArray import TVArrayBase, TVArray, TVBaseShard, TVReadShard, TVArraySeries

        #1. Initial build of depsgraph from parent register
        for a in self.state.tasks:
            task = TVDepsTask(a)
            self.tasks[len(self.tasks) + 1] = task
        for a in self.state.data:
            assert isinstance(a, TVArray)

            aname = a.name()
            d = TVDepsData(a)
            assert aname not in self.data, namestr(aname)
            self.data[aname] = d
            a._cache.graph = self
            if isinstance(a, TVBaseShard): continue

            aname2 = aname
            if a._parent is not None and aname[-1] is None:
                aname2 = aname[0], 0
            assert aname2 not in self.resources, namestr(aname2)
            r = TVDepsResource(a)
            self.resources[aname2] = r
        for a in self.state.data_containers:
            assert isinstance(a, TVArraySeries), type(a)
            r = TVDepsResource(a)
            aname = (a.name()[0], None)
            assert aname not in self.resources, namestr(aname)
            self.resources[aname] = r

        outputs = set()
        output_names = set()

        for tasknr, task in self.tasks.items():
            a = task.tvtask()
            for arg, argname in zip(a._inputs, a._inputs_names):
                if isinstance(arg, TVArray):
                    assert arg.name() in self.data, namestr(arg.name())
                    task.data.append(arg.name())
            if a._accumulate:
                for arg in a._outputs:
                    prev = arg._previous()
                    assert prev.name() in self.data, namestr(prev)
                    task.data.append(prev.name())
            for arg, argname in zip(a._outputs, range(1, len(a._outputs) + 1)):
                assert isinstance(arg, TVArray), (str(a), arg)
                assert arg.name() in self.data, namestr(arg.name())
                assert arg.name() not in output_names, namestr(arg.name())
                assert id(arg) not in outputs, namestr(arg.name())
                outputs.add(id(arg))
                output_names.add(arg.name())
                task.resources.append(arg.name()[:2])
                self.data[arg.name()].task = tasknr
        r = TVDepsResource(None)
        self.resources["@return"] = r
        for v in self.state.return_:
            mode, index = v
            assert mode == "data"
            a = self.state.data[index]
            if a._parent is not None and isinstance(a._parent, TVArraySeries):
                a = a._parent._children[-1]
            assert a.name() in self.data, namestr(a)
            r.data.append(a.name())

        # 2: remove unassigned data
        for name0 in list(self.data.keys()):
            ok = False
            for name in list(self.data.keys()):
                if name[:2] != name0[:2]: continue
                if self.data[name].task is not None:
                    ok = True
                if not ok:
                    for r in self.resources.values():
                        rdata = [d for d in r.data]
                        if name in rdata:
                            ok = True
                            break
                if not ok:
                    for r in self.tasks.values():
                        rdata = [d for d in r.data]
                        if name in rdata:
                            ok = True
                            break
                if ok: break
            if not ok:
                self.data.pop(name0)

        # 3: check that every data has dtype and shape
        for d in self.data:
            r = self.data[d]
            tvdata = r.tvdata()
            if tvdata is None: continue
            assert tvdata.dtype is not None, d
            assert tvdata.shape is not None, d

        # 4: check consistency within containers
        for c in self.state.data_containers:
            assert isinstance(c, TVArraySeries)
            firstchild = c._children[0]
            for child in c._children[1:]:
                assert child.dtype == firstchild.dtype, (namestr(c.name()),
                                                         child.dtype,
                                                         firstchild.dtype)
                assert len(child.shape) == len(firstchild.shape), (namestr(
                    c.name()), child.shape, firstchild.shape)

        # 5: rewrite the depsgraph based on handlers
        for rname, r in list(self.resources.items()):
            if rname not in self.resources:
                continue  #if we were deleted by a previous handler
            if self.resources[rname] is not r:
                continue  #if we were replaced by a previous handler
            tvdata = r.tvdata()
            if tvdata is None: continue
            if tvdata._parent is not None: continue
            assert isinstance(tvdata, TVArraySeries), r
            exceptions = []
            for handler in _handlers:
                try:
                    handler(self, rname)
                    break
                except:
                    raise  ###
                    e = traceback.format_exc()
                    exceptions.append(e)
            else:
                for e, handler in zip(exceptions, _handlers):
                    print handler.__name__
                    print e
                    print
                raise Exception("All handlers failed for resource '%s'" %
                                rname)

        # 6: add dependencies for shards, remove writeshards that point to eliminated data:
        for dname, d in list(self.data.items()):
            dname2 = dname[:2]
            if isinstance(d.tvdata(), TVWriteShard):
                if dname2 not in self.data:
                    self.data.pop(dname)
                    continue
                assert dname2 in self.data, dname2
                resname = (namestr(dname2) + "::join", None)
                if resname not in self.resources:
                    self.resources[resname] = TVDepsResource(None)
                    r = self.data[dname2].resource
                    if r is None:
                        self.data[dname2].resource = resname
                    else:
                        self.resources[r].resources.append(resname)
                    if d.tvdata()._parent._cache_active:
                        cache = TVAllocator("cache", tvname=dname2)
                        cachename = (namestr(dname2) + "::cache", None)
                        self.allocators[cachename] = cache
                        self.resources[resname].allocators.append(cachename)
                self.resources[resname].data.append(dname)
            elif isinstance(d.tvdata(), TVReadShard):
                assert d.resource is None, (dname, d.resource)
                resname = (namestr(dname2) + "::split", None)
                if resname not in self.resources:
                    self.resources[resname] = TVDepsResource(None)
                    self.resources[resname].data.append(dname2)
                d.resource = resname

        # 7: re-check that every data has dtype and shape (null data not allowed)
        for d in self.data:
            r = self.data[d]
            assert isinstance(r, TVDepsData), d
            tvdata = r.tvdata()
            assert tvdata is not None, d
            assert tvdata.dtype is not None, d
            assert tvdata.shape is not None, d

        # 8: calculate signatures for every task
        for task in self.tasks.values():
            t = task.tvtask()
            t.calc_signature()

        # 9: build hashtree for non cache-active data (such as those computed on the GPU)
        for d in self.data:
            r = self.data[d]
            data = r.tvdata()
            if isinstance(data, TVReadShard): continue
            if not data._cache_active and (not hasattr(data, "_hash")
                                           or data._hash is None):
                if r.hashtree is None:
                    r.hashtree = make_hashtree(d, self)

    def prune(self):
        """
    Eliminates all tasks with a cachekey, and everything that does not depend on return
    """
        #TODO: will run out of stack space for large depsgraphs
        tracing_data = set()
        tracing_resources = set()
        tracing_tasks = set()
        dep_data = set()
        dep_resources = set()
        dep_tasks = set()
        dep_allocators = set()

        def trace_resource(name):
            if name in dep_resources:
                return
            r = self.resources[name]
            tracing_resources.add(name)
            for d in r.resources:
                if d in tracing_resources:
                    raise ValueError(
                        "Resource '%s' and resource '%s' are in a cyclical dependency"
                        % (name, d))
                trace_resource(d)
            for d in r.data:
                if d in tracing_resources:
                    raise ValueError(
                        "Resource '%s' and data '%s' are in a cyclical dependency"
                        % (name, d))
                trace_data(d)
            for d in r.allocators:
                ok = True
                if self.cachedir is None:
                    alloc = self.allocators[d]
                    if alloc.alloc_command == "cache":
                        ok = False
                if ok:
                    dep_allocators.add(d)
            dep_resources.add(name)
            tracing_resources.remove(name)

        def trace_data(name):
            if name in dep_data:
                return
            r = self.data[name]
            if r.tvdata() is not None and r.tvdata()._cache.cachekey:
                dep_data.add(name)
                for t in self.tasks:
                    task = self.tasks[t]
                    if name in task.data:
                        task._cachekeys.append(r.tvdata())
                return
            tracing_data.add(name)
            d = r.task
            if d is not None:
                if d in tracing_tasks:
                    raise ValueError(
                        "Data '%s' and task %d are in a cyclical dependency" %
                        (namestr(name), d))
                trace_task(d)
            d = r.resource
            if d is not None:
                if d in tracing_resources:
                    raise ValueError(
                        "Data '%s' and resource '%s' are in a cyclical dependency"
                        % (namestr(name), d))
                trace_resource(d)
            dep_data.add(name)
            tracing_data.remove(name)

        def trace_task(name):
            if name in dep_tasks:
                return
            r = self.tasks[name]
            tracing_tasks.add(name)
            for d in r.resources:
                if d in tracing_resources:
                    raise ValueError(
                        "Task %d and resource '%s' are in a cyclical dependency"
                        % (namestr(name), d))
                trace_resource(d)
            for d in r.data:
                if d in tracing_resources:
                    raise ValueError(
                        "Task %d and data '%s' are in a cyclical dependency" %
                        (namestr(name), d))
                trace_data(d)
            dep_tasks.add(name)
            tracing_tasks.remove(name)

        trace_resource("@return")
        for d in list(self.data.keys()):
            if d not in dep_data:
                self.data.pop(d)
        for d in list(self.resources.keys()):
            if d == "@return": continue
            if d not in dep_resources:
                self.resources.pop(d)
        for d in list(self.tasks.keys()):
            if d not in dep_tasks:
                self.tasks.pop(d)
        for d in list(self.allocators.keys()):
            if d not in dep_allocators:
                self.allocators.pop(d)

        retdep = self.resources["@return"].data
        progress = True
        while progress:
            progress = False
            for d in list(self.data.keys()):
                r = self.data[d]
                r.cleanup(self)
                if d in retdep: continue
                if r.empty():
                    progress = True
                    self.data.pop(d)
            for d in list(self.resources.keys()):
                if d == "@return": continue
                r = self.resources[d]
                r.cleanup(self)
                if r.empty():
                    progress = True
                    for a in r.allocators:
                        self.allocators.pop(a)
                    self.resources.pop(d)
            for d in list(self.tasks.keys()):
                r = self.tasks[d]
                r.cleanup(self)
                if r.empty():
                    progress = True
                    self.tasks.pop(d)
        #print "PRUNE", len(self.tasks)

    def __str__(self):
        s = "@return:\n"
        namestrs = [namestr(dd) for dd in self.resources["@return"].data]
        s += "  " + " ".join(namestrs) + "\n"
        s += "Tasks:\n"
        for tnr, t in self.tasks.items():
            s += "  " + str(tnr) + ":\n"
            s += str(t)
        s += "Data:\n"
        for dname in sorted(list(self.data.keys())):
            d = self.data[dname]
            s += "  " + namestr(dname) + ":\n"
            s += str(d)
        s += "Resources:\n"
        for rname in sorted(list(self.resources.keys())):
            r = self.resources[rname]
            if rname == "@return": continue
            s += "  " + namestr(rname) + ":\n"
            s += str(r)
        s += "Allocators:\n"
        for aname in sorted(list(self.allocators.keys())):
            a = self.allocators[aname]
            s += "  " + namestr(aname) + ":\n"
            s += str(a)
        return s

    def clone(self):
        newstate = self.state.clone()
        clone = TVDepsGraph(newstate, clone=True)
        for k, v in self.tasks.items():
            d = v.dict(self.state)
            clone.tasks[k] = TVDepsTask.fromdict(d, clone.state)
        for k, v in self.data.items():
            d = v.dict(self.state)
            clone.data[k] = TVDepsData.fromdict(d, clone.state)
        for k, v in self.resources.items():
            d = v.dict(self.state)
            clone.resources[k] = TVDepsResource.fromdict(d, clone.state)
        for k, v in self.allocators.items():
            clone.allocators[k] = v.clone()
        for a in newstate.data:
            a._cache.graph = clone
        if self.cachedir is not None:
            clone.cachedir = self.cachedir
            clone.cache_indices = self.cache_indices
        return clone

    def define_cache(self, cachedir):
        from .TVCache import CacheIndex, cache_indices as global_cache_indices
        if cachedir is None:
            self.cachedir = None
            self.cache_indices = None
            return
        self.cachedir = cachedir
        if not os.path.exists(self.cachedir):
            os.mkdir(self.cachedir)
        for d in self.data.values():
            data = d.tvdata()
            if data._cache is not None:
                data._cache.searched = False
        cache_indices = {}
        for action in ("::shard", "::join") + tuple(self.state.actions):
            k = (self.cachedir, action)
            if k not in global_cache_indices:
                dir = k[0] + os.sep + k[1]
                c = CacheIndex(dir)
                global_cache_indices[k] = c
            else:
                c = global_cache_indices[k]
            cache_indices[action] = c
        self.cache_indices = cache_indices

    def search_cache_hashes(self, allocator):
        from .TVArray import TVBaseShard, TVReadShard, TVWriteShard
        if self.cachedir is None: return
        global progress

        done = set()
        progress = True

        def search_cache_hash(dname):
            global progress
            if dname in done: return
            d = self.data[dname]
            data = d.tvdata()
            shards, shards2 = [], []
            if isinstance(data, TVBaseShard):
                dname2 = dname[:2]
                if isinstance(data, TVReadShard):
                    if dname2 not in done: return
                    if data._parent._cache_active:
                        data.search_cache_hash()
                    done.add(dname)
                    progress = True
                    return
            elif data.get_hash() is not None:
                #print "HASH!", dname
                done.add(dname)
                progress = True
                return
            else:
                for sh in self.data:
                    if len(sh) < 3: continue
                    if sh[:2] != dname: continue
                    shards.append(sh)
                shards2 = [
                    sh for sh in shards
                    if isinstance(self.data[sh].tvdata(), TVWriteShard)
                ]
                if d.task is None and not len(shards2):
                    #print "UNUSED!", dname
                    done.add(dname)
                    progress = True
                    return
            if len(shards2):
                #print "SEARCHIN?", dname
                deps = shards2
            else:
                task = self.tasks[d.task]
                #print "SEARCHING", dname, str(task).split("\n")[0]
                deps = task.data
            if not d.tvdata()._cache_active:
                ok = True
                for dep in deps:
                    search_cache_hash(dep)
                    if dep not in done:
                        ok = False
                        break
                if ok:
                    done.add(dname)
                    progress = True
                return
            for dep in deps:
                dd = self.data[dep]
                ddd = dd.tvdata()
                if isinstance(ddd, TVBaseShard):
                    if dep not in done: return
                elif ddd.get_hash() is None:
                    ok = False
                    if not ddd._cache_active:
                        if dd.hashtree.bindable(self):
                            ok = True
                    if not ok:
                        return
            #print "SEARCH", dname
            if len(shards2):
                data._cache.search_join()
            else:
                data._cache.search()
            if data._cache.arrayhash is not None:
                #print "HASH2!", dname
                data.set_hash(data._cache.arrayhash)
                done.add(dname)
                progress = True
            #else:
            #  print "MISSED", task

        while progress:
            progress = False
            for d in self.data:
                search_cache_hash(d)

    def evaluate(self, cls_evaluator, args, kwargs):
        from . import get_mode, set_mode
        #TODO: check that all previous stages are complete
        assert not self._evaluated
        self.state.bind_args(args, kwargs)

        evaluator = cls_evaluator(self)
        self.data_unpruned = self.data.copy()

        oldmode = get_mode()
        set_mode("search_cache")
        self.search_cache_hashes(evaluator.allocator)

        #prune depsgraph based on cachekeys
        self.prune()
        log.set_plan(str(self))

        try:
            set_mode("evaluate")
            evaluator.init()
            evaluator.evaluate_resource("@return")
            evaluator.run()
            r = self.resources["@return"]
            ret = []
            for d in r.data:
                dat = self.data[d]
                ret.append(dat.tvdata().get_data())
        finally:
            set_mode(oldmode)
            for d in self.data.values():
                dd = d.tvdata()
                if dd is not None:
                    try:
                        dd.set_data(None)
                    except AttributeError:
                        pass
        self._evaluated = True
        self.cache_indices = None
        ret = tuple(ret)
        if len(ret) == 1:
            ret = ret[0]
        return ret
Exemplo n.º 40
0
    def __sub__(self, other):
        """
        Allows to remove common fields of two schema.Struct from self by
        using '-' operator. If two Struct have common field names, the
        removal is conducted recursively. If a child struct has no fields
        inside, it will be removed from its parent. Here are examples:

        Example 1
        s1 = Struct(
            ('a', Scalar()),
            ('b', Scalar()),
        )
        s2 = Struct(('a', Scalar()))
        s1 - s2 == Struct(('b', Scalar()))

        Example 2
        s1 = Struct(
            ('b', Struct(
                ('c', Scalar()),
                ('d', Scalar()),
            ))
        )
        s2 = Struct(
            ('b', Struct(('c', Scalar()))),
        )
        s1 - s2 == Struct(
            ('b', Struct(
                ('d', Scalar()),
            )),
        )

        Example 3
        s1 = Struct(
            ('a', Scalar()),
            ('b', Struct(
                ('d', Scalar()),
            ))
        )
        s2 = Struct(
            ('b', Struct(
                ('c', Scalar())
                ('d', Scalar())
            )),
        )
        s1 - s2 == Struct(
            ('a', Scalar()),
        )
        """
        if not isinstance(other, Struct):
            return NotImplemented

        children = OrderedDict(self.get_children())
        for name, right_field in other.get_children():
            if name in children:
                left_field = children[name]
                if type(left_field) == type(right_field):
                    if isinstance(left_field, Struct):
                        child = left_field - right_field
                        if child.get_children():
                            children[name] = child
                            continue
                    children.pop(name)
                else:
                    raise TypeError(
                        "Type of left_field, " + str(type(left_field)) +
                        ", is not the same as that of right_field, " +
                        str(type(right_field)) +
                        ", yet they have the same field name, " + name)
        return Struct(*(children.items()))
Exemplo n.º 41
0
    def _extract_metadata(self):
        """
        Reads the metadata in the header

        Returns
        -------
        meas_parms : OrderedDict
            Ordered dictionary of Ordered dictionaries (one per image / force channel, etc.)
        other_parms : OrderedDict
            Ordered Dictionary of Ordered dictionaries containing all other metadata
        """
        other_parms = OrderedDict()
        meas_parms = OrderedDict()
        curr_category = ''
        temp_dict = OrderedDict()
        with open(self.file_path, "rb") as file_handle:
            for ind, line in enumerate(file_handle):
                line = line.decode("utf-8", 'ignore')
                trimmed = line.strip().replace("\\", "").replace('@', '')
                split_data = trimmed.split(':')

                # First account for wierdly formatted metadata that
                if len(split_data) == 3:
                    split_data = [
                        split_data[1] + '_' + split_data[0], split_data[-1]
                    ]
                elif len(split_data) > 3:
                    # Date:
                    split_ind = trimmed.index(':')
                    split_data = [trimmed[:split_ind], trimmed[split_ind + 1:]]

                # At this point, split_data should only contain either 1 (class header) or 2 elements
                if len(split_data) == 1:
                    if len(temp_dict) > 0:
                        if 'Ciao image list' in curr_category or 'Ciao force image list' in curr_category:
                            # In certain cases the same class name occurs multiple times.
                            # Append suffix to existing name and to this name
                            count = 0
                            for class_name in meas_parms.keys():
                                if curr_category in class_name:
                                    count += 1
                            if count == 0:
                                meas_parms[curr_category] = temp_dict.copy()
                            else:
                                if count == 1:
                                    for class_name in meas_parms.keys():
                                        if curr_category == class_name:
                                            # Remove and add back again with suffix
                                            # This should only ever happen once.
                                            # The next time we come across the same class, all elements already have
                                            # suffixes
                                            meas_parms[curr_category +
                                                       '_0'] = meas_parms.pop(
                                                           curr_category)
                                            break
                                meas_parms[curr_category + '_' +
                                           str(count)] = temp_dict.copy()
                        else:
                            curr_category = curr_category.replace('Ciao ', '')
                            other_parms[curr_category] = temp_dict.copy()

                    if "*File list end" in trimmed:
                        break
                    curr_category = split_data[0].replace('*', '')
                    temp_dict = OrderedDict()
                elif len(split_data) == 2:
                    split_data = [item.strip() for item in split_data]
                    try:
                        num_val = float(split_data[1])
                        if int(num_val) == num_val:
                            num_val = int(num_val)
                        temp_dict[split_data[0]] = num_val
                    except ValueError:
                        temp_dict[split_data[0]] = split_data[1]
                else:
                    print(split_data)

        return meas_parms, other_parms
Exemplo n.º 42
0
class CBindings(object):
    def __init__(self, hfiles, includes = [], include_dirs = [], predefs = {}, prelude = [], debug = False):
        self.hfiles = hfiles
        self.types = OrderedDict()
        self.funcs = OrderedDict()
        self.macros = OrderedDict()
        self.macros.update(predefs)
        self.include_dirs = include_dirs
        
        lines = []
        self._included = set()
        for hfile in hfiles:
            lines.extend(self._load_with_includes(hfile, includes))
        lines[0:0] = prelude
        text = self._preprocess(lines)
        if debug:
            with open("tmp.c", "w") as f:
                f.write(text)
        
        self.packed_lines = set(self._get_packed_regions(text))
        
        parser = CParser()
        ast = parser.parse(text, "tmp.c")
        visitor = CtypesGenVisitor(self)
        visitor.visit(ast)
    
    def _get_packed_regions(self, text):
        lines = text.splitlines()
        in_packed = False
        for i, l in enumerate(lines, 1):
            l = l.strip().lower()
            if not l.startswith("#pragma") or "pack" not in l:
                if in_packed:
                    yield i
                continue
            
            if "push" in l:
                if in_packed:
                    raise ValueError("Detected #pragma pack without a matching pop")
                in_packed = True
            elif "pop" in l:
                if not in_packed:
                    raise ValueError("Detected #pragma pop without a matching pack")
                in_packed = False
            else:
                raise ValueError("Unknown #pragma pack option %r" % (l,))
        if in_packed:
            raise ValueError("Detected #pragma pack without a matching pop")
    
    @classmethod
    def _find_inc(cls, name, include_dirs):
        for path in include_dirs:
            fn = os.path.join(path, name)
            if os.path.isfile(fn):
                return fn
        raise Exception("Cannot open %r - not found in %r" % (name, include_dirs))
    
    def _load_with_includes(self, rootfile, includes):
        include_dirs = [os.path.dirname(os.path.abspath(rootfile))] + self.include_dirs 
        lines = strip_comments(open(rootfile, "rU").read()+"\n\n\n").splitlines()
        i = -1
        while i + 1 < len(lines):
            i += 1
            line = lines[i]
            m = include_pattern.match(line)
            if not m:
                continue
            del lines[i]
            i -= 1
            filename = m.group(1)
            if filename not in includes or filename in self._included:
                continue
            self._included.add(filename)
            fullname = self._find_inc(filename, include_dirs)
            lines2 = strip_comments(open(fullname, "rU").read()+"\n\n\n").splitlines()
            lines[i+1:i+1] = lines2
        return [l.strip() for l in lines if l.strip()]
    
    def _preprocess(self, lines):
        lines2 = []
        for line in lines:
            m = define_pattern.match(line)
            if m:
                name, args, body = m.groups()
                self.macros[name] = ([a.strip() for a in args.split(",")] if args else (), body)
                continue
            m = undef_pattern.match(line)
            if m:
                name = m.group(1)
                self.macros.pop(name, None)
                continue
            if ifdef_pattern.match(line) or ifndef_pattern.match(line) or endif_pattern.match(line):
                continue
            if if_pattern.match(line) or elif_pattern.match(line) or else_pattern.match(line):
                continue
            
            replaced = True
            while replaced:
                replaced = False
                tokens = delimiters.split(line)
                for name, (args, body) in self.macros.items():
                    while name in tokens:
                        replaced = True
                        i = tokens.index(name)
                        tokens[i:i+1] = body
                line = "".join(tokens).strip()
            lines2.append(line)
        return "\n".join(lines2)
    
    def export(self, filename):
        m = PythonModule()
        m.comment("Auto-generated file; do not edit directly")
        m.comment(time.ctime())
        m.comment("Generated from %s" % (", ".join(self.hfiles,)))
        m.sep()
        m.import_("sys")
        m.import_("ctypes")
        self.emit_prelude(m)
        m.sep()
        generated_macros = set()
        for name, (args, value) in self.macros.items():
            if not self.filter_macro(name, args, value):
                continue
            self.emit_macro(m, name, args, value, generated_macros)
        m.sep()
        self.emit_get_calling_conv(m)
        m.sep()
        for t in self.types.values():
            if not self.filter_type(t):
                continue
            if isinstance(t, CEnum):
                self.emit_enum(m, t)
            elif isinstance(t, CStruct):
                self.emit_struct_decl(m, t)
            elif isinstance(t, CUnion):
                self.emit_union_decl(m, t)
            elif isinstance(t, CTypedef):
                pass
            else:
                raise TypeError(t)
        m.sep()
        for t in self.types.values():
            if not self.filter_type(t):
                continue
            if isinstance(t, CTypedef):
                self.emit_typedef(m, t)
        m.sep()
        for t in self.types.values():
            if not self.filter_type(t):
                continue
            if isinstance(t, CStruct):
                self.emit_struct_fields(m, t)
            elif isinstance(t, CUnion):
                self.emit_union_fields(m, t)
        m.sep()
        m.stmt("_the_dll = UnloadedDLL")
        for f in self.funcs.values():
            if self.filter_func(f):
                self.emit_func_unloaded(m, f)
        m.sep()
        with m.def_("load_dll", "dllname"):
            m.stmt("global _the_dll")
            with m.if_("_the_dll"):
                m.raise_("ValueError('DLL already loaded')")
            m.stmt("dll = ctypes.CDLL(dllname)")
            m.sep()
            for f in self.funcs.values():
                if not self.filter_func(f):
                    continue
                self.emit_func_prototype(m, f)
                m.sep()
            m.stmt("_the_dll = dll")
        
        self.before_funcs_hook(m)
        for f in self.funcs.values():
            if not self.filter_func(f):
                continue
            self.emit_func(m, f)

        m.stmt("all_types = [")
        for t in self.types.values():
            if not self.filter_type(t):
                continue
            m.stmt("    {0},", t.name)
        m.stmt("]")
        m.sep()
        m.stmt("all_funcs = [")
        for f in self.funcs.values():
            if not self.filter_func(f):
                continue
            m.stmt("    {0},", f.name)
        m.stmt("]")
        
        m.dump(filename)
        return m
    
    def filter_macro(self, name, args, value):
        return True
    
    def filter_type(self, tp):
        return True
    
    def filter_func(self, func):
        return True
    
    def emit_prelude(self, m):
        m.from_("cbinder.lib", "CEnum", "UnloadedDLL")

    def emit_get_calling_conv(self, m):
        with m.if_("sys.platform == 'win32'"):
            m.stmt("_get_calling_conv = ctypes.WINFUNCTYPE)")
        with m.else_():
            m.stmt("_get_calling_conv = ctypes.CFUNCTYPE)")
    
    def emit_macro(self, m, name, args, value, generated_macros):
        if not value:
            return
        tokens = delimiters.split(value)
        identifiers = [tok for tok in tokens if identifier_pattern.match(tok)]
        if any((tok not in generated_macros and tok not in args) for tok in identifiers):
            return
        python_tokens = []
        while tokens:
            t = tokens.pop(0)
            if not t.strip():
                continue
            if t == "##":
                if not tokens or not python_tokens:
                    #print "unexpected end of tokens!"
                    return
                python_tokens[-1] = "str(%s)" % (python_tokens[-1],)
                python_tokens.append("+ str(%s)" % (tokens.pop(0),))
            elif t == "#":
                if not tokens:
                    #print "unexpected end of tokens!"
                    return
                python_tokens.append("str(%s)" % (tokens.pop(0),))
            else:
                python_tokens.append(t)
        
        code = " ".join(t.encode("unicode_escape") for t in python_tokens).strip()
        if args:
            code = "lambda %s: %s" % (", ".join(args), code)
        try:
            compile(code, "?", "eval")
        except Exception:
            return
        
        m.stmt("{0} = {1}", name, code)
        generated_macros.add(name)
    
    def emit_struct_decl(self, m, tp):
        with m.class_(tp.name, ["ctypes.Structure"]):
            if tp.packed:
                m.stmt("_packed_ = 1")
            for name, type in tp.members:         # @ReservedAssignment
                m.stmt("{0} = {1!r}", name, type.get_ctype())
            m.sep()
            with m.def_("__repr__", "self"):
                m.return_("'{0}({1})' % ({2})", tp.name, ", ".join("%s = %%r" % (n,) for n, _ in tp.members), 
                    ", ".join("self.%s" % (n,) for n, _ in tp.members))

    def emit_struct_fields(self, m, tp):
        m.stmt("{0}._fields_ = [", tp.name)
        for name, type in tp.members:         # @ReservedAssignment
            m.stmt("    ({0!r}, {1}),", name, type.get_ctype())
        m.stmt("]")
        m.sep()

    def emit_union_decl(self, m, tp):
        with m.class_(tp.name, ["ctypes.Union"]):
            if tp.packed:
                m.stmt("_packed_ = 1")
            for name, type in tp.members:         # @ReservedAssignment
                m.stmt("{0} = {1!r}", name, type.get_ctype())
            m.sep()
            with m.def_("__repr__", "self"):
                m.return_("'{0}({1})' % ({2})", tp.name, ", ".join("%s = %%r" % (n,) for n, _ in tp.members), 
                    ", ".join("self.%s" % (n,) for n, _ in tp.members))

    def emit_union_fields(self, m, tp):
        self.emit_struct_fields(m, tp)
    
    def emit_enum(self, m, tp):
        with m.class_(tp.name, ["CEnum"]):
            m.stmt("_names_ = {0!r}", dict(tp.members))
            m.stmt("_values_ = {0!r}", dict((v, k) for k, v in tp.members))
            for name, value in tp.members:
                m.stmt("{0} = {1}", name, value)
        self.emit_enum_top_level_fields(m, tp)
        m.sep()
    
    def emit_enum_top_level_fields(self, m, tp):
        if tp.name.startswith("_anon_"):
            for k, _ in tp.members:
                m.stmt("{0} = {1}.{0}", k, tp.name)
        m.sep()
    
    def emit_typedef(self, m, tp):
        m.stmt("{0} = {1}", tp.name, tp.type.get_ctype())
    
    def emit_func_prototype(self, m, func):
        m.stmt("global _{0}", func.name)
        m.stmt("_{0} = dll.{0}", func.name)
        restype = func.type.get_ctype()
        if restype == "void":
            restype = None
        m.stmt("_{0}.restype  = {1}", func.name, restype)
        m.stmt("_{0}.argtypes = [{1}]", func.name, ", ".join(arg.get_ctype() for _, arg in func.args))
    
    def emit_func_unloaded(self, m, func):
        m.stmt("_{0} = UnloadedDLL", func.name)
    
    def before_funcs_hook(self, m):
        pass
    
    def emit_func(self, m, func):
        with m.def_(func.name, *(n for n, _ in func.args)):
            textargs = ", ".join("%s" % (arg.as_pretty_type(name),) for name, arg in func.args)
            m.stmt("'''{0}({1})'''", func.type.as_pretty_type(func.name), textargs)
            m.return_("_{0}({1})", func.name, ", ".join(n for n, _ in func.args))
Exemplo n.º 43
0
class ThumbnailCache(object):

    ' This is a persistent disk cache to speed up loading and resizing of covers '

    def __init__(self,
                 max_size=1024,  # The maximum disk space in MB
                 name='thumbnail-cache',  # The name of this cache (should be unique in location)
                 thumbnail_size=(100, 100),   # The size of the thumbnails, can be changed
                 location=None,   # The location for this cache, if None cache_dir() is used
                 test_mode=False,  # Used for testing
                 min_disk_cache=0):  # If the size is set less than or equal to this value, the cache is disabled.
        self.location = os.path.join(location or cache_dir(), name)
        if max_size <= min_disk_cache:
            max_size = 0
        self.max_size = int(max_size * (1024**2))
        self.group_id = 'group'
        self.thumbnail_size = thumbnail_size
        self.size_changed = False
        self.lock = Lock()
        self.min_disk_cache = min_disk_cache
        if test_mode:
            self.log = self.fail_on_error

    def log(self, *args, **kwargs):
        kwargs['file'] = sys.stderr
        prints(*args, **kwargs)

    def fail_on_error(self, *args, **kwargs):
        msg = ' '.join(args)
        raise CacheError(msg)

    def _do_delete(self, path):
        try:
            os.remove(path)
        except EnvironmentError as err:
            self.log('Failed to delete cached thumbnail file:', as_unicode(err))

    def _load_index(self):
        'Load the index, automatically removing incorrectly sized thumbnails and pruning to fit max_size'
        try:
            os.makedirs(self.location)
        except OSError as err:
            if err.errno != errno.EEXIST:
                self.log('Failed to make thumbnail cache dir:', as_unicode(err))
        self.total_size = 0
        self.items = OrderedDict()
        order = self._read_order()

        def listdir(*args):
            try:
                return os.listdir(os.path.join(*args))
            except EnvironmentError:
                return ()  # not a directory or no permission or whatever
        entries = ('/'.join((parent, subdir, entry))
                   for parent in listdir(self.location)
                   for subdir in listdir(self.location, parent)
                   for entry in listdir(self.location, parent, subdir))

        invalidate = set()
        try:
            with open(os.path.join(self.location, 'invalidate'), 'rb') as f:
                raw = f.read().decode('utf-8')
        except EnvironmentError as err:
            if getattr(err, 'errno', None) != errno.ENOENT:
                self.log('Failed to read thumbnail invalidate data:', as_unicode(err))
        else:
            try:
                os.remove(os.path.join(self.location, 'invalidate'))
            except EnvironmentError as err:
                self.log('Failed to remove thumbnail invalidate data:', as_unicode(err))
            else:
                def record(line):
                    try:
                        uuid, book_id = line.partition(' ')[0::2]
                        book_id = int(book_id)
                        return (uuid, book_id)
                    except Exception:
                        return None
                invalidate = {record(x) for x in raw.splitlines()}
        items = []
        try:
            for entry in entries:
                try:
                    uuid, name = entry.split('/')[0::2]
                    book_id, timestamp, size, thumbnail_size = name.split('-')
                    book_id, timestamp, size = int(book_id), float(timestamp), int(size)
                    thumbnail_size = tuple(map(int, thumbnail_size.partition('x')[0::2]))
                except (ValueError, TypeError, IndexError, KeyError, AttributeError):
                    continue
                key = (uuid, book_id)
                path = os.path.join(self.location, entry)
                if self.thumbnail_size == thumbnail_size and key not in invalidate:
                    items.append((key, Entry(path, size, timestamp, thumbnail_size)))
                    self.total_size += size
                else:
                    self._do_delete(path)
        except EnvironmentError as err:
            self.log('Failed to read thumbnail cache dir:', as_unicode(err))

        self.items = OrderedDict(sorted(items, key=lambda x:order.get(x[0], 0)))
        self._apply_size()

    def _invalidate_sizes(self):
        if self.size_changed:
            size = self.thumbnail_size
            remove = tuple(key for key, entry in iteritems(self.items) if size != entry.thumbnail_size)
            for key in remove:
                self._remove(key)
            self.size_changed = False

    def _remove(self, key):
        entry = self.items.pop(key, None)
        if entry is not None:
            self._do_delete(entry.path)
            self.total_size -= entry.size

    def _apply_size(self):
        while self.total_size > self.max_size and self.items:
            entry = self.items.popitem(last=False)[1]
            self._do_delete(entry.path)
            self.total_size -= entry.size

    def _write_order(self):
        if hasattr(self, 'items'):
            try:
                data = '\n'.join(group_id + ' ' + unicode_type(book_id) for (group_id, book_id) in self.items)
                with lopen(os.path.join(self.location, 'order'), 'wb') as f:
                    f.write(data.encode('utf-8'))
            except EnvironmentError as err:
                self.log('Failed to save thumbnail cache order:', as_unicode(err))

    def _read_order(self):
        order = {}
        try:
            with lopen(os.path.join(self.location, 'order'), 'rb') as f:
                for line in f.read().decode('utf-8').splitlines():
                    parts = line.split(' ', 1)
                    if len(parts) == 2:
                        order[(parts[0], int(parts[1]))] = len(order)
        except Exception as err:
            if getattr(err, 'errno', None) != errno.ENOENT:
                self.log('Failed to load thumbnail cache order:', as_unicode(err))
        return order

    def shutdown(self):
        with self.lock:
            self._write_order()

    def set_group_id(self, group_id):
        with self.lock:
            self.group_id = group_id

    def set_thumbnail_size(self, width, height):
        new_size = (width, height)
        with self.lock:
            if new_size != self.thumbnail_size:
                self.thumbnail_size = new_size
                self.size_changed = True
                return True
        return False

    def insert(self, book_id, timestamp, data):
        if self.max_size < len(data):
            return
        with self.lock:
            if not hasattr(self, 'total_size'):
                self._load_index()
            self._invalidate_sizes()
            ts = ('%.2f' % timestamp).replace('.00', '')
            path = '%s%s%s%s%d-%s-%d-%dx%d' % (
                self.group_id, os.sep, book_id % 100, os.sep,
                book_id, ts, len(data), self.thumbnail_size[0], self.thumbnail_size[1])
            path = os.path.join(self.location, path)
            key = (self.group_id, book_id)
            e = self.items.pop(key, None)
            self.total_size -= getattr(e, 'size', 0)
            try:
                with open(path, 'wb') as f:
                    f.write(data)
            except EnvironmentError as err:
                d = os.path.dirname(path)
                if not os.path.exists(d):
                    try:
                        os.makedirs(d)
                        with open(path, 'wb') as f:
                            f.write(data)
                    except EnvironmentError as err:
                        self.log('Failed to write cached thumbnail:', path, as_unicode(err))
                        return self._apply_size()
                else:
                    self.log('Failed to write cached thumbnail:', path, as_unicode(err))
                    return self._apply_size()
            self.items[key] = Entry(path, len(data), timestamp, self.thumbnail_size)
            self.total_size += len(data)
            self._apply_size()

    def __len__(self):
        with self.lock:
            try:
                return len(self.items)
            except AttributeError:
                self._load_index()
                return len(self.items)

    def __contains__(self, book_id):
        with self.lock:
            try:
                return (self.group_id, book_id) in self.items
            except AttributeError:
                self._load_index()
                return (self.group_id, book_id) in self.items

    def __getitem__(self, book_id):
        with self.lock:
            if not hasattr(self, 'total_size'):
                self._load_index()
            self._invalidate_sizes()
            key = (self.group_id, book_id)
            entry = self.items.pop(key, None)
            if entry is None:
                return None, None
            if entry.thumbnail_size != self.thumbnail_size:
                try:
                    os.remove(entry.path)
                except EnvironmentError as err:
                    if getattr(err, 'errno', None) != errno.ENOENT:
                        self.log('Failed to remove cached thumbnail:', entry.path, as_unicode(err))
                self.total_size -= entry.size
                return None, None
            self.items[key] = entry
            try:
                with open(entry.path, 'rb') as f:
                    data = f.read()
            except EnvironmentError as err:
                self.log('Failed to read cached thumbnail:', entry.path, as_unicode(err))
                return None, None
            return data, entry.timestamp

    def invalidate(self, book_ids):
        with self.lock:
            if hasattr(self, 'total_size'):
                for book_id in book_ids:
                    self._remove((self.group_id, book_id))
            elif os.path.exists(self.location):
                try:
                    raw = '\n'.join('%s %d' % (self.group_id, book_id) for book_id in book_ids)
                    with open(os.path.join(self.location, 'invalidate'), 'ab') as f:
                        f.write(raw.encode('ascii'))
                except EnvironmentError as err:
                    self.log('Failed to write invalidate thumbnail record:', as_unicode(err))

    @property
    def current_size(self):
        with self.lock:
            if not hasattr(self, 'total_size'):
                self._load_index()
            return self.total_size

    def empty(self):
        with self.lock:
            try:
                os.remove(os.path.join(self.location, 'order'))
            except EnvironmentError:
                pass
            if not hasattr(self, 'total_size'):
                self._load_index()
            for entry in itervalues(self.items):
                self._do_delete(entry.path)
            self.total_size = 0
            self.items = OrderedDict()

    def __hash__(self):
        return id(self)

    def set_size(self, size_in_mb):
        if size_in_mb <= self.min_disk_cache:
            size_in_mb = 0
        size_in_mb = max(0, size_in_mb)
        with self.lock:
            self.max_size = int(size_in_mb * (1024**2))
            if hasattr(self, 'total_size'):
                self._apply_size()
Exemplo n.º 44
0
class WorkerQueue(object):
    def __init__(self):
        self.queue = OrderedDict()
        self.context = None
        self.frontend = None
        self.backend = None
        self.poll_both = None
        self.poll_workers = None
        self.heartbeat_at = 0.0
        self.__conf = _QueueConf()
        self.HEARTBEAT_LIVENESS = self.__conf.getWorkerHeartBeatLiveness()
        self.HEARTBEAT_INTERVAL = self.__conf.getHeartBeatInterval()

        #  Paranoid Pirate Protocol constants
        self.PPP_READY = "\x01"      # Signals worker is ready
        self.PPP_HEARTBEAT = "\x02"  # Signals worker heartbeat

    def connect(self):
        self.context = zmq.Context(1)
        self.frontend = self.context.socket(zmq.ROUTER)
        self.backend = self.context.socket(zmq.ROUTER)
        self.frontend.bind("tcp://*:{0}".format(self.__conf.getFrontEndPort())) # For clients
        self.backend.bind("tcp://*:{0}".format(self.__conf.getBackEndPort()))  # For workers

        self.poll_workers = zmq.Poller()
        self.poll_workers.register(self.backend, zmq.POLLIN)

        self.poll_both = zmq.Poller()
        self.poll_both.register(self.frontend, zmq.POLLIN)
        self.poll_both.register(self.backend, zmq.POLLIN)
        self.heartbeat_at = time.time() + self.HEARTBEAT_INTERVAL


    def ready(self, worker):
        self.queue.pop(worker.address, None)
        self.queue[worker.address] = worker

    def run(self):
        heartbeat_at = time.time() + self.HEARTBEAT_INTERVAL

        while True:
            if len(self.queue) > 0:
                poller = self.poll_both
            else:
                poller = self.poll_workers
            socks = dict(poller.poll(self.HEARTBEAT_INTERVAL * 1000))

            # Handle worker activity on backend
            if socks.get(self.backend) == zmq.POLLIN:
                # Use worker address for LRU routing
                frames = self.backend.recv_multipart()
                if not frames:
                    break

                address = frames[0]
                self.ready(_Worker(address, self.HEARTBEAT_LIVENESS, self.HEARTBEAT_INTERVAL))

                # Validate control message, or return reply to client
                msg = frames[1:]
                if len(msg) == 1:
                    if msg[0] not in (self.PPP_READY, self.PPP_HEARTBEAT):
                        logging.error("Invalid message from worker: %s" % msg)
                else:
                    self.frontend.send_multipart(msg)

                # Send heartbeats to idle workers if it's time
                if time.time() >= heartbeat_at:
                    for worker in self.queue:
                        msg = [worker, self.PPP_HEARTBEAT]
                        self.backend.send_multipart(msg)
                    heartbeat_at = time.time() + self.HEARTBEAT_INTERVAL
            if socks.get(self.frontend) == zmq.POLLIN:
                frames = self.frontend.recv_multipart()
                if not frames:
                    break

                frames.insert(0, self.next())
                self.backend.send_multipart(frames)

            self.purge()
            
    def purge(self):
        """Look for & kill expired workers."""
        t = time.time()
        expired = []
        for address,worker in self.queue.iteritems():
            if t > worker.expiry:  # Worker expired
                expired.append(address)
        for address in expired:
            logging.info("[WORKER]: Idle worker expired: %s" % address)
            self.queue.pop(address, None)

    def next(self):
        address, worker = self.queue.popitem(False)
        return address
Exemplo n.º 45
0
class SymbolTable(object):
    # pylint: disable=too-many-public-methods
    '''Encapsulates the symbol table and provides methods to add new
    symbols and look up existing symbols. Nested scopes are supported
    and, by default, the add and lookup methods take any ancestor
    symbol tables into consideration (ones attached to nodes that are
    ancestors of the node that this symbol table is attached to).

    :param node: reference to the Schedule or Container to which this \
        symbol table belongs.
    :type node: :py:class:`psyclone.psyir.nodes.Schedule`, \
        :py:class:`psyclone.psyir.nodes.Container` or NoneType

    :raises TypeError: if node argument is not a Schedule or a Container.

    '''
    def __init__(self, node=None):
        # Dict of Symbol objects with the symbol names as keys. Make
        # this ordered so that different versions of Python always
        # produce code with declarations in the same order.
        self._symbols = OrderedDict()
        # Ordered list of the arguments.
        self._argument_list = []
        # Dict of tags. Some symbols can be identified with a tag.
        self._tags = {}
        # Reference to the node to which this symbol table belongs.
        # pylint: disable=import-outside-toplevel
        from psyclone.psyir.nodes import Schedule, Container
        if node and not isinstance(node, (Schedule, Container)):
            raise TypeError(
                "Optional node argument to SymbolTable should be a "
                "Schedule or a Container but found '{0}'."
                "".format(type(node).__name__))
        self._node = node

    @property
    def node(self):
        '''
        :returns: the Schedule or Container to which this symbol table belongs.
        :rtype: :py:class:`psyclone.psyir.nodes.Schedule`, \
            :py:class:`psyclone.psyir.nodes.Container` or NoneType

        '''
        return self._node

    def parent_symbol_table(self, scope_limit=None):
        '''If this symbol table is enclosed in another scope, return the
        symbol table of the next outer scope. Otherwise return None.

        :param scope_limit: optional Node which limits the symbol \
            search space to the symbol tables of the nodes within the \
            given scope. If it is None (the default), the whole \
            scope (all symbol tables in ancestor nodes) is searched \
            otherwise ancestors of the scope_limit node are not \
            searched.
        :type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \
            `NoneType`

        :returns: the 'parent' SymbolTable of the current SymbolTable (i.e.
                  the one that encloses this one in the PSyIR hierarchy).
        :rtype: :py:class:`psyclone.psyir.symbols.SymbolTable` or NoneType

        '''
        # Validate the supplied scope_limit
        if scope_limit is not None:
            # pylint: disable=import-outside-toplevel
            from psyclone.psyir.nodes import Node
            if not isinstance(scope_limit, Node):
                raise TypeError(
                    "The scope_limit argument '{0}', is not of type `Node`."
                    "".format(str(scope_limit)))

        # We use the Node with which this table is associated in order to
        # move up the Node hierarchy
        if self.node:
            search_next = self.node
            while search_next is not scope_limit and search_next.parent:
                search_next = search_next.parent
                if hasattr(search_next, 'symbol_table'):
                    return search_next.symbol_table
        return None

    def get_symbols(self, scope_limit=None):
        '''Return symbols from this symbol table and all symbol tables
        associated with ancestors of the node that this symbol table
        is attached to. If there are name duplicates we only return the
        one from the closest ancestor including self. It accepts an
        optional scope_limit argument.

        :param scope_limit: optional Node which limits the symbol \
            search space to the symbol tables of the nodes within the \
            given scope. If it is None (the default), the whole \
            scope (all symbol tables in ancestor nodes) is searched \
            otherwise ancestors of the scope_limit node are not \
            searched.
        :type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \
            `NoneType`

        :returns: ordered dictionary of symbols indexed by symbol name.
        :rtype: OrderedDict[str] = :py:class:`psyclone.psyir.symbols.Symbol`

        '''
        all_symbols = OrderedDict()
        current = self
        while current:
            for symbol_name, symbol in current.symbols_dict.items():
                if symbol_name not in all_symbols:
                    all_symbols[symbol_name] = symbol
            current = current.parent_symbol_table(scope_limit)
        return all_symbols

    def get_tags(self, scope_limit=None):
        '''Return tags from this symbol table and all symbol tables associated
        with ancestors of the node that this symbol table is attached
        to. If there are tag duplicates we only return the one from the closest
        ancestor including self. It accepts an optional scope_limit argument.

        :param scope_limit: optional Node which limits the symbol \
            search space to the symbol tables of the nodes within the \
            given scope. If it is None (the default), the whole \
            scope (all symbol tables in ancestor nodes) is searched \
            otherwise ancestors of the scope_limit node are not \
            searched.
        :type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \
            `NoneType`

        :returns: ordered dictionary of symbols indexed by tag.
        :rtype: OrderedDict[str] = :py:class:`psyclone.psyir.symbols.Symbol`

        '''
        all_tags = OrderedDict()
        current = self
        while current:
            for tag, symbol in current.tags_dict.items():
                if tag not in all_tags:
                    all_tags[tag] = symbol
            current = current.parent_symbol_table(scope_limit)
        return all_tags

    def shallow_copy(self):
        '''Create a copy of the symbol table with new instances of the
        top-level data structures but keeping the same existing symbol
        objects. Symbols added to the new symbol table will not be added
        in the original but the existing objects are still the same.

        :returns: a shallow copy of this symbol table.
        :rtype: :py:class:`psyclone.psyir.symbols.SymbolTable`

        '''
        # pylint: disable=protected-access
        new_st = SymbolTable()
        new_st._symbols = copy.copy(self._symbols)
        new_st._argument_list = copy.copy(self._argument_list)
        new_st._tags = copy.copy(self._tags)
        new_st._node = self.node
        return new_st

    def deep_copy(self):
        '''Create a copy of the symbol table with new instances of the
        top-level data structures and also new instances of the symbols
        contained in these data structures. Modifying a symbol attribute
        will not affect the equivalent named symbol in the original symbol
        table.

        :returns: a deep copy of this symbol table.
        :rtype: :py:class:`psyclone.psyir.symbols.SymbolTable`

        '''
        # pylint: disable=protected-access
        new_st = SymbolTable(self.node)

        # Make a copy of each symbol in the symbol table
        for symbol in self.symbols:
            new_st.add(symbol.copy())

        # Prepare the new argument list
        new_arguments = []
        for name in [arg.name for arg in self.argument_list]:
            new_arguments.append(new_st.lookup(name))
        new_st.specify_argument_list(new_arguments)

        # Prepare the new tag dict
        for tag, symbol in self._tags.items():
            new_st._tags[tag] = new_st.lookup(symbol.name)

        return new_st

    @staticmethod
    def _normalize(key):
        '''Normalises the symboltable key strings.

        :param str key: an input key.

        :returns: the normalized key.
        :rtype: str

        '''
        # The symbol table is currently case insensitive
        new_key = key.lower()
        return new_key

    def new_symbol(self,
                   root_name=None,
                   tag=None,
                   shadowing=False,
                   symbol_type=None,
                   **symbol_init_args):
        ''' Create a new symbol. Optional root_name and shadowing
        arguments can be given to choose the name following the rules of
        next_available_name(). An optional tag can also be given.
        By default it creates a generic symbol but a symbol_type argument
        and any additional initialization keyword arguments of this
        symbol_type can be provided to refine the created Symbol.

        :param root_name: optional name to use when creating a new \
            symbol name. This will be appended with an integer if the name \
            clashes with an existing symbol name.
        :type root_name: str or NoneType
        :param str tag: optional tag identifier for the new symbol.
        :param bool shadowing: optional logical flag indicating whether the \
            name can be overlapping with a symbol in any of the ancestors \
            symbol tables. Defaults to False.
        :param symbol_type: class type of the new symbol.
        :type symbol_type: type object of class (or subclasses) of \
                           :py:class:`psyclone.psyir.symbols.Symbol`
        :param symbol_init_args: arguments to create a new symbol.
        :type symbol_init_args: unwrapped Dict[str] = object

        :raises TypeError: if the type_symbol argument is not the type of a \
                           Symbol object class or one of its subclasses.

        '''
        # Only type-check symbol_type, the other arguments are just passed down
        # and type checked inside each relevant method.
        if symbol_type is not None:
            if not (isinstance(symbol_type, type)
                    and Symbol in inspect.getmro(symbol_type)):
                raise TypeError(
                    "The symbol_type parameter should be a type class of "
                    "Symbol or one of its sub-classes but found '{0}' instead."
                    .format(type(symbol_type).__name__))
        else:
            symbol_type = Symbol

        available_name = self.next_available_name(root_name, shadowing)
        symbol = symbol_type(available_name, **symbol_init_args)
        self.add(symbol, tag)
        return symbol

    def symbol_from_tag(self, tag, root_name=None, **new_symbol_args):
        ''' Lookup a tag, if it doesn't exist create a new symbol with the
        given tag. By default it creates a generic Symbol with the tag as the
        root of the symbol name. Optionally, a different root_name or any of
        the arguments available in the new_symbol() method can be given to
        refine the name and the type of the created Symbol.

        :param str tag: tag identifier.
        :param str root_name: optional name of the new symbol if it needs \
                              to be created. Otherwise it is ignored.
        :param new_symbol_args: arguments to create a new symbol.
        :type new_symbol_args: unwrapped Dict[str, object]

        :returns: symbol associated with the given tag.
        :rtype: :py:class:`psyclone.psyir.symbols.Symbol`

        :raises SymbolError: if the symbol already exists but the type_symbol \
                             argument does not match the type of the symbol \
                             found.

        '''
        try:
            symbol = self.lookup_with_tag(tag)
            # Check that the symbol found matches the requested description
            if 'symbol_type' in new_symbol_args:
                symbol_type = new_symbol_args['symbol_type']
                if not isinstance(symbol, new_symbol_args['symbol_type']):
                    raise SymbolError(
                        "Expected symbol with tag '{0}' to be of type '{1}' "
                        "but found type '{2}'.".format(tag,
                                                       symbol_type.__name__,
                                                       type(symbol).__name__))
            # TODO #1057: If the symbol is found and some unmatching arguments
            # were given it should also fail here.
            return symbol
        except KeyError:
            if not root_name:
                root_name = tag
            return self.new_symbol(root_name, tag, **new_symbol_args)

    def next_available_name(self, root_name=None, shadowing=False):
        '''Return a name that is not in the symbol table and therefore can
        be used to declare a new symbol.
        If the `root_name` argument is not supplied or if it is
        an empty string then the name is generated internally,
        otherwise the `root_name` is used. If required, an additional
        integer is appended to avoid clashes.
        If the shadowing argument is True (is False by default), the names
        in parent symbol tables will not be considered.

        :param str root_name: optional name to use when creating a new \
            symbol name. This will be appended with an integer if the name \
            clashes with an existing symbol name.
        :param bool shadowing: optional logical flag indicating whether the \
            name can be overlapping with a symbol in any of the ancestors \
            symbol tables. Defaults to False.

        :returns: the new unique symbol name.
        :rtype: str

        :raises TypeError: if the root_name argument is not a string \
            or None.
        :raises TypeError: if the shadowing argument is not a bool.

        '''
        if not isinstance(shadowing, bool):
            raise TypeError("Argument shadowing should be of type bool"
                            " but found '{0}'.".format(
                                type(shadowing).__name__))

        if shadowing:
            symbols = self._symbols
        else:
            # If symbol shadowing is not permitted, the list of symbols names
            # that can't be used includes all the symbols from all the ancestor
            # symbol tables.
            symbols = self.get_symbols()

        if root_name is not None:
            if not isinstance(root_name, six.string_types):
                raise TypeError(
                    "Argument root_name should be of type str or NoneType but "
                    "found '{0}'.".format(type(root_name).__name__))
        if not root_name:
            root_name = Config.get().psyir_root_name
        candidate_name = root_name
        idx = 1
        while candidate_name in symbols:
            candidate_name = "{0}_{1}".format(root_name, idx)
            idx += 1
        return candidate_name

    def add(self, new_symbol, tag=None):
        '''Add a new symbol to the symbol table if the symbol name is not
        already in use.

        :param new_symbol: the symbol to add to the symbol table.
        :type new_symbol: :py:class:`psyclone.psyir.symbols.Symbol`
        :param str tag: a tag identifier for the new symbol, by default no \
            tag is given.

        :raises InternalError: if the new_symbol argument is not a \
            symbol.
        :raises KeyError: if the symbol name is already in use.
        :raises KeyError: if a tag is supplied and it is already in \
            use.

        '''
        if not isinstance(new_symbol, Symbol):
            raise InternalError(
                "Symbol '{0}' is not a symbol, but '{1}'.'".format(
                    new_symbol,
                    type(new_symbol).__name__))

        key = self._normalize(new_symbol.name)
        if key in self._symbols:
            raise KeyError("Symbol table already contains a symbol with"
                           " name '{0}'.".format(new_symbol.name))

        if tag:
            if tag in self.get_tags():
                raise KeyError(
                    "This symbol table, or an outer scope ancestor symbol "
                    "table, already contains the tag '{0}' for the symbol"
                    " '{1}', so it can not be associated with symbol '{2}'.".
                    format(tag, self.lookup_with_tag(tag), new_symbol.name))
            self._tags[tag] = new_symbol

        self._symbols[key] = new_symbol

    def swap_symbol_properties(self, symbol1, symbol2):
        '''Swaps the properties of symbol1 and symbol2 apart from the symbol
        name. Argument list positions are also updated appropriately.

        :param symbol1: the first symbol.
        :type symbol1: :py:class:`psyclone.psyir.symbols.Symbol`
        :param symbol2: the second symbol.
        :type symbol2: :py:class:`psyclone.psyir.symbols.Symbol`

        :raises KeyError: if either of the supplied symbols are not in \
                          the symbol table.
        :raises TypeError: if the supplied arguments are not symbols, \
                 or the names of the symbols are the same in the SymbolTable \
                 instance.

        '''
        for symbol in [symbol1, symbol2]:
            if not isinstance(symbol, Symbol):
                raise TypeError("Arguments should be of type 'Symbol' but "
                                "found '{0}'.".format(type(symbol).__name__))
            if symbol.name not in self._symbols:
                raise KeyError("Symbol '{0}' is not in the symbol table."
                               "".format(symbol.name))
        if symbol1.name == symbol2.name:
            raise ValueError("The symbols should have different names, but "
                             "found '{0}' for both.".format(symbol1.name))

        tmp_symbol = symbol1.copy()
        symbol1.copy_properties(symbol2)
        symbol2.copy_properties(tmp_symbol)

        # Update argument list if necessary
        index1 = None
        if symbol1 in self._argument_list:
            index1 = self._argument_list.index(symbol1)
        index2 = None
        if symbol2 in self._argument_list:
            index2 = self._argument_list.index(symbol2)
        if index1 is not None:
            self._argument_list[index1] = symbol2
        if index2 is not None:
            self._argument_list[index2] = symbol1

    def specify_argument_list(self, argument_symbols):
        '''
        Sets-up the internal list storing the order of the arguments to this
        kernel.

        :param list argument_symbols: ordered list of the DataSymbols \
            representing the kernel arguments.

        :raises ValueError: if the new argument_list is not consistent with \
            the existing entries in the SymbolTable.

        '''
        self._validate_arg_list(argument_symbols)
        self._argument_list = argument_symbols[:]

    def lookup(self, name, visibility=None, scope_limit=None):
        '''Look up a symbol in the symbol table. The lookup can be limited
        by visibility (e.g. just show public methods) or by scope_limit (e.g.
        just show symbols up to a certain scope).

        :param str name: name of the symbol.
        :param visibilty: the visibility or list of visibilities that the \
                          symbol must have.
        :type visibility: [list of] :py:class:`psyclone.symbols.Visibility`
        :param scope_limit: optional Node which limits the symbol \
            search space to the symbol tables of the nodes within the \
            given scope. If it is None (the default), the whole \
            scope (all symbol tables in ancestor nodes) is searched \
            otherwise ancestors of the scope_limit node are not \
            searched.
        :type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \
            `NoneType`

        :returns: the symbol with the given name and, if specified, visibility.
        :rtype: :py:class:`psyclone.psyir.symbols.Symbol`

        :raises TypeError: if the name argument is not a string.
        :raises SymbolError: if the name exists in the Symbol Table but does \
                             not have the specified visibility.
        :raises TypeError: if the visibility argument has the wrong type.
        :raises KeyError: if the given name is not in the Symbol Table.

        '''
        if not isinstance(name, six.string_types):
            raise TypeError(
                "Expected the name argument to the lookup() method to be "
                "a str but found '{0}'."
                "".format(type(name).__name__))

        try:
            symbol = self.get_symbols(scope_limit)[self._normalize(name)]
            if visibility:
                if not isinstance(visibility, list):
                    vis_list = [visibility]
                else:
                    vis_list = visibility
                if symbol.visibility not in vis_list:
                    vis_names = []
                    # Take care here in case the 'visibility' argument
                    # is of the wrong type
                    for vis in vis_list:
                        if not isinstance(vis, Symbol.Visibility):
                            raise TypeError(
                                "the 'visibility' argument to lookup() must be"
                                " an instance (or list of instances) of "
                                "Symbol.Visibility but got '{0}' when "
                                "searching for symbol '{1}'".format(
                                    type(vis).__name__, name))
                        vis_names.append(vis.name)
                    raise SymbolError(
                        "Symbol '{0}' exists in the Symbol Table but has "
                        "visibility '{1}' which does not match with the "
                        "requested visibility: {2}".format(
                            name, symbol.visibility.name, vis_names))
            return symbol
        except KeyError as err:
            six.raise_from(
                KeyError("Could not find '{0}' in the Symbol Table."
                         "".format(name)), err)

    def lookup_with_tag(self, tag, scope_limit=None):
        '''Look up a symbol by its tag. The lookup can be limited by
        scope_limit (e.g. just show symbols up to a certain scope).

        :param str tag: tag identifier.
        :param scope_limit: optional Node which limits the symbol \
            search space to the symbol tables of the nodes within the \
            given scope. If it is None (the default), the whole \
            scope (all symbol tables in ancestor nodes) is searched \
            otherwise ancestors of the scope_limit node are not \
            searched.

        :returns: symbol with the given tag.
        :rtype: :py:class:`psyclone.psyir.symbols.Symbol`

        :raises TypeError: if the tag argument is not a string.
        :raises KeyError: if the given tag is not in the Symbol Table.

        '''
        if not isinstance(tag, six.string_types):
            raise TypeError(
                "Expected the tag argument to the lookup_with_tag() method "
                "to be a str but found '{0}'.".format(type(tag).__name__))

        try:
            return self.get_tags(scope_limit)[tag]
        except KeyError as err:
            six.raise_from(
                KeyError("Could not find the tag '{0}' in the Symbol Table."
                         "".format(tag)), err)

    def __contains__(self, key):
        '''Check if the given key is part of the Symbol Table.

        :param str key: key to check for existance.

        :returns: whether the Symbol Table contains the given key.
        :rtype: bool
        '''
        return self._normalize(key.lower()) in self._symbols

    def imported_symbols(self, csymbol):
        '''
        Examines the contents of this symbol table to see which DataSymbols
        (if any) are imported from the supplied ContainerSymbol (which must
        be present in the SymbolTable).

        :param csymbol: the ContainerSymbol to search for imports from.
        :type csymbol: :py:class:`psyclone.psyir.symbols.ContainerSymbol`

        :returns: list of DataSymbols that are imported from the supplied \
                  ContainerSymbol. If none are found then the list is empty.
        :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol`

        :raises TypeError: if the supplied object is not a ContainerSymbol.
        :raises KeyError: if the supplied object is not in this SymbolTable.

        '''
        if not isinstance(csymbol, ContainerSymbol):
            raise TypeError(
                "imported_symbols() expects a ContainerSymbol but got an "
                "object of type '{0}'".format(type(csymbol).__name__))
        # self.lookup(name) will raise a KeyError if there is no symbol with
        # that name in the table.
        if self.lookup(csymbol.name) is not csymbol:
            raise KeyError("The '{0}' entry in this SymbolTable is not the "
                           "supplied ContainerSymbol.".format(csymbol.name))

        return [
            symbol for symbol in self.global_symbols
            if symbol.interface.container_symbol is csymbol
        ]

    def swap(self, old_symbol, new_symbol):
        '''
        Remove the `old_symbol` from the table and replace it with the
        `new_symbol`.

        :param old_symbol: the symbol to remove from the table.
        :type old_symbol: :py:class:`psyclone.psyir.symbols.Symbol`
        :param new_symbol: the symbol to add to the table.
        :type new_symbol: :py:class:`psyclone.psyir.symbols.Symbol`

        :raises TypeError: if either old/new_symbol are not Symbols.
        :raises SymbolError: if `old_symbol` and `new_symbol` don't have \
                             the same name (after normalising).
        '''
        if not isinstance(old_symbol, Symbol):
            raise TypeError("Symbol to remove must be of type Symbol but "
                            "got '{0}'".format(type(old_symbol).__name__))
        if not isinstance(new_symbol, Symbol):
            raise TypeError("Symbol to add must be of type Symbol but "
                            "got '{0}'".format(type(new_symbol).__name__))
        # The symbol table is not case sensitive so we must normalise the
        # symbol names before comparing them.
        if (self._normalize(old_symbol.name) != self._normalize(
                new_symbol.name)):
            raise SymbolError(
                "Cannot swap symbols that have different names, got: '{0}' "
                "and '{1}'".format(old_symbol.name, new_symbol.name))
        # TODO #898 remove() does not currently check for any uses of
        # old_symbol.
        self.remove(old_symbol)
        self.add(new_symbol)

    def remove(self, symbol):
        '''
        Remove the supplied symbol from the Symbol Table. This has a high
        potential to leave broken links, so this method checks for some
        references to the removed symbol depending on the symbol type.

        Currently, generic Symbols, ContainerSymbols and RoutineSymbols are
        supported. Support for removing other types of Symbol will be added
        as required.

        TODO #898. This method should check for any references/uses of
        the target symbol.

        :param symbol: the container symbol to remove.
        :type symbol: :py:class:`psyclone.psyir.symbols.ContainerSymbol`

        :raises TypeError: if the supplied symbol is not of type Symbol.
        :raises NotImplementedError: the removal of this symbol type is not \
                                     supported yet.
        :raises KeyError: if the supplied symbol is not in the symbol table.
        :raises ValueError: if the supplied container symbol is referenced \
                            by one or more DataSymbols.
        :raises InternalError: if the supplied symbol is not the same as the \
                               entry with that name in this SymbolTable.
        '''
        if not isinstance(symbol, Symbol):
            raise TypeError("remove() expects a Symbol argument but found: "
                            "'{0}'.".format(type(symbol).__name__))

        # pylint: disable=unidiomatic-typecheck
        if not (isinstance(symbol, (ContainerSymbol, RoutineSymbol))
                or type(symbol) == Symbol):
            raise NotImplementedError(
                "remove() currently only supports generic Symbol, "
                "ContainerSymbol and RoutineSymbol types but got: '{0}'"
                "".format(type(symbol).__name__))
        # pylint: enable=unidiomatic-typecheck

        # Since we are manipulating the _symbols dict directly we must use
        # the normalised name of the symbol.
        norm_name = self._normalize(symbol.name)

        if norm_name not in self._symbols:
            raise KeyError("Cannot remove Symbol '{0}' from symbol table "
                           "because it does not exist.".format(symbol.name))
        # Sanity-check that the entry in the table is the symbol we've
        # been passed.
        if self._symbols[norm_name] is not symbol:
            raise InternalError(
                "The Symbol with name '{0}' in this symbol table is not the "
                "same Symbol object as the one that has been supplied to the "
                "remove() method.".format(symbol.name))

        # We can only remove a ContainerSymbol if no DataSymbols are
        # being imported from it
        if (isinstance(symbol, ContainerSymbol)
                and self.imported_symbols(symbol)):
            raise ValueError(
                "Cannot remove ContainerSymbol '{0}' because symbols "
                "{1} are imported from it - remove them first.".format(
                    symbol.name,
                    [sym.name for sym in self.imported_symbols(symbol)]))

        # If the symbol had a tag, it should be disassociated
        for tag, tagged_symbol in list(self._tags.items()):
            if symbol is tagged_symbol:
                del self._tags[tag]

        self._symbols.pop(norm_name)

    @property
    def argument_list(self):
        '''
        Checks that the contents of the SymbolTable are self-consistent
        and then returns the list of kernel arguments.

        :returns: ordered list of arguments.
        :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol`

        :raises InternalError: if the entries of the SymbolTable are not \
            self-consistent.

        '''
        try:
            self._validate_arg_list(self._argument_list)
            self._validate_non_args()
        except ValueError as err:
            # If the SymbolTable is inconsistent at this point then
            # we have an InternalError.
            six.raise_from(InternalError(str(err.args)), err)
        return self._argument_list

    @staticmethod
    def _validate_arg_list(arg_list):
        '''
        Checks that the supplied list of Symbols are valid kernel arguments.

        :param arg_list: the proposed kernel arguments.
        :type param_list: list of :py:class:`psyclone.psyir.symbols.DataSymbol`

        :raises TypeError: if any item in the supplied list is not a \
            DataSymbol.
        :raises ValueError: if any of the symbols does not have an argument \
            interface.

        '''
        for symbol in arg_list:
            if not isinstance(symbol, DataSymbol):
                raise TypeError("Expected a list of DataSymbols but found an "
                                "object of type '{0}'.".format(type(symbol)))
            if not symbol.is_argument:
                raise ValueError(
                    "DataSymbol '{0}' is listed as a kernel argument but has "
                    "an interface of type '{1}' rather than "
                    "ArgumentInterface"
                    "".format(str(symbol), type(symbol.interface)))

    def _validate_non_args(self):
        '''
        Performs internal consistency checks on the current entries in the
        SymbolTable that do not represent kernel arguments.

        :raises ValueError: if a symbol that is not in the argument list \
            has an argument interface.

        '''
        for symbol in self.datasymbols:
            if symbol not in self._argument_list:
                # DataSymbols not in the argument list must not have a
                # Symbol.Argument interface
                if symbol.is_argument:
                    raise ValueError(
                        "Symbol '{0}' is not listed as a kernel argument and "
                        "yet has an ArgumentInterface interface."
                        "".format(str(symbol)))

    def get_unresolved_datasymbols(self, ignore_precision=False):
        '''
        Create a list of the names of all of the DataSymbols in the table that
        do not have a resolved interface. If ignore_precision is True then
        those DataSymbols that are used to define the precision of other
        DataSymbols are ignored. If no unresolved DataSymbols are found then an
        empty list is returned.

        :param bool ignore_precision: whether or not to ignore DataSymbols \
                    that are used to define the precision of other DataSymbols.

        :returns: the names of those DataSymbols with unresolved interfaces.
        :rtype: list of str

        '''
        unresolved_symbols = [
            sym for sym in self.datasymbols if sym.is_unresolved
        ]
        if ignore_precision:
            unresolved_datasymbols = list(
                set(unresolved_symbols) - set(self.precision_datasymbols))
        else:
            unresolved_datasymbols = unresolved_symbols
        return [sym.name for sym in unresolved_datasymbols]

    @property
    def symbols_dict(self):
        '''
        :returns: ordered dictionary of symbols indexed by symbol name.
        :rtype: OrderedDict[str] = :py:class:`psyclone.psyir.symbols.Symbol`

        '''
        return self._symbols

    @property
    def tags_dict(self):
        '''
        :returns: ordered dictionary of symbols indexed by tag.
        :rtype: OrderedDict[str] = :py:class:`psyclone.psyir.symbols.Symbol`

        '''
        return self._tags

    @property
    def symbols(self):
        '''
        :returns: list of symbols.
        :rtype: list of :py:class:`psyclone.psyir.symbols.Symbol`
        '''
        return list(self._symbols.values())

    @property
    def datasymbols(self):
        '''
        :returns: list of symbols representing data variables.
        :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol`
        '''
        return [
            sym for sym in self._symbols.values()
            if isinstance(sym, DataSymbol)
        ]

    @property
    def local_datasymbols(self):
        '''
        :returns: list of symbols representing local variables.
        :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol`
        '''
        return [sym for sym in self.datasymbols if sym.is_local]

    @property
    def argument_datasymbols(self):
        '''
        :returns: list of symbols representing arguments.
        :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol`
        '''
        return [sym for sym in self.datasymbols if sym.is_argument]

    @property
    def global_symbols(self):
        '''
        :returns: list of symbols that have 'global' interface (are \
            associated with data that exists outside the current scope).
        :rtype: list of :py:class:`psyclone.psyir.symbols.Symbol`

        '''
        return [sym for sym in self.symbols if sym.is_global]

    @property
    def precision_datasymbols(self):
        '''
        :returns: list of all symbols used to define the precision of \
                  other symbols within the table.
        :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol`

        '''
        # Accumulate into a set so as to remove any duplicates
        precision_symbols = set()
        for sym in self.datasymbols:
            # Not all types have the 'precision' attribute (e.g. DeferredType)
            if (hasattr(sym.datatype, "precision")
                    and isinstance(sym.datatype.precision, DataSymbol)):
                precision_symbols.add(sym.datatype.precision)
        return list(precision_symbols)

    @property
    def containersymbols(self):
        '''
        :returns: a list of the ContainerSymbols present in the Symbol Table.
        :rtype: list of :py:class:`psyclone.psyir.symbols.ContainerSymbol`
        '''
        return [
            sym for sym in self.symbols if isinstance(sym, ContainerSymbol)
        ]

    @property
    def local_typesymbols(self):
        '''
        :returns: the local TypeSymbols present in the Symbol Table.
        :rtype: list of :py:class:`psyclone.psyir.symbols.TypeSymbol`
        '''
        return [
            sym for sym in self.symbols
            if (isinstance(sym, TypeSymbol) and sym.is_local)
        ]

    @property
    def iteration_indices(self):
        '''
        :returns: list of symbols representing kernel iteration indices.
        :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol`

        :raises NotImplementedError: this method is abstract.
        '''
        raise NotImplementedError(
            "Abstract property. Which symbols are iteration indices is"
            " API-specific.")

    @property
    def data_arguments(self):
        '''
        :returns: list of symbols representing kernel data arguments.
        :rtype: list of :py:class:`psyclone.psyir.symbols.DataSymbol`

        :raises NotImplementedError: this method is abstract.
        '''
        raise NotImplementedError(
            "Abstract property. Which symbols are data arguments is"
            " API-specific.")

    def copy_external_global(self, globalvar, tag=None):
        '''
        Copy the given global variable (and its referenced ContainerSymbol if
        needed) into the SymbolTable.

        :param globalvar: the variable to be copied in.
        :type globalvar: :py:class:`psyclone.psyir.symbols.DataSymbol`
        :param str tag: a tag identifier for the new copy, by default no tag \
            is given.

        :raises TypeError: if the given variable is not a global variable.
        :raises KeyError: if the given variable name already exists in the \
            symbol table.

        '''
        if not isinstance(globalvar, DataSymbol):
            raise TypeError(
                "The globalvar argument of SymbolTable.copy_external_global"
                " method should be a DataSymbol, but found '{0}'."
                "".format(type(globalvar).__name__))

        if not globalvar.is_global:
            raise TypeError(
                "The globalvar argument of SymbolTable.copy_external_"
                "global method should have a GlobalInterface interface, "
                "but found '{0}'.".format(type(globalvar.interface).__name__))

        external_container_name = globalvar.interface.container_symbol.name

        # If the Container is not yet in the SymbolTable we need to
        # create one and add it.
        if external_container_name not in self:
            self.add(ContainerSymbol(external_container_name))
        container_ref = self.lookup(external_container_name)

        # Copy the variable into the SymbolTable with the appropriate interface
        if globalvar.name not in self:
            new_symbol = globalvar.copy()
            # Update the interface of this new symbol
            new_symbol.interface = GlobalInterface(container_ref)
            self.add(new_symbol, tag)
        else:
            # If it already exists it must refer to the same Container and have
            # the same tag.
            local_instance = self.lookup(globalvar.name)
            if not (local_instance.is_global
                    and local_instance.interface.container_symbol.name
                    == external_container_name):
                raise KeyError("Couldn't copy '{0}' into the SymbolTable. The"
                               " name '{1}' is already used by another symbol."
                               "".format(globalvar, globalvar.name))
            if tag:
                # If the symbol already exists and a tag is provided
                try:
                    self.lookup_with_tag(tag)
                except KeyError:
                    # If the tag was not used, it will now be attached
                    # to the symbol.
                    self._tags[tag] = self.lookup(globalvar.name)

                # The tag should not refer to a different symbol
                if self.lookup(globalvar.name) != self.lookup_with_tag(tag):
                    raise KeyError(
                        "Couldn't copy '{0}' into the SymbolTable. The"
                        " tag '{1}' is already used by another symbol."
                        "".format(globalvar, tag))

    def rename_symbol(self, symbol, name):
        '''
        Rename the given symbol which should belong to this symbol table
        with the new name provided.

        :param symbol: the symbol to be renamed.
        :type symbol: :py:class:`psyclone.psyir.symbols.Symbol`
        :param str name: the new name.

        :raises TypeError: if the symbol is not a Symbol.
        :raises TypeError: if the name is not a str.
        :raises ValueError: if the given symbol does not belong to this \
                            symbol table.
        :raises KeyError: if the given variable name already exists in the \
                          symbol table.

        '''
        if not isinstance(symbol, Symbol):
            raise TypeError(
                "The symbol argument of rename_symbol() must be a Symbol, but"
                " found: '{0}'.".format(type(symbol).__name__))

        if symbol not in self.symbols:
            raise ValueError(
                "The symbol argument of rename_symbol() must belong to this "
                "symbol_table instance, but '{0}' does not.".format(symbol))

        if not isinstance(name, str):
            raise TypeError(
                "The name argument of rename_symbol() must be a str, but"
                " found: '{0}'.".format(type(symbol).__name__))

        if name in self._symbols:
            raise KeyError(
                "The name argument of rename_symbol() must not already exist "
                "in this symbol_table instance, but '{0}' does.".format(name))

        # Delete current dictionary entry
        del self._symbols[symbol.name]

        # Rename symbol using protected access as the Symbol class should not
        # expose a name attribute setter.
        # pylint: disable=protected-access
        symbol._name = name

        # Re-insert modified symbol
        self.add(symbol)

    def has_wildcard_imports(self):
        '''
        Searches this symbol table and then up through any parent symbol
        tables for a ContainerSymbol that has a wildcard import.

        :returns: True if a wildcard import is found, False otherwise.
        :rtype: bool

        '''
        current_table = self
        while current_table:
            for sym in current_table.containersymbols:
                if sym.wildcard_import:
                    return True
            current_table = current_table.parent_symbol_table()
        return False

    def view(self):
        '''
        Print a representation of this Symbol Table to stdout.
        '''
        print(str(self))

    def __str__(self):
        return ("Symbol Table:\n" +
                "\n".join(map(str, self._symbols.values())) + "\n")
Exemplo n.º 46
0
class Data(object):
    """The basic data container in Glue.

    The data object stores data as a collection of
    :class:`~glue.core.component.Component` objects.  Each component stored in a
    dataset must have the same shape.

    Catalog data sets are stored such that each column is a distinct
    1-dimensional :class:`~glue.core.component.Component`.

    There are several ways to extract the actual numerical data stored in a
    :class:`~glue.core.data.Data` object::

       data = Data(x=[1, 2, 3], label='data')
       xid = data.id['x']

       data[xid]
       data.get_component(xid).data
       data['x']  # if 'x' is a unique component name

    Likewise, datasets support :ref:`fancy indexing <numpy:basics.indexing>`::

        data[xid, 0:2]
        data[xid, [True, False, True]]

    See also: :ref:`data_tutorial`
    """
    def __init__(self, label="", coords=None, **kwargs):
        """

        :param label: label for data
        :type label: str

        Extra array-like keywords are extracted into components
        """
        # Coordinate conversion object
        self.coords = coords or Coordinates()
        self._shape = ()

        # Components
        self._components = OrderedDict()
        self._externally_derivable_components = OrderedDict()
        self._pixel_aligned_data = OrderedDict()
        self._pixel_component_ids = ComponentIDList()
        self._world_component_ids = ComponentIDList()

        self.id = ComponentIDDict(self)

        # Metadata
        self.meta = OrderedDict()

        # Subsets of the data
        self._subsets = []

        # Hub that the data is attached to
        self.hub = None

        self.style = VisualAttributes(parent=self)

        self._coordinate_links = []

        self.data = self
        self.label = label

        self.edit_subset = None

        for lbl, data in sorted(kwargs.items()):
            self.add_component(data, lbl)

        self._key_joins = {}

        # To avoid circular references when saving objects with references to
        # the data, we make sure that all Data objects have a UUID that can
        # uniquely identify them.
        self.uuid = str(uuid.uuid4())

    @property
    def subsets(self):
        """
        Tuple of subsets attached to this dataset
        """
        return tuple(self._subsets)

    @property
    def ndim(self):
        """
        Dimensionality of the dataset
        """
        return len(self.shape)

    @property
    def shape(self):
        """
        Tuple of array dimensions, like :attr:`numpy.ndarray.shape`
        """
        return self._shape

    @property
    def label(self):
        """ Convenience access to data set's label """
        return self._label

    @label.setter
    def label(self, value):
        """ Set the label to value
        """
        self._label = value
        self.broadcast(attribute='label')

    @property
    def size(self):
        """
        Total number of elements in the dataset.
        """
        return np.product(self.shape)

    @contract(component=Component)
    def _check_can_add(self, component):
        if isinstance(component, DerivedComponent):
            return component._data is self
        else:
            if len(self._components) == 0:
                return True
            else:
                if all(comp.shape == () for comp in self._components.values()):
                    return True
                else:
                    return component.shape == self.shape

    @contract(cid=ComponentID, returns=np.dtype)
    def dtype(self, cid):
        """Lookup the dtype for the data associated with a ComponentID"""

        # grab a small piece of data
        ind = tuple([slice(0, 1)] * self.ndim)
        arr = self[cid, ind]
        return arr.dtype

    @contract(component_id=ComponentID)
    def remove_component(self, component_id):
        """ Remove a component from a data set

        :param component_id: the component to remove
        :type component_id: :class:`~glue.core.component_id.ComponentID`
        """
        # TODO: avoid too many messages when removing a component triggers
        # the removal of derived components.
        if component_id in self._components:
            self._components.pop(component_id)
            self._removed_derived_that_depend_on(component_id)
            if self.hub:
                msg = DataRemoveComponentMessage(self, component_id)
                self.hub.broadcast(msg)
                msg = ComponentsChangedMessage(self)
                self.hub.broadcast(msg)

    def _removed_derived_that_depend_on(self, component_id):
        """
        Remove internal derived components that can no longer be derived.
        """
        remove = []
        for cid in self.derived_components:
            comp = self.get_component(cid)
            if component_id in comp.link.get_from_ids():
                remove.append(cid)
        for cid in remove:
            self.remove_component(cid)

    @contract(other='isinstance(Data)', cid='cid_like', cid_other='cid_like')
    def join_on_key(self, other, cid, cid_other):
        """
        Create an *element* mapping to another dataset, by joining on values of
        ComponentIDs in both datasets.

        This join allows any subsets defined on `other` to be propagated to
        self. The different ways to call this method are described in the
        **Examples** section below.

        Parameters
        ----------
        other : :class:`~glue.core.data.Data`
            Data object to join with
        cid : str or :class:`~glue.core.component_id.ComponentID` or iterable
            Component(s) in this dataset to use as a key
        cid_other : str or :class:`~glue.core.component_id.ComponentID` or iterable
            Component(s) in the other dataset to use as a key

        Examples
        --------

        There are several ways to use this function, depending on how many
        components are passed to ``cid`` and ``cid_other``.

        **Joining on single components**

        First, one can specify a single component ID for both ``cid`` and
        ``cid_other``: this is the standard mode, and joins one component from
        one dataset to the other:

            >>> d1 = Data(x=[1, 2, 3, 4, 5], k1=[0, 0, 1, 1, 2], label='d1')
            >>> d2 = Data(y=[2, 4, 5, 8, 4], k2=[1, 3, 1, 2, 3], label='d2')
            >>> d2.join_on_key(d1, 'k2', 'k1')

        Selecting all values in ``d1`` where x is greater than 2 returns
        the last three items as expected:

            >>> s = d1.new_subset()
            >>> s.subset_state = d1.id['x'] > 2
            >>> s.to_mask()
            array([False, False,  True,  True,  True], dtype=bool)

        The linking was done between k1 and k2, and the values of
        k1 for the last three items are 1 and 2 - this means that the
        first, third, and fourth item in ``d2`` will then get selected,
        since k2 has a value of either 1 or 2 for these items.

            >>> s = d2.new_subset()
            >>> s.subset_state = d1.id['x'] > 2
            >>> s.to_mask()
            array([ True, False,  True,  True, False], dtype=bool)

        **Joining on multiple components**

        .. note:: This mode is currently slow, and will be optimized
                  significantly in future.

        Next, one can specify several components for each dataset: in this
        case, the number of components given should match for both datasets.
        This causes items in both datasets to be linked when (and only when)
        the set of keys match between the two datasets:

            >>> d1 = Data(x=[1, 2, 3, 5, 5],
            ...           y=[0, 0, 1, 1, 2], label='d1')
            >>> d2 = Data(a=[2, 5, 5, 8, 4],
            ...           b=[1, 3, 2, 2, 3], label='d2')
            >>> d2.join_on_key(d1, ('a', 'b'), ('x', 'y'))

        Selecting all items where x is 5 in ``d1`` in which x is a
        component works as expected and selects the two last items::

            >>> s = d1.new_subset()
            >>> s.subset_state = d1.id['x'] == 5
            >>> s.to_mask()
            array([False, False, False,  True,  True], dtype=bool)

        If we apply this selection to ``d2``, only items where a is 5
        and b is 2 will be selected:

            >>> s = d2.new_subset()
            >>> s.subset_state = d1.id['x'] == 5
            >>> s.to_mask()
            array([False, False,  True, False, False], dtype=bool)

        and in particular, the second item (where a is 5 and b is 3) is not
        selected.

        **One-to-many and many-to-one joining**

        Finally, you can specify one component in one dataset and multiple ones
        in the other. In the case where one component is specified for this
        dataset and multiple ones for the other dataset, then when an item
        is selected in the other dataset, it will cause any item in the present
        dataset which matches any of the keys in the other data to be selected:

            >>> d1 = Data(x=[1, 2, 3], label='d1')
            >>> d2 = Data(a=[1, 1, 2],
            ...           b=[2, 3, 3], label='d2')
            >>> d1.join_on_key(d2, 'x', ('a', 'b'))

        In this case, if we select all items in ``d2`` where a is 2, this
        will select the third item:

            >>> s = d2.new_subset()
            >>> s.subset_state = d2.id['a'] == 2
            >>> s.to_mask()
            array([False, False,  True], dtype=bool)

        Since we have joined the datasets using both a and b, we select
        all items in ``d1`` where x is either the value or a or b
        (2 or 3) which means we select the second and third item:

            >>> s = d1.new_subset()
            >>> s.subset_state = d2.id['a'] == 2
            >>> s.to_mask()
            array([False,  True,  True], dtype=bool)

        We can also join the datasets the other way around:

            >>> d1 = Data(x=[1, 2, 3], label='d1')
            >>> d2 = Data(a=[1, 1, 2],
            ...           b=[2, 3, 3], label='d2')
            >>> d2.join_on_key(d1, ('a', 'b'), 'x')

        In this case, selecting items in ``d1`` where x is 1 selects the
        first item, as expected:

            >>> s = d1.new_subset()
            >>> s.subset_state = d1.id['x'] == 1
            >>> s.to_mask()
            array([ True, False, False], dtype=bool)

        This then causes any item in ``d2`` where either a or b are 1
        to be selected, i.e. the first two items:

            >>> s = d2.new_subset()
            >>> s.subset_state = d1.id['x'] == 1
            >>> s.to_mask()
            array([ True,  True, False], dtype=bool)
        """

        # To make things easier, we transform all component inputs to a tuple
        if isinstance(cid, six.string_types) or isinstance(cid, ComponentID):
            cid = (cid, )
        if isinstance(cid_other, six.string_types) or isinstance(
                cid_other, ComponentID):
            cid_other = (cid_other, )

        if len(cid) > 1 and len(cid_other) > 1 and len(cid) != len(cid_other):
            raise Exception("Either the number of components in the key join "
                            "sets should match, or one of the component sets "
                            "should contain a single component.")

        def get_component_id(data, name):
            if isinstance(name, ComponentID):
                return name
            else:
                cid = data.find_component_id(name)
                if cid is None:
                    raise ValueError("ComponentID not found in %s: %s" %
                                     (data.label, name))
                return cid

        cid = tuple(get_component_id(self, name) for name in cid)
        cid_other = tuple(get_component_id(other, name) for name in cid_other)

        self._key_joins[other] = (cid, cid_other)
        other._key_joins[self] = (cid_other, cid)

    @contract(component='component_like', label='cid_like')
    def add_component(self, component, label):
        """ Add a new component to this data set.

        :param component: object to add. Can be a Component,
                          array-like object, or ComponentLink

        :param label:
              The label. If this is a string,
              a new :class:`glue.core.component_id.ComponentID` with this label will be
              created and associated with the Component

        :type component: :class:`~glue.core.component.Component` or
                         array-like
        :type label: :class:`str` or :class:`~glue.core.component_id.ComponentID`

        :raises:

           TypeError, if label is invalid
           ValueError if the component has an incompatible shape

        :returns:

           The ComponentID associated with the newly-added component
        """

        if isinstance(component, ComponentLink):
            return self.add_component_link(component, label=label)

        if not isinstance(component, Component):
            component = Component.autotyped(component)

        if isinstance(component, DerivedComponent):
            if len(self._components) == 0:
                raise TypeError(
                    "Cannot add a derived component as a first component")
            component.set_parent(self)

        if not (self._check_can_add(component)):
            raise ValueError("The dimensions of component %s are "
                             "incompatible with the dimensions of this data: "
                             "%r vs %r" % (label, component.shape, self.shape))

        if isinstance(label, ComponentID):
            component_id = label
            if component_id.parent is None:
                component_id.parent = self
        else:
            component_id = ComponentID(label, parent=self)

        if len(self._components) == 0:
            # TODO: make sure the following doesn't raise a componentsraised message
            self._create_pixel_and_world_components(ndim=component.ndim)

        # In some cases, such as when loading a session, we actually disable the
        # auto-creation of pixel and world coordinates, so the first component
        # may be a coordinate component with no shape. Therefore we only set the
        # shape once a component has a valid shape rather than strictly on the
        # first component.
        if self._shape == () and component.shape != ():
            self._shape = component.shape

        is_present = component_id in self._components
        self._components[component_id] = component

        if self.hub and not is_present:
            msg = DataAddComponentMessage(self, component_id)
            self.hub.broadcast(msg)
            msg = ComponentsChangedMessage(self)
            self.hub.broadcast(msg)

        return component_id

    def _set_externally_derivable_components(self, derivable_components):
        """
        Externally deriable components are components identified by component
        IDs from other datasets.

        This method is meant for internal use only and is called by the link
        manager. The ``derivable_components`` argument should be set to a
        dictionary where the keys are the derivable component IDs, and the
        values are DerivedComponent instances which can be used to get the
        data.
        """
        if len(self._externally_derivable_components) == 0 and len(
                derivable_components) == 0:
            return
        self._externally_derivable_components = derivable_components
        if self.hub:
            msg = ExternallyDerivableComponentsChangedMessage(self)
            self.hub.broadcast(msg)

    def _set_pixel_aligned_data(self, pixel_aligned_data):
        """
        Pixel-aligned data are datasets that contain pixel component IDs
        that are equivalent (identically, not transformed) with all pixel
        component IDs in the present dataset.

        Note that the other datasets may have more but not fewer dimensions, so
        this information may not be symmetric between datasets with differing
        numbers of dimensions.
        """

        # First check if anything has changed, as if not then we should just
        # leave things as-is and avoid emitting a message.
        if len(self._pixel_aligned_data) == len(pixel_aligned_data):
            for data in self._pixel_aligned_data:
                if data not in pixel_aligned_data or pixel_aligned_data[
                        data] != self._pixel_aligned_data[data]:
                    break
            else:
                return

        self._pixel_aligned_data = pixel_aligned_data
        if self.hub:
            msg = PixelAlignedDataChangedMessage(self)
            self.hub.broadcast(msg)

    @property
    def pixel_aligned_data(self):
        """
        Information about other datasets in the same data collection that have
        matching or a subset of pixel component IDs.

        This is returned as a dictionary where each key is a dataset with
        matching pixel component IDs, and the value is the order in which the
        pixel component IDs of the other dataset can be found in the current
        one.
        """
        return self._pixel_aligned_data

    @contract(link=ComponentLink,
              label='cid_like|None',
              returns=DerivedComponent)
    def add_component_link(self, link, label=None):
        """
        Shortcut method for generating a new
        :class:`~glue.core.component.DerivedComponent` from a ComponentLink
        object, and adding it to a data set.

        Parameters
        ----------
        link : :class:`~glue.core.component_link.ComponentLink`
            The link to use to generate a new component
        label : :class:`~glue.core.component_id.ComponentID` or str
            The ComponentID or label to attach to.

        Returns
        -------
        component : :class:`~glue.core.component.DerivedComponent`
            The component that was added
        """
        if label is not None:
            if not isinstance(label, ComponentID):
                label = ComponentID(label, parent=self)
            link.set_to_id(label)

        if link.get_to_id() is None:
            raise TypeError("Cannot add component_link: "
                            "has no 'to' ComponentID")

        for cid in link.get_from_ids():
            if cid not in self.components:
                raise ValueError(
                    "Can only add internal links with add_component_link "
                    "- use DataCollection.add_link to add inter-data links")

        dc = DerivedComponent(self, link)
        to_ = link.get_to_id()
        self.add_component(dc, label=to_)
        return dc

    def _create_pixel_and_world_components(self, ndim):

        for i in range(ndim):
            comp = CoordinateComponent(self, i)
            label = pixel_label(i, ndim)
            cid = PixelComponentID(i, "Pixel Axis %s" % label, parent=self)
            self.add_component(comp, cid)
            self._pixel_component_ids.append(cid)

        if self.coords:
            for i in range(ndim):
                comp = CoordinateComponent(self, i, world=True)
                label = self.coords.axis_label(i)
                cid = self.add_component(comp, label)
                self._world_component_ids.append(cid)
            self._set_up_coordinate_component_links(ndim)

    def _set_up_coordinate_component_links(self, ndim):
        def make_toworld_func(i):
            def pix2world(*args):
                return self.coords.pixel2world_single_axis(*args[::-1],
                                                           axis=ndim - 1 - i)

            return pix2world

        def make_topixel_func(i):
            def world2pix(*args):
                return self.coords.world2pixel_single_axis(*args[::-1],
                                                           axis=ndim - 1 - i)

            return world2pix

        result = []
        for i in range(ndim):
            link = CoordinateComponentLink(self._pixel_component_ids,
                                           self._world_component_ids[i],
                                           self.coords, i)
            result.append(link)
            link = CoordinateComponentLink(self._world_component_ids,
                                           self._pixel_component_ids[i],
                                           self.coords,
                                           i,
                                           pixel2world=False)
            result.append(link)

        self._coordinate_links = result
        return result

    @property
    def components(self):
        """All :class:`ComponentIDs <glue.core.component_id.ComponentID>` in the Data

        :rtype: list
        """
        return list(self._components.keys())

    @property
    def externally_derivable_components(self):
        return list(self._externally_derivable_components.keys())

    @property
    def visible_components(self):
        """All :class:`ComponentIDs <glue.core.component_id.ComponentID>` in the Data that aren't coordinate.

        :rtype: list
        """
        return [
            cid for cid, comp in self._components.items()
            if not isinstance(comp, CoordinateComponent) and cid.parent is self
        ]

    @property
    def coordinate_components(self):
        """The ComponentIDs associated with a :class:`~glue.core.component.CoordinateComponent`

        :rtype: list
        """
        return [
            c for c in self.component_ids()
            if isinstance(self._components[c], CoordinateComponent)
        ]

    @property
    def main_components(self):
        return [
            c for c in self.component_ids()
            if not isinstance(self._components[c], (DerivedComponent,
                                                    CoordinateComponent))
        ]

    @property
    def primary_components(self):
        """The ComponentIDs not associated with a :class:`~glue.core.component.DerivedComponent`

        :rtype: list
        """
        return [
            c for c in self.component_ids()
            if not isinstance(self._components[c], DerivedComponent)
        ]

    @property
    def derived_components(self):
        """The ComponentIDs for each :class:`~glue.core.component.DerivedComponent`

        :rtype: list
        """
        return [
            c for c in self.component_ids()
            if isinstance(self._components[c], DerivedComponent)
        ]

    @property
    def pixel_component_ids(self):
        """
        The :class:`ComponentIDs <glue.core.component_id.ComponentID>` for each pixel coordinate.
        """
        return self._pixel_component_ids

    @property
    def world_component_ids(self):
        """
        The :class:`ComponentIDs <glue.core.component_id.ComponentID>` for each world coordinate.
        """
        return self._world_component_ids

    @contract(label='cid_like', returns='inst($ComponentID)|None')
    def find_component_id(self, label):
        """ Retrieve component_ids associated by label name.

        :param label: ComponentID or string to search for

        :returns:
            The associated ComponentID if label is found and unique, else None.
            First, this checks whether the component ID is present and unique in
            the primary (non-derived) components of the data, and if not then
            the derived components are checked. If there is one instance of the
            label in the primary and one in the derived components, the primary
            one takes precedence.
        """

        for cid_set in (self.primary_components, self.derived_components,
                        self.coordinate_components,
                        list(self._externally_derivable_components)):

            result = []
            for cid in cid_set:
                if isinstance(label, ComponentID):
                    if cid is label:
                        result.append(cid)
                else:
                    if cid.label == label:
                        result.append(cid)

            if len(result) == 1:
                return result[0]
            elif len(result) > 1:
                return None
        return None

    @property
    def links(self):
        """
        A list of all the links internal to the dataset.
        """
        return self.coordinate_links + self.derived_links

    @property
    def coordinate_links(self):
        """
        A list of the ComponentLinks that connect pixel and world. If no
        coordinate transformation object is present, return an empty list.
        """
        return self._coordinate_links

    @property
    def derived_links(self):
        """
        A list of the links present inside all of the DerivedComponent objects
        in this dataset.
        """
        return [
            self.get_component(cid).link for cid in self.derived_components
        ]

    @contract(axis=int, returns=ComponentID)
    def get_pixel_component_id(self, axis):
        """Return the pixel :class:`glue.core.component_id.ComponentID` associated with a given axis
        """
        return self._pixel_component_ids[axis]

    @contract(axis=int, returns=ComponentID)
    def get_world_component_id(self, axis):
        """Return the world :class:`glue.core.component_id.ComponentID` associated with a given axis
        """
        return self._world_component_ids[axis]

    @contract(returns='list(inst($ComponentID))')
    def component_ids(self):
        """
        Equivalent to :attr:`Data.components`
        """
        return ComponentIDList(self._components.keys())

    @contract(subset='isinstance(Subset)|None',
              color='color|None',
              label='string|None',
              returns=Subset)
    def new_subset(self, subset=None, color=None, label=None, **kwargs):
        """
        Create a new subset, and attach to self.

        .. note:: The preferred way for creating subsets is via
            :meth:`~glue.core.data_collection.DataCollection.new_subset_group`.
            Manually-instantiated subsets will **not** be
            represented properly by the UI

        :param subset: optional, reference subset or subset state.
                       If provided, the new subset will copy the logic of
                       this subset.

        :returns: The new subset object
        """
        nsub = len(self.subsets)
        color = color or settings.SUBSET_COLORS[nsub %
                                                len(settings.SUBSET_COLORS)]
        label = label or "%s.%i" % (self.label, nsub + 1)
        new_subset = Subset(self, color=color, label=label, **kwargs)
        if subset is not None:
            new_subset.subset_state = subset.subset_state.copy()

        self.add_subset(new_subset)
        return new_subset

    @contract(subset='inst($Subset, $SubsetState)')
    def add_subset(self, subset):
        """Assign a pre-existing subset to this data object.

        :param subset: A :class:`~glue.core.subset.Subset` or
                       :class:`~glue.core.subset.SubsetState` object

        If input is a :class:`~glue.core.subset.SubsetState`,
        it will be wrapped in a new Subset automatically

        .. note:: The preferred way for creating subsets is via
            :meth:`~glue.core.data_collection.DataCollection.new_subset_group`.
            Manually-instantiated subsets will **not** be
            represented properly by the UI
        """

        if subset in self.subsets:
            return  # prevents infinite recursion
        if isinstance(subset, SubsetState):
            # auto-wrap state in subset
            state = subset
            subset = Subset(None)
            subset.subset_state = state

        self._subsets.append(subset)

        if subset.data is not self:
            subset.do_broadcast(False)
            subset.data = self
            subset.label = subset.label  # hacky. disambiguates name if needed

        if self.hub is not None:
            msg = SubsetCreateMessage(subset)
            self.hub.broadcast(msg)

        subset.do_broadcast(True)

    @contract(hub=Hub)
    def register_to_hub(self, hub):
        """ Connect to a hub.

        This method usually doesn't have to be called directly, as
        DataCollections manage the registration of data objects
        """
        if not isinstance(hub, Hub):
            raise TypeError("input is not a Hub object: %s" % type(hub))
        self.hub = hub

    @contract(attribute='string')
    def broadcast(self, attribute):
        """
        Send a :class:`~glue.core.message.DataUpdateMessage` to the hub

        :param attribute: Name of an attribute that has changed (or None)
        :type attribute: str
        """
        if not self.hub:
            return
        msg = DataUpdateMessage(self, attribute=attribute)
        self.hub.broadcast(msg)

    @contract(old=ComponentID, new=ComponentID)
    def update_id(self, old, new):
        """
        Reassign a component to a different :class:`glue.core.component_id.ComponentID`

        Parameters
        ----------
        old : :class:`glue.core.component_id.ComponentID`
            The old component ID
        new : :class:`glue.core.component_id.ComponentID`
            The new component ID
        """

        if new is old:
            return

        if new.parent is None:
            new.parent = self

        changed = False
        if old in self._components:

            # We want to keep the original order, so we can't just do:
            #   self._components[new] = self._components[old]
            # which will put the new component ID at the end, but instead
            # we need to do:
            self._components = OrderedDict(
                (new, value) if key is old else (key, value)
                for key, value in self._components.items())
            changed = True

        try:
            index = self._pixel_component_ids.index(old)
            self._pixel_component_ids[index] = new
            changed = True
        except ValueError:
            pass
        try:
            index = self._world_component_ids.index(old)
            self._world_component_ids[index] = new
            changed = True
        except ValueError:
            pass

        if changed and self.hub is not None:

            # remove old component and broadcast the change
            # see #508 for discussion of this
            msg = ComponentReplacedMessage(self, old, new)
            self.hub.broadcast(msg)

    def __str__(self):
        s = "Data Set: %s\n" % self.label
        s += "Number of dimensions: %i\n" % self.ndim
        s += "Shape: %s\n" % ' x '.join([str(x) for x in self.shape])
        categories = [('Main', self.main_components),
                      ('Derived', self.derived_components),
                      ('Coordinate', self.coordinate_components)]
        for category, components in categories:
            if len(components) > 0:
                s += category + " components:\n"
                for cid in components:
                    comp = self.get_component(cid)
                    if comp.units is None or comp.units == '':
                        s += " - {0}\n".format(cid)
                    else:
                        s += " - {0} [{1}]\n".format(cid, comp.units)
        return s[:-1]

    def __repr__(self):
        return 'Data (label: %s)' % self.label

    def __setattr__(self, name, value):
        if name == "hub" and hasattr(self, 'hub') \
                and self.hub is not value and self.hub is not None:
            raise AttributeError("Data has already been assigned "
                                 "to a different hub")
        object.__setattr__(self, name, value)

    def __getitem__(self, key):
        """ Shortcut syntax to access the numerical data in a component.
        Equivalent to:

        ``component = data.get_component(component_id).data``

        :param key:
          The component to fetch data from

        :type key: :class:`~glue.core.component_id.ComponentID`

        :returns: :class:`~numpy.ndarray`
        """

        key, view = split_component_view(key)
        if isinstance(key, six.string_types):
            _k = key
            key = self.find_component_id(key)
            if key is None:
                raise IncompatibleAttribute(_k)

        if isinstance(key, ComponentLink):
            return key.compute(self, view)

        if key in self._components:
            comp = self._components[key]
        elif key in self._externally_derivable_components:
            comp = self._externally_derivable_components[key]
        else:
            raise IncompatibleAttribute(key)

        if view is not None:
            result = comp[view]
        else:
            if comp.categorical:
                result = comp.codes
            else:
                result = comp.data

        return result

    def __setitem__(self, key, value):
        """
        Wrapper for data.add_component()
        """
        self.add_component(value, key)

    @contract(component_id='cid_like|None', returns=Component)
    def get_component(self, component_id):
        """Fetch the component corresponding to component_id.

        :param component_id: the component_id to retrieve
        """
        if component_id is None:
            raise IncompatibleAttribute()

        if isinstance(component_id, six.string_types):
            component_id = self.id[component_id]

        if component_id in self._components:
            return self._components[component_id]
        elif component_id in self._externally_derivable_components:
            return self._externally_derivable_components[component_id]
        else:
            raise IncompatibleAttribute(component_id)

    def to_dataframe(self, index=None):
        """ Convert the Data object into a pandas.DataFrame object

        :param index: Any 'index-like' object that can be passed to the pandas.Series constructor

        :return: pandas.DataFrame
        """

        h = lambda comp: self.get_component(comp).to_series(index=index)
        df = pd.DataFrame(
            dict((comp.label, h(comp)) for comp in self.components))
        order = [comp.label for comp in self.components]
        return df[order]

    def reorder_components(self, component_ids):
        """
        Reorder the components using a list of component IDs. The new set
        of component IDs has to match the existing set (though order may differ).
        """

        # We need to be careful because component IDs overload == so we can't
        # use the normal ways to test whether the component IDs are the same
        # as self.components - instead we need to explicitly use id

        if len(component_ids) != len(self.components):
            raise ValueError("Number of component in component_ids does not "
                             "match existing number of components")

        if set(id(c)
               for c in self.components) != set(id(c) for c in component_ids):
            raise ValueError(
                "specified component_ids should match existing components")

        existing = self.components
        for idx in range(len(component_ids)):
            if component_ids[idx] is not existing[idx]:
                break
        else:
            # If we get here then the suggested order is the same as the existing one
            return

        # PY3: once we drop support for Python 2 we could sort in-place using
        # the move_to_end method on OrderedDict
        self._components = OrderedDict(
            (key, self._components[key]) for key in component_ids)

        if self.hub:
            msg = DataReorderComponentMessage(self, list(self._components))
            self.hub.broadcast(msg)

    @contract(mapping="dict(inst($Component, $ComponentID):array_like)")
    def update_components(self, mapping):
        """
        Change the numerical data associated with some of the Components
        in this Data object.

        All changes to component numerical data should use this method,
        which broadcasts the state change to the appropriate places.

        :param mapping: A dict mapping Components or ComponenIDs to arrays.

        This method has the following restrictions:
          - New compoments must have the same shape as old compoments
          - Component subclasses cannot be updated.
        """
        for comp, data in mapping.items():
            if isinstance(comp, ComponentID):
                comp = self.get_component(comp)
            data = np.asarray(data)
            if data.shape != self.shape:
                raise ValueError("Cannot change shape of data")

            comp._data = data

        # alert hub of the change
        if self.hub is not None:
            msg = NumericalDataChangedMessage(self)
            self.hub.broadcast(msg)

        for subset in self.subsets:
            clear_cache(subset.subset_state.to_mask)

    def update_values_from_data(self, data):
        """
        Replace numerical values in data to match values from another dataset.

        Notes
        -----

        This method drops components that aren't present in the new data, and
        adds components that are in the new data that were not in the original
        data. The matching is done by component label, and components are
        resized if needed. This means that for components with matching labels
        in the original and new data, the
        :class:`~glue.core.component_id.ComponentID` are preserved, and
        existing plots and selections will be updated to reflect the new
        values. Note that the coordinates are also copied, but the style is
        **not** copied.
        """

        old_labels = [cid.label for cid in self.components]
        new_labels = [cid.label for cid in data.components]

        if len(old_labels) == len(set(old_labels)):
            old_labels = set(old_labels)
        else:
            raise ValueError("Non-unique component labels in original data")

        if len(new_labels) == len(set(new_labels)):
            new_labels = set(new_labels)
        else:
            raise ValueError("Non-unique component labels in new data")

        # Remove components that don't have a match in new data
        for cname in old_labels - new_labels:
            cid = self.find_component_id(cname)
            self.remove_component(cid)

        # Update shape
        self._shape = data._shape

        # Update components that exist in both. Note that we can't just loop
        # over old_labels & new_labels since we need to make sure we preserve
        # the order of the components, and sets don't preserve order.
        for cid in self.components:
            cname = cid.label
            if cname in old_labels & new_labels:
                comp_old = self.get_component(cname)
                comp_new = data.get_component(cname)
                comp_old._data = comp_new._data

        # Add components that didn't exist in original one. As above, we try
        # and preserve the order of components as much as possible.
        for cid in data.components:
            cname = cid.label
            if cname in new_labels - old_labels:
                cid = data.find_component_id(cname)
                comp_new = data.get_component(cname)
                self.add_component(comp_new, cid.label)

        # Update data label
        self.label = data.label

        # Update data coordinates
        self.coords = data.coords

        # alert hub of the change
        if self.hub is not None:
            msg = NumericalDataChangedMessage(self)
            self.hub.broadcast(msg)

        for subset in self.subsets:
            clear_cache(subset.subset_state.to_mask)
Exemplo n.º 47
0
async def cmd_player_compare(ctx):

	bot = ctx.bot
	args = ctx.args
	config = ctx.config

	ctx.alt = bot.options.parse_alt(args)
	lang = bot.options.parse_lang(ctx, args)

	selected_players, error = bot.options.parse_players(ctx, args)

	selected_units = bot.options.parse_unit_names(args)

	if error:
		return error

	if args:
		return bot.errors.unknown_parameters(args)

	if not selected_players:
		return bot.errors.no_ally_code_specified(ctx)

	if not selected_units:
		return bot.errors.no_unit_selected(ctx)

	ally_codes = [ x.ally_code for x in selected_players ]
	players = await bot.client.players(ally_codes=ally_codes, units=selected_units, stats=True)
	players = { x['allyCode']: x for x in players }

	msgs = []
	for unit in selected_units:

		units = []
		fields = OrderedDict()
		for player in selected_players:

			ally_code = player.ally_code
			jplayer = players[ally_code]
			jroster = { x['defId']: x for x in jplayer['roster'] }

			units.append(unit_to_dict(config, jplayer, jroster, unit.base_id, lang))

		for someunit in units:
			for key, val in someunit.items():
				if key not in fields:
					fields[key] = []
				fields[key].append(val)

		lines = []

		key = 'Unit still locked'
		if key in fields:
			listval = fields.pop(key)
			pad = 1
			lines.append('**`%s`**`:%s%s`' % (key, pad * '\u00a0', ' | '.join(listval)))
			if fields:
				lines.append('')

		first_time = True
		for key, listval in fields.items():

			if key in base_stats:

				newlistval = []
				for item in listval:
					pad = 0
					aval = item
					if len(item) < 7 and key != 'Players':
						pad = max(0, 6 - len(item))
						aval = '%s%s ' % (pad * '\u00a0', item)

					newlistval.append(aval)

				if key == 'Players':
					lines.append('**Stats**')
					lines.append(config['separator'])

				key_string = '**`%s`**' % key
				if key in [ 'Players', 'Stars' ]:
					key_string = ''
				lines.append('`|%s|`%s' % ('|'.join(newlistval), key_string))
			else:
				if first_time:
					first_time = False
					lines.append(config['separator'])
					lines.append('')
					lines.append('**Skills**')
					lines.append(config['separator'])

				ability = listval[0]['name']
				tiers = [ x['tier'] for x in listval ]
				lines.append('`|%s|`**`%s`**' % ('|'.join(tiers), ability))

		msgs.append({
			'author': {
				'name': unit.name,
				'icon_url': unit.get_image(),
			},
			'description': '\n'.join(lines),
		})

	return msgs
Exemplo n.º 48
0
class LoadScopeScheduling(object):
    """Implement load scheduling across nodes, but grouping test by scope.

    This distributes the tests collected across all nodes so each test is run
    just once.  All nodes collect and submit the list of tests and when all
    collections are received it is verified they are identical collections.
    Then the collection gets divided up in work units, grouped by test scope,
    and those work units get submitted to nodes.  Whenever a node finishes an
    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
    to assign more work units if the number of pending tests for the node falls
    below a low-watermark.

    When created, ``numnodes`` defines how many nodes are expected to submit a
    collection. This is used to know when all nodes have finished collection.

    Attributes:

    :numnodes: The expected number of nodes taking part.  The actual number of
       nodes will vary during the scheduler's lifetime as nodes are added by
       the DSession as they are brought up and removed either because of a dead
       node or normal shutdown.  This number is primarily used to know when the
       initial collection is completed.

    :collection: The final list of tests collected by all nodes once it is
       validated to be identical between all the nodes.  It is initialised to
       None until ``.schedule()`` is called.

    :workqueue: Ordered dictionary that maps all available scopes with their
       associated tests (nodeid). Nodeids are in turn associated with their
       completion status. One entry of the workqueue is called a work unit.
       In turn, a collection of work unit is called a workload.

       ::

            workqueue = {
                '<full>/<path>/<to>/test_module.py': {
                    '<full>/<path>/<to>/test_module.py::test_case1': False,
                    '<full>/<path>/<to>/test_module.py::test_case2': False,
                    (...)
                },
                (...)
            }

    :assigned_work: Ordered dictionary that maps worker nodes with their
       assigned work units.

       ::

            assigned_work = {
                '<worker node A>': {
                    '<full>/<path>/<to>/test_module.py': {
                        '<full>/<path>/<to>/test_module.py::test_case1': False,
                        '<full>/<path>/<to>/test_module.py::test_case2': False,
                        (...)
                    },
                    (...)
                },
                (...)
            }

    :registered_collections: Ordered dictionary that maps worker nodes with
       their collection of tests gathered during test discovery.

       ::

            registered_collections = {
                '<worker node A>': [
                    '<full>/<path>/<to>/test_module.py::test_case1',
                    '<full>/<path>/<to>/test_module.py::test_case2',
                ],
                (...)
            }

    :log: A py.log.Producer instance.

    :config: Config object, used for handling hooks.
    """
    def __init__(self, config, log=None):
        self.numnodes = len(parse_spec_config(config))
        self.collection = None

        self.workqueue = OrderedDict()
        self.assigned_work = OrderedDict()
        self.registered_collections = OrderedDict()

        if log is None:
            self.log = Producer("loadscopesched")
        else:
            self.log = log.loadscopesched

        self.config = config

    @property
    def nodes(self):
        """A list of all active nodes in the scheduler."""
        return list(self.assigned_work.keys())

    @property
    def collection_is_completed(self):
        """Boolean indication initial test collection is complete.

        This is a boolean indicating all initial participating nodes have
        finished collection.  The required number of initial nodes is defined
        by ``.numnodes``.
        """
        return len(self.registered_collections) >= self.numnodes

    @property
    def tests_finished(self):
        """Return True if all tests have been executed by the nodes."""
        if not self.collection_is_completed:
            return False

        if self.workqueue:
            return False

        for assigned_unit in self.assigned_work.values():
            if self._pending_of(assigned_unit) >= 2:
                return False

        return True

    @property
    def has_pending(self):
        """Return True if there are pending test items.

        This indicates that collection has finished and nodes are still
        processing test items, so this can be thought of as
        "the scheduler is active".
        """
        if self.workqueue:
            return True

        for assigned_unit in self.assigned_work.values():
            if self._pending_of(assigned_unit) > 0:
                return True

        return False

    def add_node(self, node):
        """Add a new node to the scheduler.

        From now on the node will be assigned work units to be executed.

        Called by the ``DSession.worker_workerready`` hook when it successfully
        bootstraps a new node.
        """
        assert node not in self.assigned_work
        self.assigned_work[node] = OrderedDict()

    def remove_node(self, node):
        """Remove a node from the scheduler.

        This should be called either when the node crashed or at shutdown time.
        In the former case any pending items assigned to the node will be
        re-scheduled.

        Called by the hooks:

        - ``DSession.worker_workerfinished``.
        - ``DSession.worker_errordown``.

        Return the item being executed while the node crashed or None if the
        node has no more pending items.
        """
        workload = self.assigned_work.pop(node)
        if not self._pending_of(workload):
            return None

        # The node crashed, identify test that crashed
        for work_unit in workload.values():
            for nodeid, completed in work_unit.items():
                if not completed:
                    crashitem = nodeid
                    break
            else:
                continue
            break
        else:
            raise RuntimeError(
                "Unable to identify crashitem on a workload with pending items"
            )

        # Made uncompleted work unit available again
        self.workqueue.update(workload)

        for node in self.assigned_work:
            self._reschedule(node)

        return crashitem

    def add_node_collection(self, node, collection):
        """Add the collected test items from a node.

        The collection is stored in the ``.registered_collections`` dictionary.

        Called by the hook:

        - ``DSession.worker_collectionfinish``.
        """

        # Check that add_node() was called on the node before
        assert node in self.assigned_work

        # A new node has been added later, perhaps an original one died.
        if self.collection_is_completed:

            # Assert that .schedule() should have been called by now
            assert self.collection

            # Check that the new collection matches the official collection
            if collection != self.collection:
                other_node = next(iter(self.registered_collections.keys()))

                msg = report_collection_diff(self.collection, collection,
                                             other_node.gateway.id,
                                             node.gateway.id)
                self.log(msg)
                return

        self.registered_collections[node] = list(collection)

    def mark_test_complete(self, node, item_index, duration=0):
        """Mark test item as completed by node.

        Called by the hook:

        - ``DSession.worker_testreport``.
        """
        nodeid = self.registered_collections[node][item_index]
        scope = self._split_scope(nodeid)

        self.assigned_work[node][scope][nodeid] = True
        self._reschedule(node)

    def _assign_work_unit(self, node):
        """Assign a work unit to a node."""
        assert self.workqueue

        # Grab a unit of work
        # scope, work_unit = self.workqueue.popitem(last=False)
        scope = next(iter(self.workqueue.keys()))
        if scope == 'test_acl_0052':
            # waiting other node stop
            for node in self.nodes:
                for nodeid, complete in self.assigned_work[node].items():
                    if not complete:
                        return

            for node in self.nodes:
                if node != node:
                    self.remove_node(node)
                    node.shutdown()
        scope, work_unit = self.workqueue.popitem(last=False)

        # Keep track of the assigned work
        assigned_to_node = self.assigned_work.setdefault(node,
                                                         default=OrderedDict())
        assigned_to_node[scope] = work_unit

        # Ask the node to execute the workload
        worker_collection = self.registered_collections[node]
        nodeids_indexes = [
            worker_collection.index(nodeid)
            for nodeid, completed in work_unit.items() if not completed
        ]

        node.send_runtest_some(nodeids_indexes)

    def _split_scope(self, nodeid):
        """Determine the scope (grouping) of a nodeid.

        There are usually 3 cases for a nodeid::

            example/loadsuite/test/test_beta.py::test_beta0
            example/loadsuite/test/test_delta.py::Delta1::test_delta0
            example/loadsuite/epsilon/__init__.py::epsilon.epsilon

        #. Function in a test module.
        #. Method of a class in a test module.
        #. Doctest in a function in a package.

        This function will group tests with the scope determined by splitting
        the first ``::`` from the right. That is, classes will be grouped in a
        single work unit, and functions from a test module will be grouped by
        their module. In the above example, scopes will be::

            example/loadsuite/test/test_beta.py
            example/loadsuite/test/test_delta.py::Delta1
            example/loadsuite/epsilon/__init__.py
        """
        return nodeid.rsplit("::", 1)[0]

    def _pending_of(self, workload):
        """Return the number of pending tests in a workload."""
        pending = sum(
            list(scope.values()).count(False) for scope in workload.values())
        return pending

    def _reschedule(self, node):
        """Maybe schedule new items on the node.

        If there are any globally pending work units left then this will check
        if the given node should be given any more tests.
        """

        # Do not add more work to a node shutting down
        if node.shutting_down:
            return

        # Check that more work is available
        if not self.workqueue:
            node.shutdown()
            return

        self.log("Number of units waiting for node:", len(self.workqueue))

        # Check that the node is almost depleted of work
        # 2: Heuristic of minimum tests to enqueue more work
        if self._pending_of(self.assigned_work[node]) > 2:
            return

        # Pop one unit of work and assign it
        self._assign_work_unit(node)

    def schedule(self):
        """Initiate distribution of the test collection.

        Initiate scheduling of the items across the nodes.  If this gets called
        again later it behaves the same as calling ``._reschedule()`` on all
        nodes so that newly added nodes will start to be used.

        If ``.collection_is_completed`` is True, this is called by the hook:

        - ``DSession.worker_collectionfinish``.
        """
        assert self.collection_is_completed

        # Initial distribution already happened, reschedule on all nodes
        if self.collection is not None:
            for node in self.nodes:
                self._reschedule(node)
            return

        # Check that all nodes collected the same tests
        if not self._check_nodes_have_same_collection():
            self.log("**Different tests collected, aborting run**")
            return

        # Collections are identical, create the final list of items
        self.collection = list(next(iter(
            self.registered_collections.values())))
        if not self.collection:
            return

        # Determine chunks of work (scopes)
        for nodeid in self.collection:
            scope = self._split_scope(nodeid)
            work_unit = self.workqueue.setdefault(scope, default=OrderedDict())
            work_unit[nodeid] = False

        # Avoid having more workers than work
        extra_nodes = len(self.nodes) - len(self.workqueue)

        if extra_nodes > 0:
            self.log("Shuting down {0} nodes".format(extra_nodes))

            for _ in range(extra_nodes):
                unused_node, assigned = self.assigned_work.popitem(last=True)

                self.log("Shuting down unused node {0}".format(unused_node))
                unused_node.shutdown()

        # Assign initial workload
        for node in self.nodes:
            self._assign_work_unit(node)

        # Ensure nodes start with at least two work units if possible (#277)
        for node in self.nodes:
            self._reschedule(node)

        # Initial distribution sent all tests, start node shutdown
        if not self.workqueue:
            for node in self.nodes:
                node.shutdown()

    def _check_nodes_have_same_collection(self):
        """Return True if all nodes have collected the same items.

        If collections differ, this method returns False while logging
        the collection differences and posting collection errors to
        pytest_collectreport hook.
        """
        node_collection_items = list(self.registered_collections.items())
        first_node, col = node_collection_items[0]
        same_collection = True

        for node, collection in node_collection_items[1:]:
            msg = report_collection_diff(col, collection,
                                         first_node.gateway.id,
                                         node.gateway.id)
            if not msg:
                continue

            same_collection = False
            self.log(msg)

            if self.config is None:
                continue

            rep = CollectReport(node.gateway.id,
                                "failed",
                                longrepr=msg,
                                result=[])
            self.config.hook.pytest_collectreport(report=rep)

        return same_collection
Exemplo n.º 49
0
class OWDataTable(OWWidget):
    name = "Data Table"
    description = "View the dataset in a spreadsheet."
    icon = "icons/Table.svg"
    priority = 50
    keywords = []

    buttons_area_orientation = Qt.Vertical

    class Inputs:
        data = Input("Data", Table, multiple=True)

    class Outputs:
        selected_data = Output("Selected Data", Table, default=True)
        annotated_data = Output(ANNOTATED_DATA_SIGNAL_NAME, Table)

    show_distributions = Setting(False)
    dist_color_RGB = Setting((220, 220, 220, 255))
    show_attribute_labels = Setting(True)
    select_rows = Setting(True)
    auto_commit = Setting(True)

    color_by_class = Setting(True)
    settingsHandler = DomainContextHandler(
        match_values=DomainContextHandler.MATCH_VALUES_ALL)
    selected_rows = Setting([], schema_only=True)
    selected_cols = Setting([], schema_only=True)

    def __init__(self):
        super().__init__()

        self._inputs = OrderedDict()

        self.__pending_selected_rows = self.selected_rows
        self.selected_rows = None
        self.__pending_selected_cols = self.selected_cols
        self.selected_cols = None

        self.dist_color = QColor(*self.dist_color_RGB)

        info_box = gui.vBox(self.controlArea, "Info")
        self.info_ex = gui.widgetLabel(info_box, 'No data on input.', )
        self.info_ex.setWordWrap(True)
        self.info_attr = gui.widgetLabel(info_box, ' ')
        self.info_attr.setWordWrap(True)
        self.info_class = gui.widgetLabel(info_box, ' ')
        self.info_class.setWordWrap(True)
        self.info_meta = gui.widgetLabel(info_box, ' ')
        self.info_meta.setWordWrap(True)
        info_box.setMinimumWidth(200)
        gui.separator(self.controlArea)

        box = gui.vBox(self.controlArea, "Variables")
        self.c_show_attribute_labels = gui.checkBox(
            box, self, "show_attribute_labels",
            "Show variable labels (if present)",
            callback=self._on_show_variable_labels_changed)

        gui.checkBox(box, self, "show_distributions",
                     'Visualize numeric values',
                     callback=self._on_distribution_color_changed)
        gui.checkBox(box, self, "color_by_class", 'Color by instance classes',
                     callback=self._on_distribution_color_changed)

        box = gui.vBox(self.controlArea, "Selection")

        gui.checkBox(box, self, "select_rows", "Select full rows",
                     callback=self._on_select_rows_changed)

        gui.rubber(self.controlArea)

        reset = gui.button(
            None, self, "Restore Original Order", callback=self.restore_order,
            tooltip="Show rows in the original order", autoDefault=False)
        self.buttonsArea.layout().insertWidget(0, reset)
        gui.auto_send(self.buttonsArea, self, "auto_commit")

        # GUI with tabs
        self.tabs = gui.tabWidget(self.mainArea)
        self.tabs.currentChanged.connect(self._on_current_tab_changed)

    def copy_to_clipboard(self):
        self.copy()

    @staticmethod
    def sizeHint():
        return QSize(800, 500)

    @Inputs.data
    def set_dataset(self, data, tid=None):
        """Set the input dataset."""
        self.closeContext()
        if data is not None:
            datasetname = getattr(data, "name", "Data")
            if tid in self._inputs:
                # update existing input slot
                slot = self._inputs[tid]
                view = slot.view
                # reset the (header) view state.
                view.setModel(None)
                view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder)
                assert self.tabs.indexOf(view) != -1
                self.tabs.setTabText(self.tabs.indexOf(view), datasetname)
            else:
                view = QTableView()
                view.setSortingEnabled(True)
                view.setHorizontalScrollMode(QTableView.ScrollPerPixel)

                if self.select_rows:
                    view.setSelectionBehavior(QTableView.SelectRows)

                header = view.horizontalHeader()
                header.setSectionsMovable(True)
                header.setSectionsClickable(True)
                header.setSortIndicatorShown(True)
                header.setSortIndicator(-1, Qt.AscendingOrder)

                # QHeaderView does not 'reset' the model sort column,
                # because there is no guaranty (requirement) that the
                # models understand the -1 sort column.
                def sort_reset(index, order):
                    if view.model() is not None and index == -1:
                        view.model().sort(index, order)

                header.sortIndicatorChanged.connect(sort_reset)
                self.tabs.addTab(view, datasetname)

            view.dataset = data
            self.tabs.setCurrentWidget(view)

            self._setup_table_view(view, data)
            slot = TableSlot(tid, data, table_summary(data), view)
            view._input_slot = slot  # pylint: disable=protected-access
            self._inputs[tid] = slot

            self.tabs.setCurrentIndex(self.tabs.indexOf(view))

            self.set_info(slot.summary)

            if isinstance(slot.summary.len, concurrent.futures.Future):
                def update(_):
                    QMetaObject.invokeMethod(
                        self, "_update_info", Qt.QueuedConnection)

                slot.summary.len.add_done_callback(update)

        elif tid in self._inputs:
            slot = self._inputs.pop(tid)
            view = slot.view
            view.hide()
            view.deleteLater()
            self.tabs.removeTab(self.tabs.indexOf(view))

            current = self.tabs.currentWidget()
            if current is not None:
                # pylint: disable=protected-access
                self.set_info(current._input_slot.summary)

        self.tabs.tabBar().setVisible(self.tabs.count() > 1)
        self.openContext(data)

        if data and self.__pending_selected_rows is not None:
            self.selected_rows = self.__pending_selected_rows
            self.__pending_selected_rows = None
        else:
            self.selected_rows = []

        if data and self.__pending_selected_cols is not None:
            self.selected_cols = self.__pending_selected_cols
            self.__pending_selected_cols = None
        else:
            self.selected_cols = []

        self.set_selection()
        self.unconditional_commit()

    def _setup_table_view(self, view, data):
        """Setup the `view` (QTableView) with `data` (Orange.data.Table)
        """
        if data is None:
            view.setModel(None)
            return

        datamodel = RichTableModel(data)

        rowcount = data.approx_len()

        if self.color_by_class and data.domain.has_discrete_class:
            color_schema = [
                QColor(*c) for c in data.domain.class_var.colors]
        else:
            color_schema = None
        if self.show_distributions:
            view.setItemDelegate(
                gui.TableBarItem(
                    self, color=self.dist_color, color_schema=color_schema)
            )
        else:
            view.setItemDelegate(QStyledItemDelegate(self))

        # Enable/disable view sorting based on data's type
        view.setSortingEnabled(is_sortable(data))
        header = view.horizontalHeader()
        header.setSectionsClickable(is_sortable(data))
        header.setSortIndicatorShown(is_sortable(data))

        view.setModel(datamodel)

        vheader = view.verticalHeader()
        option = view.viewOptions()
        size = view.style().sizeFromContents(
            QStyle.CT_ItemViewItem, option,
            QSize(20, 20), view)

        vheader.setDefaultSectionSize(size.height() + 2)
        vheader.setMinimumSectionSize(5)
        vheader.setSectionResizeMode(QHeaderView.Fixed)

        # Limit the number of rows displayed in the QTableView
        # (workaround for QTBUG-18490 / QTBUG-28631)
        maxrows = (2 ** 31 - 1) // (vheader.defaultSectionSize() + 2)
        if rowcount > maxrows:
            sliceproxy = TableSliceProxy(
                parent=view, rowSlice=slice(0, maxrows))
            sliceproxy.setSourceModel(datamodel)
            # First reset the view (without this the header view retains
            # it's state - at this point invalid/broken)
            view.setModel(None)
            view.setModel(sliceproxy)

        assert view.model().rowCount() <= maxrows
        assert vheader.sectionSize(0) > 1 or datamodel.rowCount() == 0

        # update the header (attribute names)
        self._update_variable_labels(view)

        selmodel = BlockSelectionModel(
            view.model(), parent=view, selectBlocks=not self.select_rows)
        view.setSelectionModel(selmodel)
        view.selectionModel().selectionChanged.connect(self.update_selection)

    #noinspection PyBroadException
    def set_corner_text(self, table, text):
        """Set table corner text."""
        # As this is an ugly hack, do everything in
        # try - except blocks, as it may stop working in newer Qt.
        # pylint: disable=broad-except
        if not hasattr(table, "btn") and not hasattr(table, "btnfailed"):
            try:
                btn = table.findChild(QAbstractButton)

                class Efc(QObject):
                    @staticmethod
                    def eventFilter(o, e):
                        if (isinstance(o, QAbstractButton) and
                                e.type() == QEvent.Paint):
                            # paint by hand (borrowed from QTableCornerButton)
                            btn = o
                            opt = QStyleOptionHeader()
                            opt.initFrom(btn)
                            state = QStyle.State_None
                            if btn.isEnabled():
                                state |= QStyle.State_Enabled
                            if btn.isActiveWindow():
                                state |= QStyle.State_Active
                            if btn.isDown():
                                state |= QStyle.State_Sunken
                            opt.state = state
                            opt.rect = btn.rect()
                            opt.text = btn.text()
                            opt.position = QStyleOptionHeader.OnlyOneSection
                            painter = QStylePainter(btn)
                            painter.drawControl(QStyle.CE_Header, opt)
                            return True     # eat event
                        return False
                table.efc = Efc()
                # disconnect default handler for clicks and connect a new one, which supports
                # both selection and deselection of all data
                btn.clicked.disconnect()
                btn.installEventFilter(table.efc)
                btn.clicked.connect(self._on_select_all)
                table.btn = btn

                if sys.platform == "darwin":
                    btn.setAttribute(Qt.WA_MacSmallSize)

            except Exception:
                table.btnfailed = True

        if hasattr(table, "btn"):
            try:
                btn = table.btn
                btn.setText(text)
                opt = QStyleOptionHeader()
                opt.text = btn.text()
                s = btn.style().sizeFromContents(
                    QStyle.CT_HeaderSection,
                    opt, QSize(),
                    btn).expandedTo(QApplication.globalStrut())
                if s.isValid():
                    table.verticalHeader().setMinimumWidth(s.width())
            except Exception:
                pass

    def _on_select_all(self, _):
        # pylint: disable=protected-access
        data_info = self.tabs.currentWidget()._input_slot.summary
        if len(self.selected_rows) == data_info.len \
                and len(self.selected_cols) == len(data_info.domain):
            self.tabs.currentWidget().clearSelection()
        else:
            self.tabs.currentWidget().selectAll()

    def _on_current_tab_changed(self, index):
        """Update the info box on current tab change"""
        view = self.tabs.widget(index)
        if view is not None and view.model() is not None:
            # pylint: disable=protected-access
            self.set_info(view._input_slot.summary)
        else:
            self.set_info(None)

    def _update_variable_labels(self, view):
        "Update the variable labels visibility for `view`"
        model = view.model()
        if isinstance(model, TableSliceProxy):
            model = model.sourceModel()

        if self.show_attribute_labels:
            model.setRichHeaderFlags(
                RichTableModel.Labels | RichTableModel.Name)

            labelnames = set()
            domain = model.source.domain
            for a in itertools.chain(domain.metas, domain.variables):
                labelnames.update(a.attributes.keys())
            labelnames = sorted(
                [label for label in labelnames if not label.startswith("_")])
            self.set_corner_text(view, "\n".join([""] + labelnames))
        else:
            model.setRichHeaderFlags(RichTableModel.Name)
            self.set_corner_text(view, "")

    def _on_show_variable_labels_changed(self):
        """The variable labels (var.attribues) visibility was changed."""
        for slot in self._inputs.values():
            self._update_variable_labels(slot.view)

    def _on_distribution_color_changed(self):
        for ti in range(self.tabs.count()):
            widget = self.tabs.widget(ti)
            model = widget.model()
            while isinstance(model, QAbstractProxyModel):
                model = model.sourceModel()
            data = model.source
            class_var = data.domain.class_var
            if self.color_by_class and class_var and class_var.is_discrete:
                color_schema = [QColor(*c) for c in class_var.colors]
            else:
                color_schema = None
            if self.show_distributions:
                delegate = gui.TableBarItem(self, color=self.dist_color,
                                            color_schema=color_schema)
            else:
                delegate = QStyledItemDelegate(self)
            widget.setItemDelegate(delegate)
        tab = self.tabs.currentWidget()
        if tab:
            tab.reset()

    def _on_select_rows_changed(self):
        for slot in self._inputs.values():
            selection_model = slot.view.selectionModel()
            selection_model.setSelectBlocks(not self.select_rows)
            if self.select_rows:
                slot.view.setSelectionBehavior(QTableView.SelectRows)
                # Expand the current selection to full row selection.
                selection_model.select(
                    selection_model.selection(),
                    QItemSelectionModel.Select | QItemSelectionModel.Rows
                )
            else:
                slot.view.setSelectionBehavior(QTableView.SelectItems)

    def restore_order(self):
        """Restore the original data order of the current view."""
        table = self.tabs.currentWidget()
        if table is not None:
            table.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder)

    def set_info(self, summary):
        if summary is None:
            self.info_ex.setText("No data on input.")
            self.info_attr.setText("")
            self.info_class.setText("")
            self.info_meta.setText("")
        else:
            info_len, info_attr, info_class, info_meta = \
                format_summary(summary)

            self.info_ex.setText(info_len)
            self.info_attr.setText(info_attr)
            self.info_class.setText(info_class)
            self.info_meta.setText(info_meta)

    @Slot()
    def _update_info(self):
        current = self.tabs.currentWidget()
        if current is not None and current.model() is not None:
            # pylint: disable=protected-access
            self.set_info(current._input_slot.summary)

    def update_selection(self, *_):
        self.commit()

    def set_selection(self):
        if self.selected_rows and self.selected_cols:
            view = self.tabs.currentWidget()
            model = view.model()
            if model.rowCount() <= self.selected_rows[-1] or \
                    model.columnCount() <= self.selected_cols[-1]:
                return

            selection = QItemSelection()
            rowranges = list(ranges(self.selected_rows))
            colranges = list(ranges(self.selected_cols))

            for rowstart, rowend in rowranges:
                for colstart, colend in colranges:
                    selection.append(
                        QItemSelectionRange(
                            view.model().index(rowstart, colstart),
                            view.model().index(rowend - 1, colend - 1)
                        )
                    )
            view.selectionModel().select(
                selection, QItemSelectionModel.ClearAndSelect)

    @staticmethod
    def get_selection(view):
        """
        Return the selected row and column indices of the selection in view.
        """
        selmodel = view.selectionModel()

        selection = selmodel.selection()
        model = view.model()
        # map through the proxies into input table.
        while isinstance(model, QAbstractProxyModel):
            selection = model.mapSelectionToSource(selection)
            model = model.sourceModel()

        assert isinstance(selmodel, BlockSelectionModel)
        assert isinstance(model, TableModel)

        row_spans, col_spans = selection_blocks(selection)
        rows = list(itertools.chain.from_iterable(itertools.starmap(range, row_spans)))
        cols = list(itertools.chain.from_iterable(itertools.starmap(range, col_spans)))
        rows = numpy.array(rows, dtype=numpy.intp)
        # map the rows through the applied sorting (if any)
        rows = model.mapToSourceRows(rows)
        rows.sort()
        rows = rows.tolist()
        return rows, cols

    @staticmethod
    def _get_model(view):
        model = view.model()
        while isinstance(model, QAbstractProxyModel):
            model = model.sourceModel()
        return model

    def commit(self):
        """
        Commit/send the current selected row/column selection.
        """
        selected_data = table = rowsel = None
        view = self.tabs.currentWidget()
        if view and view.model() is not None:
            model = self._get_model(view)
            table = model.source  # The input data table

            # Selections of individual instances are not implemented
            # for SqlTables
            if isinstance(table, SqlTable):
                self.Outputs.selected_data.send(selected_data)
                self.Outputs.annotated_data.send(None)
                return

            rowsel, colsel = self.get_selection(view)
            self.selected_rows, self.selected_cols = rowsel, colsel

            def select(data, rows, domain):
                """
                Select the data subset with specified rows and domain subsets.

                If either rows or domain is None they mean select all.
                """
                if rows is not None and domain is not None:
                    return data.from_table(domain, data, rows)
                elif rows is not None:
                    return data.from_table(data.domain, rows)
                elif domain is not None:
                    return data.from_table(domain, data)
                else:
                    return data

            domain = table.domain

            if len(colsel) < len(domain) + len(domain.metas):
                # only a subset of the columns is selected
                allvars = domain.class_vars + domain.metas + domain.attributes
                columns = [(c, model.headerData(c, Qt.Horizontal,
                                                TableModel.DomainRole))
                           for c in colsel]
                assert all(role is not None for _, role in columns)

                def select_vars(role):
                    """select variables for role (TableModel.DomainRole)"""
                    return [allvars[c] for c, r in columns if r == role]

                attrs = select_vars(TableModel.Attribute)
                if attrs and issparse(table.X):
                    # for sparse data you can only select all attributes
                    attrs = table.domain.attributes
                class_vars = select_vars(TableModel.ClassVar)
                metas = select_vars(TableModel.Meta)
                domain = Orange.data.Domain(attrs, class_vars, metas)

            # Avoid a copy if all/none rows are selected.
            if not rowsel:
                selected_data = None
            elif len(rowsel) == len(table):
                selected_data = select(table, None, domain)
            else:
                selected_data = select(table, rowsel, domain)

        self.Outputs.selected_data.send(selected_data)
        self.Outputs.annotated_data.send(create_annotated_table(table, rowsel))

    def copy(self):
        """
        Copy current table selection to the clipboard.
        """
        view = self.tabs.currentWidget()
        if view is not None:
            mime = table_selection_to_mime_data(view)
            QApplication.clipboard().setMimeData(
                mime, QClipboard.Clipboard
            )

    def send_report(self):
        view = self.tabs.currentWidget()
        if not view or not view.model():
            return
        model = self._get_model(view)
        self.report_data_brief(model.source)
        self.report_table(view)
Exemplo n.º 50
0
class BaseFlow(JAMLCompatible, ExitStack, metaclass=FlowType):
    """An abstract Flow object in Jina.

    .. note::

        :class:`BaseFlow` does not provide `train`, `index`, `search` interfaces.
        Please use :class:`Flow` or :class:`AsyncFlow`.

    Explanation on ``optimize_level``:

    As an example, the following Flow will generate 6 Peas,

    .. highlight:: python
    .. code-block:: python

        f = Flow(optimize_level=FlowOptimizeLevel.NONE).add(uses='forward', parallel=3)

    The optimized version, i.e. :code:`Flow(optimize_level=FlowOptimizeLevel.FULL)`
    will generate 4 Peas, but it will force the :class:`GatewayPea` to take BIND role,
    as the head and tail routers are removed.

    :param kwargs: other keyword arguments that will be shared by all Pods in this Flow
    :param args: Namespace args
    :param env: environment variables shared by all Pods
    """

    _cls_client = Client  #: the type of the Client, can be changed to other class

    def __init__(self,
                 args: Optional['argparse.Namespace'] = None,
                 env: Optional[Dict] = None,
                 **kwargs):
        """Initialize a Flow object"""
        super().__init__()
        self._version = '1'  #: YAML version number, this will be later overridden if YAML config says the other way
        self._pod_nodes = OrderedDict()  # type: Dict[str, 'BasePod']
        self._inspect_pods = {}  # type: Dict[str, str]
        self._build_level = FlowBuildLevel.EMPTY
        self._last_changed_pod = [
            'gateway'
        ]  #: default first pod is gateway, will add when build()
        self._update_args(args, **kwargs)
        self._env = env
        if isinstance(self.args, argparse.Namespace):
            self.logger = JinaLogger(self.__class__.__name__,
                                     **vars(self.args))
        else:
            self.logger = JinaLogger(self.__class__.__name__)

    def _update_args(self, args, **kwargs):
        from ..parsers.flow import set_flow_parser
        from ..helper import ArgNamespace

        _flow_parser = set_flow_parser()
        if args is None:
            args = ArgNamespace.kwargs2namespace(kwargs, _flow_parser)
        self.args = args
        self._common_kwargs = kwargs
        self._kwargs = ArgNamespace.get_non_defaults_args(
            args, _flow_parser)  #: for yaml dump

    @property
    def yaml_spec(self):
        """
        get the YAML representation of the instance
        # noqa: DAR401
        # noqa: DAR201
        """
        return JAML.dump(self)

    @staticmethod
    def _parse_endpoints(op_flow,
                         pod_name,
                         endpoint,
                         connect_to_last_pod=False) -> Set:
        # parsing needs
        if isinstance(endpoint, str):
            endpoint = [endpoint]
        elif not endpoint:
            if op_flow._last_changed_pod and connect_to_last_pod:
                endpoint = [op_flow.last_pod]
            else:
                endpoint = []

        if isinstance(endpoint, (list, tuple)):
            for idx, s in enumerate(endpoint):
                if s == pod_name:
                    raise FlowTopologyError(
                        'the income/output of a pod can not be itself')
        else:
            raise ValueError(f'endpoint={endpoint} is not parsable')

        # if an endpoint is being inspected, then replace it with inspected Pod
        endpoint = set(op_flow._inspect_pods.get(ep, ep) for ep in endpoint)
        return endpoint

    @property
    def last_pod(self):
        """Last pod
        # noqa: DAR401
        # noqa: DAR201
        """
        return self._last_changed_pod[-1]

    @last_pod.setter
    def last_pod(self, name: str):
        """
        Set a Pod as the last Pod in the Flow, useful when modifying the Flow.

        # noqa: DAR401
        :param name: the name of the existing Pod
        """
        if name not in self._pod_nodes:
            raise FlowMissingPodError(f'{name} can not be found in this Flow')

        if self._last_changed_pod and name == self.last_pod:
            pass
        else:
            self._last_changed_pod.append(name)

        # graph is now changed so we need to
        # reset the build level to the lowest
        self._build_level = FlowBuildLevel.EMPTY

    def _add_gateway(self, needs, **kwargs):
        pod_name = 'gateway'

        kwargs.update(
            dict(
                name=pod_name,
                ctrl_with_ipc=True,  # otherwise ctrl port would be conflicted
                read_only=True,
                runtime_cls='GRPCRuntime',
                pod_role=PodRoleType.GATEWAY,
                identity=self.args.identity))

        kwargs.update(self._common_kwargs)
        args = ArgNamespace.kwargs2namespace(kwargs, set_gateway_parser())

        self._pod_nodes[pod_name] = BasePod(args, needs)

    def needs(self,
              needs: Union[Tuple[str], List[str]],
              name: str = 'joiner',
              *args,
              **kwargs) -> 'BaseFlow':
        """
        Add a blocker to the Flow, wait until all peas defined in **needs** completed.

        # noqa: DAR401
        :param needs: list of service names to wait
        :param name: the name of this joiner, by default is ``joiner``
        :param *args: *args for .add
        :param **kwargs: **kwargs for .add
        :return: the modified Flow
        """
        if len(needs) <= 1:
            raise FlowTopologyError(
                'no need to wait for a single service, need len(needs) > 1')
        return self.add(name=name,
                        needs=needs,
                        pod_role=PodRoleType.JOIN,
                        *args,
                        **kwargs)

    def needs_all(self, name: str = 'joiner', *args, **kwargs) -> 'BaseFlow':
        """
        Collect all hanging Pods so far and add a blocker to the Flow; wait until all handing peas completed.
        
        :param name: the name of this joiner (default is ``joiner``)
        :param *args: *args for .add or .needs
        :param **kwargs: **kwargs for .add or .needs
        :return: the modified Flow
        """
        needs = _hanging_pods(self)
        if len(needs) == 1:
            return self.add(name=name, needs=needs, *args, **kwargs)

        return self.needs(name=name, needs=needs, *args, **kwargs)

    def add(self,
            needs: Union[str, Tuple[str], List[str]] = None,
            copy_flow: bool = True,
            pod_role: 'PodRoleType' = PodRoleType.POD,
            **kwargs) -> 'BaseFlow':
        """
        Add a Pod to the current Flow object and return the new modified Flow object.
        The attribute of the Pod can be later changed with :py:meth:`set` or deleted with :py:meth:`remove`

        Note there are shortcut versions of this method.
        Recommend to use :py:meth:`add_encoder`, :py:meth:`add_preprocessor`,
        :py:meth:`add_router`, :py:meth:`add_indexer` whenever possible.

        # noqa: DAR401
        :param needs: the name of the Pod(s) that this Pod receives data from.
                           One can also use 'pod.Gateway' to indicate the connection with the gateway.
        :param pod_role: the role of the Pod, used for visualization and route planning
        :param copy_flow: when set to true, then always copy the current Flow and do the modification on top of it then return, otherwise, do in-line modification
        :param **kwargs: other keyword-value arguments that the Pod CLI supports
        :return: a (new) Flow object with modification
        """

        op_flow = copy.deepcopy(self) if copy_flow else self

        # pod naming logic
        pod_name = kwargs.get('name')

        if pod_name in op_flow._pod_nodes:
            new_name = f'{pod_name}{len(op_flow._pod_nodes)}'
            self.logger.debug(
                f'"{pod_name}" is used in this Flow already! renamed it to "{new_name}"'
            )
            pod_name = new_name

        if not pod_name:
            pod_name = f'pod{len(op_flow._pod_nodes)}'

        if not pod_name.isidentifier():
            # hyphen - can not be used in the name
            raise ValueError(
                f'name: {pod_name} is invalid, please follow the python variable name conventions'
            )

        # needs logic
        needs = op_flow._parse_endpoints(op_flow,
                                         pod_name,
                                         needs,
                                         connect_to_last_pod=True)

        # set the kwargs inherit from `Flow(kwargs1=..., kwargs2=)`
        for key, value in op_flow._common_kwargs.items():
            if key not in kwargs:
                kwargs[key] = value

        # check if host is set to remote:port
        if 'host' in kwargs:
            m = re.match(_regex_port, kwargs['host'])
            if kwargs.get(
                    'host', __default_host__
            ) != __default_host__ and m and 'port_expose' not in kwargs:
                kwargs['port_expose'] = m.group(2)
                kwargs['host'] = m.group(1)

        # update kwargs of this Pod
        kwargs.update(
            dict(name=pod_name, pod_role=pod_role, num_part=len(needs)))

        parser = set_pod_parser()
        if pod_role == PodRoleType.GATEWAY:
            parser = set_gateway_parser()

        args = ArgNamespace.kwargs2namespace(kwargs, parser)

        op_flow._pod_nodes[pod_name] = BasePod(args, needs=needs)
        op_flow.last_pod = pod_name

        return op_flow

    def inspect(self, name: str = 'inspect', *args, **kwargs) -> 'BaseFlow':
        """Add an inspection on the last changed Pod in the Flow

        Internally, it adds two Pods to the Flow. But don't worry, the overhead is minimized and you
        can remove them by simply using `Flow(inspect=FlowInspectType.REMOVE)` before using the Flow.

        .. highlight:: bash
        .. code-block:: bash

            Flow -- PUB-SUB -- BasePod(_pass) -- Flow
                    |
                    -- PUB-SUB -- InspectPod (Hanging)

        In this way, :class:`InspectPod` looks like a simple ``_pass`` from outside and
        does not introduce side-effects (e.g. changing the socket type) to the original Flow.
        The original incoming and outgoing socket types are preserved.

        This function is very handy for introducing an Evaluator into the Flow.

        .. seealso::

            :meth:`gather_inspect`

        :param name: name of the Pod
        :param *args: *args for .add()
        :param **kwargs: **kwargs for .add()
        :return: the new instance of the Flow
        """
        _last_pod = self.last_pod
        op_flow = self.add(name=name,
                           needs=_last_pod,
                           pod_role=PodRoleType.INSPECT,
                           *args,
                           **kwargs)

        # now remove uses and add an auxiliary Pod
        if 'uses' in kwargs:
            kwargs.pop('uses')
        op_flow = op_flow.add(name=f'_aux_{name}',
                              needs=_last_pod,
                              pod_role=PodRoleType.INSPECT_AUX_PASS,
                              *args,
                              **kwargs)

        # register any future connection to _last_pod by the auxiliary Pod
        op_flow._inspect_pods[_last_pod] = op_flow.last_pod

        return op_flow

    def gather_inspect(self,
                       name: str = 'gather_inspect',
                       uses='_merge_eval',
                       include_last_pod: bool = True,
                       *args,
                       **kwargs) -> 'BaseFlow':
        """ Gather all inspect Pods output into one Pod. When the Flow has no inspect Pod then the Flow itself
        is returned.

        .. note::

            If ``--no-inspect`` is **not** given, then :meth:`gather_inspect` is auto called before :meth:`build`. So
            in general you don't need to manually call :meth:`gather_inspect`.

        :param name: the name of the gather Pod
        :param uses: the config of the executor, by default is ``_pass``
        :param include_last_pod: if to include the last modified Pod in the Flow
        :param *args: *args for .add()
        :param **kwargs: **kwargs for .add()
        :return: the modified Flow or the copy of it


        .. seealso::

            :meth:`inspect`

        """
        needs = [
            k for k, v in self._pod_nodes.items()
            if v.role == PodRoleType.INSPECT
        ]
        if needs:
            if include_last_pod:
                needs.append(self.last_pod)
            return self.add(name=name,
                            uses=uses,
                            needs=needs,
                            pod_role=PodRoleType.JOIN_INSPECT,
                            *args,
                            **kwargs)
        else:
            # no inspect node is in the graph, return the current graph
            return self

    def build(self, copy_flow: bool = False) -> 'BaseFlow':
        """
        Build the current Flow and make it ready to use

        .. note::

            No need to manually call it since 0.0.8. When using Flow with the
            context manager, or using :meth:`start`, :meth:`build` will be invoked.

        :param copy_flow: when set to true, then always copy the current Flow and do the modification on top of it then return, otherwise, do in-line modification
        :return: the current Flow (by default)

        .. note::
            ``copy_flow=True`` is recommended if you are building the same Flow multiple times in a row. e.g.

            .. highlight:: python
            .. code-block:: python

                f = Flow()
                with f:
                    f.index()

                with f.build(copy_flow=True) as fl:
                    fl.search()

        # noqa: DAR401
        """

        op_flow = copy.deepcopy(self) if copy_flow else self

        _pod_edges = set()
        if op_flow.args.inspect == FlowInspectType.COLLECT:
            op_flow.gather_inspect(copy_flow=False)

        if 'gateway' not in op_flow._pod_nodes:
            op_flow._add_gateway(needs={op_flow.last_pod})

        # construct a map with a key a start node and values an array of its end nodes
        _outgoing_map = defaultdict(list)

        # if set no_inspect then all inspect related nodes are removed
        if op_flow.args.inspect == FlowInspectType.REMOVE:
            op_flow._pod_nodes = {
                k: v
                for k, v in op_flow._pod_nodes.items() if not v.role.is_inspect
            }
            reverse_inspect_map = {
                v: k
                for k, v in op_flow._inspect_pods.items()
            }

        for end, pod in op_flow._pod_nodes.items():
            # if an endpoint is being inspected, then replace it with inspected Pod
            # but not those inspect related node
            if op_flow.args.inspect.is_keep:
                pod.needs = set(ep if pod.role.is_inspect else op_flow.
                                _inspect_pods.get(ep, ep) for ep in pod.needs)
            else:
                pod.needs = set(
                    reverse_inspect_map.get(ep, ep) for ep in pod.needs)

            for start in pod.needs:
                if start not in op_flow._pod_nodes:
                    raise FlowMissingPodError(
                        f'{start} is not in this flow, misspelled name?')
                _outgoing_map[start].append(end)
                _pod_edges.add((start, end))

        op_flow = _build_flow(op_flow, _outgoing_map)
        op_flow = _optimize_flow(op_flow, _outgoing_map, _pod_edges)
        hanging_pods = _hanging_pods(op_flow)
        if hanging_pods:
            self.logger.warning(
                f'{hanging_pods} are hanging in this flow with no pod receiving from them, '
                f'you may want to double check if it is intentional or some mistake'
            )
        op_flow._build_level = FlowBuildLevel.GRAPH
        self._update_client()
        return op_flow

    def __call__(self, *args, **kwargs):
        """Builds the Flow
        :param *args: *args for build
        :param **kwargs: **kwargs for build
        :return: the built Flow
        """
        return self.build(*args, **kwargs)

    def __enter__(self):
        return self.start()

    def __exit__(self, exc_type, exc_val, exc_tb):
        super().__exit__(exc_type, exc_val, exc_tb)

        # unset all envs to avoid any side-effect
        if self._env:
            for k in self._env.keys():
                os.unsetenv(k)

        self._pod_nodes.pop('gateway')
        self._build_level = FlowBuildLevel.EMPTY
        self.logger.success(
            f'flow is closed and all resources are released, current build level is {self._build_level}'
        )
        self.logger.close()

    def start(self):
        """Start to run all Pods in this Flow.

        Remember to close the Flow with :meth:`close`.

        Note that this method has a timeout of ``timeout_ready`` set in CLI,
        which is inherited all the way from :class:`jina.peapods.peas.BasePea`

        # noqa: DAR401

        :return: this instance
        """

        if self._build_level.value < FlowBuildLevel.GRAPH.value:
            self.build(copy_flow=False)

        # set env only before the Pod get started
        if self._env:
            for k, v in self._env.items():
                os.environ[k] = str(v)

        for k, v in self:
            v.args.noblock_on_start = True
            self.enter_context(v)

        for k, v in self:
            try:
                v.wait_start_success()
            except Exception as ex:
                self.logger.error(
                    f'{k}:{v!r} can not be started due to {ex!r}, Flow is aborted'
                )
                self.close()
                raise

        self.logger.info(
            f'{self.num_pods} Pods (i.e. {self.num_peas} Peas) are running in this Flow'
        )

        self._show_success_message()

        return self

    @property
    def num_pods(self) -> int:
        """Get the number of Pods in this Flow
        # noqa: DAR201"""
        return len(self._pod_nodes)

    @property
    def num_peas(self) -> int:
        """Get the number of peas (parallel count) in this Flow
        # noqa: DAR201"""
        return sum(v.num_peas for v in self._pod_nodes.values())

    def __eq__(self, other: 'BaseFlow') -> bool:
        """
        Compare the topology of a Flow with another Flow.
        Identification is defined by whether two flows share the same set of edges.

        :param other: the second Flow object
        :return: result of equality check
        """

        if self._build_level.value < FlowBuildLevel.GRAPH.value:
            a = self.build()
        else:
            a = self

        if other._build_level.value < FlowBuildLevel.GRAPH.value:
            b = other.build()
        else:
            b = other

        return a._pod_nodes == b._pod_nodes

    @build_required(FlowBuildLevel.GRAPH)
    def _get_client(self, **kwargs) -> 'Client':
        kwargs.update(self._common_kwargs)
        if 'port_expose' not in kwargs:
            kwargs['port_expose'] = self.port_expose
        if 'host' not in kwargs:
            kwargs['host'] = self.host

        args = ArgNamespace.kwargs2namespace(kwargs, set_client_cli_parser())
        return self._cls_client(args)

    @property
    def _mermaid_str(self):
        mermaid_graph = [
            "%%{init: {'theme': 'base', "
            "'themeVariables': { 'primaryColor': '#32C8CD', "
            "'edgeLabelBackground':'#fff', 'clusterBkg': '#FFCC66'}}}%%",
            'graph LR'
        ]

        start_repl = {}
        end_repl = {}
        for node, v in self._pod_nodes.items():
            if not v.is_singleton and v.role != PodRoleType.GATEWAY:
                mermaid_graph.append(
                    f'subgraph sub_{node} ["{node} ({v.args.parallel})"]')
                if v.is_head_router:
                    head_router = node + '_HEAD'
                    end_repl[node] = (head_router, '((fa:fa-random))')
                if v.is_tail_router:
                    tail_router = node + '_TAIL'
                    start_repl[node] = (tail_router, '((fa:fa-random))')

                p_r = '((%s))'
                p_e = '[[%s]]'
                for j in range(v.args.parallel):
                    r = node + (f'_{j}' if v.args.parallel > 1 else '')
                    if v.is_head_router:
                        mermaid_graph.append(
                            f'\t{head_router}{p_r % "head"}:::pea-->{r}{p_e % r}:::pea'
                        )
                    if v.is_tail_router:
                        mermaid_graph.append(
                            f'\t{r}{p_e % r}:::pea-->{tail_router}{p_r % "tail"}:::pea'
                        )
                mermaid_graph.append('end')

        for node, v in self._pod_nodes.items():
            ed_str = str(v.head_args.socket_in).split('_')[0]
            for need in sorted(v.needs):
                edge_str = ''
                if need in self._pod_nodes:
                    st_str = str(
                        self._pod_nodes[need].tail_args.socket_out).split(
                            '_')[0]
                    edge_str = f'|{st_str}-{ed_str}|'

                _s = start_repl.get(need, (need, f'({need})'))
                _e = end_repl.get(node, (node, f'({node})'))
                _s_role = self._pod_nodes[need].role
                _e_role = self._pod_nodes[node].role
                line_st = '-->'

                if _s_role in {PodRoleType.INSPECT, PodRoleType.JOIN_INSPECT}:
                    _s = start_repl.get(need, (need, f'{{{{{need}}}}}'))

                if _e_role == PodRoleType.GATEWAY:
                    _e = ('gateway_END', f'({node})')
                elif _e_role in {
                        PodRoleType.INSPECT, PodRoleType.JOIN_INSPECT
                }:
                    _e = end_repl.get(node, (node, f'{{{{{node}}}}}'))

                if _s_role == PodRoleType.INSPECT or _e_role == PodRoleType.INSPECT:
                    line_st = '-.->'

                mermaid_graph.append(
                    f'{_s[0]}{_s[1]}:::{str(_s_role)} {line_st} {edge_str}{_e[0]}{_e[1]}:::{str(_e_role)}'
                )
        mermaid_graph.append(
            f'classDef {str(PodRoleType.POD)} fill:#32C8CD,stroke:#009999')
        mermaid_graph.append(
            f'classDef {str(PodRoleType.INSPECT)} fill:#ff6666,color:#fff')
        mermaid_graph.append(
            f'classDef {str(PodRoleType.JOIN_INSPECT)} fill:#ff6666,color:#fff'
        )
        mermaid_graph.append(
            f'classDef {str(PodRoleType.GATEWAY)} fill:#6E7278,color:#fff')
        mermaid_graph.append(
            f'classDef {str(PodRoleType.INSPECT_AUX_PASS)} fill:#fff,color:#000,stroke-dasharray: 5 5'
        )
        mermaid_graph.append('classDef pea fill:#009999,stroke:#1E6E73')
        return '\n'.join(mermaid_graph)

    def plot(self,
             output: str = None,
             vertical_layout: bool = False,
             inline_display: bool = False,
             build: bool = True,
             copy_flow: bool = False) -> 'BaseFlow':
        """
        Visualize the Flow up to the current point
        If a file name is provided it will create a jpg image with that name,
        otherwise it will display the URL for mermaid.
        If called within IPython notebook, it will be rendered inline,
        otherwise an image will be created.

        Example,

        .. highlight:: python
        .. code-block:: python

            flow = Flow().add(name='pod_a').plot('flow.svg')

        :param output: a filename specifying the name of the image to be created,
                    the suffix svg/jpg determines the file type of the output image
        :param vertical_layout: top-down or left-right layout
        :param inline_display: show image directly inside the Jupyter Notebook
        :param build: build the Flow first before plotting, gateway connection can be better showed
        :param copy_flow: when set to true, then always copy the current Flow and
                do the modification on top of it then return, otherwise, do in-line modification
        :return: the Flow
        """

        # deepcopy causes the below error while reusing a Flow in Jupyter
        # 'Pickling an AuthenticationString object is disallowed for security reasons'
        op_flow = copy.deepcopy(self) if copy_flow else self

        if build:
            op_flow.build(False)

        mermaid_str = op_flow._mermaid_str
        if vertical_layout:
            mermaid_str = mermaid_str.replace('graph LR', 'graph TD')

        image_type = 'svg'
        if output and output.endswith('jpg'):
            image_type = 'jpg'

        url = op_flow._mermaid_to_url(mermaid_str, image_type)
        showed = False
        if inline_display:
            try:
                from IPython.display import display, Image

                display(Image(url=url))
                showed = True
            except:
                # no need to panic users
                pass

        if output:
            download_mermaid_url(url, output)
        elif not showed:
            op_flow.logger.info(f'flow visualization: {url}')

        return self

    def _ipython_display_(self):
        """Displays the object in IPython as a side effect"""
        self.plot(inline_display=True,
                  build=(self._build_level != FlowBuildLevel.GRAPH))

    def _mermaid_to_url(self, mermaid_str: str, img_type: str) -> str:
        """
        Render the current Flow as URL points to a SVG. It needs internet connection

        :param mermaid_str: the mermaid representation
        :param img_type: image type (svg/jpg)
        :return: the url points to a SVG
        """
        if img_type == 'jpg':
            img_type = 'img'

        encoded_str = base64.b64encode(bytes(mermaid_str,
                                             'utf-8')).decode('utf-8')

        return f'https://mermaid.ink/{img_type}/{encoded_str}'

    @build_required(FlowBuildLevel.GRAPH)
    def to_swarm_yaml(self, path: TextIO):
        """
        Generate the docker swarm YAML compose file

        :param path: the output yaml path
        """
        swarm_yml = {'version': '3.4', 'services': {}}

        for k, v in self._pod_nodes.items():
            if v.role == PodRoleType.GATEWAY:
                cmd = 'jina gateway'
            else:
                cmd = 'jina pod'
            swarm_yml['services'][k] = {
                'command':
                f'{cmd} {" ".join(ArgNamespace.kwargs2list(vars(v.args)))}',
                'deploy': {
                    'parallel': 1
                }
            }

        JAML.dump(swarm_yml, path)

    @property
    @build_required(FlowBuildLevel.GRAPH)
    def port_expose(self) -> int:
        """Return the exposed port of the gateway
        # noqa: DAR201"""
        return self._pod_nodes['gateway'].port_expose

    @property
    @build_required(FlowBuildLevel.GRAPH)
    def host(self) -> str:
        """Return the local address of the gateway
        # noqa: DAR201"""
        return self._pod_nodes['gateway'].host

    @property
    @build_required(FlowBuildLevel.GRAPH)
    def address_private(self) -> str:
        """Return the private IP address of the gateway for connecting from other machine in the same network
        # noqa: DAR201"""
        return get_internal_ip()

    @property
    @build_required(FlowBuildLevel.GRAPH)
    def address_public(self) -> str:
        """Return the public IP address of the gateway for connecting from other machine in the public network
        # noqa: DAR201"""
        return get_public_ip()

    def __iter__(self):
        return self._pod_nodes.items().__iter__()

    def _show_success_message(self):
        if self._pod_nodes['gateway'].args.restful:
            header = 'http://'
            protocol = 'REST'
        else:
            header = 'tcp://'
            protocol = 'gRPC'

        address_table = [
            f'\t🖥️ Local access:\t' +
            colored(f'{header}{self.host}:{self.port_expose}',
                    'cyan',
                    attrs='underline'), f'\t🔒 Private network:\t' +
            colored(f'{header}{self.address_private}:{self.port_expose}',
                    'cyan',
                    attrs='underline')
        ]
        if self.address_public:
            address_table.append(
                f'\t🌐 Public address:\t' +
                colored(f'{header}{self.address_public}:{self.port_expose}',
                        'cyan',
                        attrs='underline'))
        self.logger.success(
            f'🎉 Flow is ready to use, accepting {colored(protocol + " request", attrs="bold")}'
        )
        self.logger.info('\n' + '\n'.join(address_table))

    def block(self):
        """Block the process until user hits KeyboardInterrupt """
        try:
            threading.Event().wait()
        except KeyboardInterrupt:
            pass

    def use_grpc_gateway(self, port: int = None):
        """Change to use gRPC gateway for IO
        :param port: the port to change"""
        self._common_kwargs['restful'] = False
        if port:
            self._common_kwargs['port_expose'] = port

    def use_rest_gateway(self, port: int = None):
        """Change to use REST gateway for IO
        :param port: the port to change"""
        self._common_kwargs['restful'] = True
        if port:
            self._common_kwargs['port_expose'] = port

    def __getitem__(self, item):
        if isinstance(item, str):
            return self._pod_nodes[item]
        elif isinstance(item, int):
            return list(self._pod_nodes.values())[item]
        else:
            raise TypeError(f'{typename(item)} is not supported')

    def _update_client(self):
        if self._pod_nodes['gateway'].args.restful:
            self._cls_client = WebSocketClient

    @property
    def workspace_id(self) -> Dict[str, str]:
        """Get all Pods' ``workspace_id`` values in a dict
        # noqa: DAR201"""
        return {
            k: p.args.workspace_id
            for k, p in self if hasattr(p.args, 'workspace_id')
        }

    @workspace_id.setter
    def workspace_id(self, value: str):
        """Set all Pods' ``workspace_id`` to ``value``

        :param value: a hexadecimal UUID string
        """
        uuid.UUID(value)
        for k, p in self:
            if hasattr(p.args, 'workspace_id'):
                p.args.workspace_id = value
                for k, v in p.peas_args.items():
                    if v and isinstance(v, argparse.Namespace):
                        v.workspace_id = value
                    if v and isinstance(v, List):
                        for i in v:
                            i.workspace_id = value

    @property
    def identity(self) -> Dict[str, str]:
        """Get all Pods' ``identity`` values in a dict
        # noqa: DAR201
        """
        return {k: p.args.identity for k, p in self}

    @identity.setter
    def identity(self, value: str):
        """Set all Pods' ``identity`` to ``value``

        :param value: a hexadecimal UUID string
        """
        uuid.UUID(value)
        self.args.identity = value
        # Re-initiating logger with new identity
        self.logger = JinaLogger(self.__class__.__name__, **vars(self.args))
        for _, p in self:
            p.args.identity = value

    # for backward support
    join = needs
Exemplo n.º 51
0
def read_json(json_file):
    """Read Json dataset package files

    Load each json and get the appropriate encoding for the dataset
    Reload the json using the encoding to ensure correct character sets
    """
    json_object = OrderedDict()
    json_file_encoding = None
    json_file = str(json_file) + ".json"

    try:
        file_obj = open_fr(json_file)
        json_object = json.load(file_obj)
        if "encoding" in json_object:
            json_file_encoding = json_object['encoding']
        file_obj.close()
    except ValueError:
        return None

    # Reload json using encoding if available
    try:
        if json_file_encoding:
            file_obj = open_fr(json_file, encoding=json_file_encoding)
        else:
            file_obj = open_fr(json_file)
        json_object = json.load(file_obj)
        file_obj.close()

    except ValueError:
        return None

    if isinstance(json_object, dict) and "resources" in json_object.keys():
        # Note::formats described by frictionless data may need to change
        tabular_exts = {"csv", "tab"}
        vector_exts = {"shp", "kmz"}
        raster_exts = {"tif", "tiff", "bil", "hdr", "h5", "hdf5", "hr", "image"}
        for resource_item in json_object["resources"]:
            if "format" not in resource_item:
                if "format" in json_object:
                    resource_item["format"] = json_object["format"]
                else:
                    resource_item["format"] = "tabular"
            if "extensions" in resource_item:
                exts = set(resource_item["extensions"])
                if exts <= tabular_exts:
                    resource_item["format"] = "tabular"
                elif exts <= vector_exts:
                    resource_item["format"] = "vector"
                elif exts <= raster_exts:
                    resource_item["format"] = "raster"
            if "url" in resource_item:
                if "urls" in json_object:
                    json_object["urls"][resource_item["name"]] = resource_item["url"]

        json_object["tables"] = OrderedDict()
        temp_tables = {}
        table_names = [item["name"] for item in json_object["resources"]]
        temp_tables["tables"] = OrderedDict(zip(table_names, json_object["resources"]))
        for table_name, table_spec in temp_tables["tables"].items():
            json_object["tables"][table_name] = myTables[temp_tables["tables"][table_name]
                                                         ["format"]](**table_spec)
        json_object.pop("resources", None)
        return TEMPLATES["default"](**json_object)
    return None
Exemplo n.º 52
0
class KafkaBrokerClient(ReconnectingClientFactory):
    """The low-level client which handles transport to a single Kafka broker.

    The KafkaBrokerClient object is responsible for maintaining a connection to
    a single Kafka broker, reconnecting as needed, over which is sends requests
    and receives responses.  Callers can register as 'subscribers' which will
    cause them to be notified of changes in the state of the connection to the
    Kafka broker. Callers make requests with :py:method:`makeRequest`

    """

    # What class protocol instances do we produce?
    protocol = KafkaProtocol

    # Reduce log spam from twisted
    noisy = False

    def __init__(self,
                 host,
                 port=DefaultKafkaPort,
                 clientId=CLIENT_ID,
                 subscribers=None,
                 maxDelay=MAX_RECONNECT_DELAY_SECONDS,
                 maxRetries=None,
                 reactor=None):
        """Create a KafkaBrokerClient for a given host/port.

        Create a new object to manage the connection to a single Kafka broker.
        KafkaBrokerClient will reconnect as needed to keep the connection to
        the broker up and ready to service requests. Requests are retried when
        the connection fails before the client receives the response. Requests
        can be cancelled at any time.

        Args:
            host (str): hostname or IP address of Kafka broker
            port (int): port number of Kafka broker on `host`. Defaulted: 9092
            clientId (str): Identifying string for log messages. NOTE: not the
                ClientId in the RequestMessage PDUs going over the wire.
            subscribers (list of callbacks): Initial list of callbacks to be
                called when the connection changes state.
            maxDelay (seconds): The maximum amount of time between reconnect
                attempts when a connection has failed.
            maxRetries: The maximum number of times a reconnect attempt will be
                made.
            reactor: the twisted reactor to use when making connections or
                scheduling iDelayedCall calls. Used primarily for testing.
        """
        # Set the broker host & port
        self.host = host
        self.port = port
        # No connector until we try to connect
        self.connector = None
        # Set our clientId
        self.clientId = clientId
        # If the caller set maxRetries, we will retry that many
        # times to reconnect, otherwise we retry forever
        self.maxRetries = maxRetries
        # Set max delay between reconnect attempts
        self.maxDelay = maxDelay
        # clock/reactor for testing...
        self.clock = reactor

        # The protocol object for the current connection
        self.proto = None
        # ordered dict of _Requests, keyed by requestId
        self.requests = OrderedDict()
        # deferred which fires when the close() completes
        self.dDown = None
        # Deferred list for any on-going notification
        self.notifydList = None
        # list of subscribers to our connection status:
        # when the connection goes up or down, we call the callback
        # with ourself and True/False for Connection-Up/Down
        if subscribers is None:
            self.connSubscribers = []
        else:
            self.connSubscribers = subscribers

    def __repr__(self):
        """return a string representing this KafkaBrokerClient."""
        return ('<KafkaBrokerClient {0}:{1}:{2}'.format(
            self.host, self.port, self.clientId))

    def makeRequest(self, requestId, request, expectResponse=True):
        """
        Send a request to our broker via our self.proto KafkaProtocol object.

        Return a deferred which will fire when the reply matching the requestId
        comes back from the server, or, if expectResponse is False, then
        return None instead.
        If we are not currently connected, then we buffer the request to send
        when the connection comes back up.
        """
        if requestId in self.requests:
            # Id is duplicate to 'in-flight' request. Reject it, as we
            # won't be able to properly deliver the response(s)
            # Note that this won't protect against a client calling us
            # twice with the same ID, but first with expectResponse=False
            # But that's pathological, and the only defense is to track
            # all requestIds sent regardless of whether we expect to see
            # a response, which is effectively a memory leak...
            raise DuplicateRequestError(
                'Reuse of requestId:{}'.format(requestId))

        # If we've been told to shutdown (close() called) then fail request
        if self.dDown:
            return fail(ClientError('makeRequest() called after close()'))

        # Ok, we are going to save/send it, create a _Request object to track
        canceller = partial(
            self.cancelRequest, requestId,
            CancelledError("Request:{} was cancelled".format(requestId)))
        tReq = _Request(requestId, request, expectResponse, canceller)

        # add it to our requests dict
        self.requests[requestId] = tReq

        # Add an errback to the tReq.d to remove it from our requests dict
        # if something goes wrong...
        tReq.d.addErrback(self._handleRequestFailure, requestId)

        # Do we have a connection over which to send the request?
        if self.proto:
            # Send the request
            self._sendRequest(tReq)
        # Have we not even started trying to connect yet? Do so now
        elif not self.connector:
            self._connect()
        return tReq.d

    def addSubscriber(self, cb):
        """Add a callback to be called when the connection changes state."""
        self.connSubscribers.append(cb)

    def delSubscriber(self, cb):
        """Remove a previously added 'subscriber' callback."""
        if cb in self.connSubscribers:
            self.connSubscribers.remove(cb)

    def close(self):
        """Close the brokerclient's connection, cancel any pending requests."""
        log.debug('%r: close proto:%r connector:%r', self, self.proto,
                  self.connector)
        # Give our proto a 'heads up'...
        if self.proto is not None:
            self.proto.closing = True
        # Don't try to reconnect, and if we have an outstanding 'callLater',
        # cancel it. Also, if we are in the middle of connecting, stop
        # and call our clientConnectionFailed method with UserError
        self.stopTrying()
        # Ok, stopTrying() call above took care of the 'connecting' state,
        # now handle 'connected' state
        connector, self.connector = self.connector, None
        if connector and connector.state != "disconnected":
            # Create a deferred to return
            self.dDown = Deferred()
            connector.disconnect()
        else:
            # Fake a cleanly closing connection
            self.dDown = succeed(None)
        # Cancel any requests
        for tReq in self.requests.values():  # can't use itervalues() may del()
            tReq.d.cancel()
        return self.dDown

    def buildProtocol(self, addr):
        """Create a KafkaProtocol object, store it in self.proto, return it."""
        # Schedule notification of subscribers
        self._get_clock().callLater(0, self._notify, True)
        # Build the protocol
        self.proto = ReconnectingClientFactory.buildProtocol(self, addr)
        # point it at us for notifications of arrival of messages
        self.proto.factory = self
        return self.proto

    def clientConnectionLost(self, connector, reason):
        """Handle notification from the lower layers of connection loss.

        If we are shutting down, and twisted sends us the expected type of
        error, eat the error. Otherwise, log it and pass it along.
        Also, schedule notification of our subscribers at the next pass
        through the reactor.
        """
        if self.dDown and reason.check(ConnectionDone):
            # We initiated the close, this is an expected close/lost
            log.debug('%r: Connection Closed:%r:%r', self, connector, reason)
            notifyReason = None  # Not a failure
        else:
            log.debug('%r: clientConnectionLost:%r:%r', self, connector,
                      reason)
            notifyReason = reason

        # Reset our proto so we don't try to send to a down connection
        self.proto = None
        # Schedule notification of subscribers
        self._get_clock().callLater(0, self._notify, False, notifyReason)
        # Call our superclass's method to handle reconnecting
        ReconnectingClientFactory.clientConnectionLost(self, connector, reason)

    def clientConnectionFailed(self, connector, reason):
        """Handle notification from the lower layers of connection failure.

        If we are shutting down, and twisted sends us the expected type of
        error, eat the error. Otherwise, log it and pass it along.
        Also, schedule notification of our subscribers at the next pass
        through the reactor.
        """
        if self.dDown and reason.check(UserError):
            # We initiated the close, this is an expected connectionFailed,
            # given we were trying to connect when close() was called
            log.debug('%r: clientConnectionFailed:%r:%r', self, connector,
                      reason)
            notifyReason = None  # Not a failure
        else:
            log.error('%r: clientConnectionFailed:%r:%r', self, connector,
                      reason)
            notifyReason = reason

        # Reset our proto so we don't try to send to a down connection
        # Needed?  I'm not sure we should even _have_ a proto at this point...
        self.proto = None

        # Schedule notification of subscribers
        self._get_clock().callLater(0, self._notify, False, notifyReason)
        # Call our superclass's method to handle reconnecting
        return ReconnectingClientFactory.clientConnectionFailed(
            self, connector, reason)

    def handleResponse(self, response):
        """Handle the response string received by KafkaProtocol.

        Ok, we've received the response from the broker. Find the requestId
        in the message, lookup & fire the deferred with the response.
        """
        requestId = KafkaCodec.get_response_correlation_id(response)
        # Protect against responses coming back we didn't expect
        tReq = self.requests.pop(requestId, None)
        if tReq is None:
            # This could happen if we've sent it, are waiting on the response
            # when it's cancelled, causing us to remove it from self.requests
            log.warning('Unexpected response:%r, %r', requestId, response)
        else:
            tReq.d.callback(response)

    # # Private Methods # #

    def _sendRequest(self, tReq):
        """Send a single request over our protocol to the Kafka broker."""
        try:
            tReq.sent = True
            self.proto.sendString(tReq.data)
        except Exception as e:
            log.exception('%r: request id: %d send failed:', self, tReq.id)
            del self.requests[tReq.id]
            tReq.d.errback(e)
        else:
            if not tReq.expect:
                # Once we've sent a request for which we don't expect a reply,
                # we're done, remove it from requests, and fire the deferred
                # with 'None', since there is no reply to be expected
                del self.requests[tReq.id]
                tReq.d.callback(None)

    def _sendQueued(self):
        """Connection just came up, send the unsent requests."""
        for tReq in self.requests.values():  # can't use itervalues() may del()
            if not tReq.sent:
                self._sendRequest(tReq)

    def cancelRequest(self, requestId, reason=CancelledError(), _=None):
        """Cancel a request: remove it from requests, & errback the deferred.

        NOTE: Attempts to cancel a request which is no longer tracked
          (expectResponse == False and already sent, or response already
          received) will raise KeyError
        """
        tReq = self.requests.pop(requestId)
        tReq.d.errback(reason)

    def _handlePending(self, reason):
        """Connection went down: handle in-flight & unsent as configured.

        Note: for now, we just 'requeue' all the in-flight by setting their
          'sent' variable to False and let '_sendQueued()' handle resending
          when the connection comes back.
          In the future, we may want to extend this so we can errback()
          to our client's any in-flight (and possibly queued) so they can deal
          with it at the application level.
        """
        for tReq in self.requests.itervalues():
            tReq.sent = False
        return reason

    def _handleRequestFailure(self, failure, requestId):
        """Remove a failed request from our bookkeeping dict.

        Not an error if already removed (canceller removes).
        """
        self.requests.pop(requestId, None)
        return failure

    def _get_clock(self):
        """Reactor to use for connecting, callLater, etc [for testing]."""
        if self.clock is None:
            from twisted.internet import reactor
            self.clock = reactor
        return self.clock

    def _connect(self):
        """Initiate a connection to the Kafka Broker."""
        log.debug('%r: _connect', self)
        # We can't connect, we're not disconnected!
        if self.connector:
            raise ClientError('_connect called but not disconnected')
        # Needed to enable retries after a disconnect
        self.resetDelay()
        self.connector = self._get_clock().connectTCP(self.host, self.port,
                                                      self)
        log.debug('%r: _connect got connector: %r', self, self.connector)

    def _notify(self, connected, reason=None, subs=None):
        """Notify the caller of :py:method:`close` of completion.

        Also notify any subscribers of the state change of the connection.
        """
        if connected:
            self._sendQueued()
        else:
            if self.dDown and not self.dDown.called:
                self.dDown.callback(reason)
            # If the connection just went down, we need to handle any
            # outstanding requests.
            self._handlePending(reason)

        # Notify if requested. We call all of the callbacks, but don't
        # wait for any returned deferreds to fire here. Instead we add
        # them to a deferredList which we check for and wait on before
        # calling any callbacks for subsequent events.
        # This should keep any state-changes done by these callbacks in the
        # proper order. Note however that the ordering of the individual
        # callbacks in each (connect/disconnect) list isn't guaranteed, and
        # they can all be progressing in parallel if they yield or otherwise
        # deal with deferreds
        if self.notifydList:
            # We already have a notify list in progress, so just call back here
            # when the deferred list fires, with the _current_ list of subs
            subs = list(self.connSubscribers)
            self.notifydList.addCallback(
                lambda _: self._notify(connected, reason=reason, subs=subs))
            return

        # Ok, no notifications currently in progress. Notify all the
        # subscribers, keep track of any deferreds, so we can make sure all
        # the subs have had a chance to completely process this event before
        # we send them any new ones.
        dList = []
        if subs is None:
            subs = list(self.connSubscribers)
        for cb in subs:
            dList.append(maybeDeferred(cb, self, connected, reason))
        self.notifydList = DeferredList(dList)

        def clearNotifydList(_):
            """Reset the notifydList once we've notified our subscribers."""
            self.notifydList = None

        self.notifydList.addCallback(clearNotifydList)
Exemplo n.º 53
0
    def __new__(mcs, name, bases, attrs):
        attrs["_meta"] = opts = TableOptions(attrs.get("Meta", None), name)

        # extract declared columns
        cols, remainder = [], {}
        for attr_name, attr in attrs.items():
            if isinstance(attr, columns.Column):
                attr._explicit = True
                cols.append((attr_name, attr))
            else:
                remainder[attr_name] = attr
        attrs = remainder

        cols.sort(key=lambda x: x[1].creation_counter)

        # If this class is subclassing other tables, add their fields as
        # well. Note that we loop over the bases in *reverse* - this is
        # necessary to preserve the correct order of columns.
        parent_columns = []
        for base in reversed(bases):
            if hasattr(base, "base_columns"):
                parent_columns = list(
                    base.base_columns.items()) + parent_columns

        # Start with the parent columns
        base_columns = OrderedDict(parent_columns)

        # Possibly add some generated columns based on a model
        if opts.model:
            extra = OrderedDict()
            # honor Table.Meta.fields, fallback to model._meta.fields
            if opts.fields is not None:
                # Each item in opts.fields is the name of a model field or a
                # normal attribute on the model
                for field_name in opts.fields:
                    field = Accessor(field_name).get_field(opts.model)
                    extra[field_name] = columns.library.column_for_field(field)
            else:
                for field in opts.model._meta.fields:
                    extra[field.name] = columns.library.column_for_field(field)

            # update base_columns with extra columns
            for key, col in extra.items():
                # skip current col because the parent was explicitly defined,
                # and the current column is not.
                if key in base_columns and base_columns[key]._explicit is True:
                    continue
                base_columns[key] = col

        # Explicit columns override both parent and generated columns
        base_columns.update(OrderedDict(cols))

        # Apply any explicit exclude setting
        for exclusion in opts.exclude:
            if exclusion in base_columns:
                base_columns.pop(exclusion)

        # Remove any columns from our remainder, else columns from our parent class will remain
        for attr_name in remainder:
            if attr_name in base_columns:
                base_columns.pop(attr_name)

        # Set localize on columns
        for col_name in base_columns.keys():
            localize_column = None
            if col_name in opts.localize:
                localize_column = True
            # unlocalize gets higher precedence
            if col_name in opts.unlocalize:
                localize_column = False

            if localize_column is not None:
                base_columns[col_name].localize = localize_column
        attrs["base_columns"] = base_columns
        return super(DeclarativeColumnsMetaclass,
                     mcs).__new__(mcs, name, bases, attrs)
Exemplo n.º 54
0
class FitSettingsDialog(QtWidgets.QDialog):
    """ A dialog that is used to configure the fits in a FitContainer. """
    sigFitsUpdated = QtCore.Signal(dict)

    def __init__(self, fit_container):
        """ Create a FitSettingsDialog for a matching FitContainer.
            @param fit_container FitContainer: the FitContainer that this dialog should manipulate
        """
        super().__init__()

        self.fc = fit_container
        self.title = '{0} fit settings'.format(self.fc.name)

        # set up window
        self.setModal(False)
        self.setWindowTitle(self.title)

        # variables
        self.all_functions = self.fc.fit_logic.fit_list[self.fc.dimension]
        self.tabs = {}
        self.parameters = {}
        self.parameterUse = {}
        self.currentFits = OrderedDict()
        self.fitWidgets = OrderedDict()
        self.currentFitWidgets = OrderedDict()

        # widgets and layouts
        self._dialogLayout = QtWidgets.QVBoxLayout()
        self._btnLayout = QtWidgets.QHBoxLayout()
        self._tabWidget = QtWidgets.QTabWidget()
        self._firstPage = QtWidgets.QWidget()
        self._firstPageLayout = QtWidgets.QVBoxLayout()
        self._scrollArea = QtWidgets.QScrollArea()
        self._scrollWidget = QtWidgets.QWidget()
        self._scrLayout = QtWidgets.QVBoxLayout()
        self._dbox = QtWidgets.QDialogButtonBox(
            QtWidgets.QDialogButtonBox.Ok
                | QtWidgets.QDialogButtonBox.Apply
                | QtWidgets.QDialogButtonBox.Cancel,
            QtCore.Qt.Horizontal
        )
        self._addFitButton = QtWidgets.QPushButton('Add fit')
        self._saveFitButton = QtWidgets.QPushButton('Save fits')
        self._loadFitButton = QtWidgets.QPushButton('Load fits')

        # layout construction
        self._scrollWidget.setLayout(self._scrLayout)
        self._scrollArea.setWidget(self._scrollWidget)
        self._scrollArea.setWidgetResizable(True)
        self._btnLayout.addWidget(self._addFitButton)
        self._btnLayout.addWidget(self._saveFitButton)
        self._btnLayout.addWidget(self._loadFitButton)
        self._firstPageLayout.addLayout(self._btnLayout)
        self._firstPageLayout.addWidget(self._scrollArea)
        self._firstPage.setLayout(self._firstPageLayout)
        self._tabWidget.addTab(self._firstPage, 'Fit functions')
        self._dialogLayout.addWidget(self._tabWidget)
        self._dialogLayout.addWidget(self._dbox)
        self.setLayout(self._dialogLayout)

        # connections
        self._dbox.accepted.connect(self.accept)
        self._dbox.rejected.connect(self.reject)
        self._dbox.clicked.connect(self.buttonClicked)
        self.accepted.connect(self.applySettings)
        self.rejected.connect(self.resetSettings)
        self._addFitButton.clicked.connect(self.addFitButtonClicked)
        self._loadFitButton.clicked.connect(self.loadFitButtonClicked)
        self._saveFitButton.clicked.connect(self.saveFitButtonClicked)
        self.sigFitsUpdated.connect(self.fc.set_fit_functions)
        self.fc.sigNewFitParameters.connect(self.updateParameters)

        # load user defined fits from fit container
        if len(self.fc.fit_list) > 0:
            self.loadFits(self.fc.fit_list)

    @QtCore.Slot(QtWidgets.QAbstractButton)
    def buttonClicked(self, button):
        """ Slot for signals from dialog button box.
            @param button QAbstractButton: designates which button was clicked.
        """
        if self._dbox.buttonRole(button) ==  QtWidgets.QDialogButtonBox.ApplyRole:
            self.applySettings()

    @QtCore.Slot()
    def addFitButtonClicked(self):
        """ The 'Add Fit' button was clicked. Display a name input dialog and add the fit. """
        res = QtWidgets.QInputDialog.getText(
            self,
            'New fit',
            'Enter a name for this fit:',
            )
        if res[1]:
            self.addFit(res[0])

    @QtCore.Slot()
    def saveFitButtonClicked(self):
        """ The 'Save Fits' button was clicked. Display file chooser and save fits to file. """
        res = QtWidgets.QFileDialog.getSaveFileName(
            self,
            'Save fit collection for {0}'.format(self.title),
            '/path',
            'Fit files (*.fit *.yml)'
            )
        self.fc.fit_logic.save_fits(res[0], {self.fc.dimension: self.fc.fit_list})

    @QtCore.Slot()
    def loadFitButtonClicked(self):
        """ The 'Load Fits' button was clicked. Display file chooser and load fits from file. """
        res = QtWidgets.QFileDialog.getOpenFileName(
            self,
            'Load fit collection for {0}'.format(self.title),
            '/path',
            'Fit files (*.fit *.yml)'
            )
        fits = self.fc.fit_logic.load_fits(res[0])
        self.loadFits(fits[self.fc.dimension])

    def loadFits(self, user_fits):
        """ Take a fit config dictionary and create widgets for the fits inside.
            @param user_fits dict: configured fits dictionary
        """
        self.removeAllFits()
        self.applySettings()

        for name, fit in user_fits.items():
            self.addFit(name, fit=fit['fit_name'], estimator=fit['est_name'])

            # add new tab for new fit
            model, params = self.all_functions[fit['fit_name']]['make_model']()
            self.tabs[name] = FitParametersWidget(params)
            self._tabWidget.addTab(self.tabs[name], name)
            self.updateParameters(name, fit['parameters'])
        # build fit list and send update signals
        self.applySettings()

    def addFit(self, name, fit=None, estimator=None):
        """ Add a new fit to the dialog.
            @param name str: configured name for fit
            @param fit str: name of the fit function for this fit
            @param estimator str: name of the estimator function for this fit
        """
        if len(name) < 1:
            return
        if name in self.fitWidgets:
            logging.error('{0}: Fit {1} already exists.'.format(self.title, name))
            return
        fcw = FitConfigWidget(name, self.all_functions, fit, estimator)
        self.currentFitWidgets[name] = fcw
        self._scrLayout.addWidget(fcw)
        fcw.sigRemoveFit.connect(self.removeFit)
        fcw.show()

    @QtCore.Slot(str)
    def removeFit(self, name):
        """ Remove a fit from the dialog. Hides the FitonfigWidet and disables the parameter tab.
            @param name str: name of fit to remove
        """
        widget = self.currentFitWidgets.pop(name)
        widget.hide()
        self._scrLayout.removeWidget(widget)
        tab = self.tabs[name]
        tab.setEnabled(False)

    def removeAllFits(self):
        """ Remove all configured fits from dialog.
        """
        for name in self.currentFits.keys():
            self.removeFit(name)

    def getFits(self):
        """ Return all configured fits from this dialog.
        """
        return self.currentFits

    def applySettings(self):
        """ Apply all settings that the user has made in the dialog and send out update signals.

        This copies all input widget values to thei coresponding internal data structures,
        creates an removes widgets and parameter tabs.
        """
        # remove all settings tabs and config widgets that the user removed
        for name, widget in self.fitWidgets.items():
            if name not in self.currentFitWidgets:
                tab = self.tabs.pop(name)
                index = self._tabWidget.indexOf(tab)
                if index >= 0:
                    self._tabWidget.removeTab(index)
                self._scrLayout.removeWidget(widget)

        self.fitWidgets = OrderedDict()

        # add tabs for new fits, replace tabs for changed fits
        for name, widget in self.currentFitWidgets.items():
            oldfit = widget.fit
            oldest = widget.estimator
            widget.applySettings()

            if name in self.tabs and widget.fit != oldfit:
                # remove old tab, add new tab, preferably in the same place, for changed fit
                tab = self.tabs.pop(name)
                index = self._tabWidget.indexOf(tab)
                if index >= 0:
                    self._tabWidget.removeTab(index)
                model, params = self.all_functions[widget.fit]['make_model']()
                self.tabs[name] = FitParametersWidget(params)
                if index >= 0:
                    self._tabWidget.insertTab(index, self.tabs[name], name)
                else:
                    self._tabWidget.addTab(self.tabs[name], name)

            elif name not in self.tabs:
                # add new tab for new fir
                model, params = self.all_functions[widget.fit]['make_model']()
                self.tabs[name] = FitParametersWidget(params)
                self._tabWidget.addTab(self.tabs[name], name)

            # put all widgets here
            self.fitWidgets[name] = widget

        # reset update tabs and put new values in dict
        self.parameters = {}
        self.parameterUse = {}

        for name, tab in self.tabs.items():
            self.parameters[name], self.parameterUse[name] = tab.applyFitParameters()

        self.buildCurrentFits()
        self.sigFitsUpdated.emit(self.currentFits)

    def buildCurrentFits(self):
        """ Update dictionary of the configured fits for FitContainer or other componenrs.
        """
        # arrange all of this information in a convenient form
        self.currentFits = OrderedDict()
        for name, widget in self.fitWidgets.items():
            try:
                self.currentFits[name] = {
                    'fit_name': widget.fit,
                    'est_name': widget.estimator,
                    'make_fit': self.all_functions[widget.fit]['make_fit'],
                    'make_model': self.all_functions[widget.fit]['make_model'],
                    'estimator': self.all_functions[widget.fit][widget.estimator],
                    'parameters': self.parameters[name],
                    'use_settings': self.parameterUse[name]
                }
            except KeyError:
                continue

    def resetSettings(self):
        """ Reset all input widgets to their stored values, discarding changes by the user.
        """
        for name, widget in self.currentFitWidgets.items():
            if name not in self.fitWidgets:
                self._scrLayout.removeWidget(widget)

        self.currentFitWidgets = OrderedDict()
        for name, widget in self.fitWidgets.items():
            widget.resetSettings()
            widget.show()
            self.currentFitWidgets[name] = widget

        # reset tabs, do not update dicts
        for name, tab in self.tabs.items():
            tab.setEnabled(True)
            tab.resetFitParameters()

    def getParameters(self, fit_name):
        """ Return Parameters object for a given fit.
            @param fit_name str: name of the fit

            @return Parameters: lmfit parameters container
        """
        return self.parameters[fit_name]

    def updateParameters(self, fit_name, parameters):
        """ Update parameters of a given fit.
            @param fit_name str: name of fit
            @param parameters Parameters: lmfit Parameters container

        This function updates all parameters for a fit that the dialog has stored and ingores
        any other parameters in he parameter container.
        """
        if fit_name in self.parameters:
            for name, param in parameters.items():
                if name in self.parameters[fit_name]:
                    self.parameters[fit_name][name] = param
            self.tabs[fit_name].updateFitParameters(parameters)
Exemplo n.º 55
0
    def evaluate(self,
                 results,
                 metric='mIoU',
                 logger=None,
                 efficient_test=False,
                 **kwargs):
        """Evaluate the dataset.

        Args:
            results (list): Testing results of the dataset.
            metric (str | list[str]): Metrics to be evaluated. 'mIoU',
                'mDice' and 'mFscore' are supported.
            logger (logging.Logger | None | str): Logger used for printing
                related information during evaluation. Default: None.

        Returns:
            dict[str, float]: Default metrics.
        """

        if isinstance(metric, str):
            metric = [metric]
        allowed_metrics = ['mIoU', 'mDice', 'mFscore']
        if not set(metric).issubset(set(allowed_metrics)):
            raise KeyError('metric {} is not supported'.format(metric))
        eval_results = {}
        gt_seg_maps = self.get_gt_seg_maps(efficient_test)
        if self.CLASSES is None:
            num_classes = len(
                reduce(np.union1d, [np.unique(_) for _ in gt_seg_maps]))
        else:
            num_classes = len(self.CLASSES)
        ret_metrics = eval_metrics(
            results,
            gt_seg_maps,
            num_classes,
            self.ignore_index,
            metric,
            label_map=self.label_map,
            reduce_zero_label=self.reduce_zero_label)

        if self.CLASSES is None:
            class_names = tuple(range(num_classes))
        else:
            class_names = self.CLASSES

        # summary table
        ret_metrics_summary = OrderedDict({
            ret_metric: np.round(np.nanmean(ret_metric_value) * 100, 2)
            for ret_metric, ret_metric_value in ret_metrics.items()
        })

        # each class table
        ret_metrics.pop('aAcc', None)
        ret_metrics_class = OrderedDict({
            ret_metric: np.round(ret_metric_value * 100, 2)
            for ret_metric, ret_metric_value in ret_metrics.items()
        })
        ret_metrics_class.update({'Class': class_names})
        ret_metrics_class.move_to_end('Class', last=False)

        # for logger
        class_table_data = PrettyTable()
        for key, val in ret_metrics_class.items():
            class_table_data.add_column(key, val)

        summary_table_data = PrettyTable()
        for key, val in ret_metrics_summary.items():
            if key == 'aAcc':
                summary_table_data.add_column(key, [val])
            else:
                summary_table_data.add_column('m' + key, [val])

        print_log('per class results:', logger)
        print_log('\n' + class_table_data.get_string(), logger=logger)
        print_log('Summary:', logger)
        print_log('\n' + summary_table_data.get_string(), logger=logger)

        # each metric dict
        for key, value in ret_metrics_summary.items():
            if key == 'aAcc':
                eval_results[key] = value / 100.0
            else:
                eval_results['m' + key] = value / 100.0

        ret_metrics_class.pop('Class', None)
        for key, value in ret_metrics_class.items():
            eval_results.update({
                key + '.' + str(name): value[idx] / 100.0
                for idx, name in enumerate(class_names)
            })

        if mmcv.is_list_of(results, str):
            for file_name in results:
                os.remove(file_name)
        return eval_results
Exemplo n.º 56
0
class Bibliography:
    def __init__(self, entries: dict or list):
        if isinstance(entries, dict):
            self._data: OrderedDict = OrderedDict(
                {k: Entry.entry_factory(v)
                 for k, v in entries.items()})
        elif isinstance(entries, list):
            self._data: OrderedDict = OrderedDict(
                {e['ID']: Entry.entry_factory(e)
                 for e in entries})
        else:
            raise WrongEntryTypeError(
                f"Expected list or dict, got {type(entries)}")
        self._handle_duplicates()

    def __len__(self):
        return len(self._data)

    def __add__(self, other):
        if not isinstance(other, Bibliography):
            raise NotBibliographyError()
        for k, v in other._data.items():
            if k in self._data:
                self._data[k].update(v)
            else:
                self._data[k] = v
        self.sort()
        self._handle_duplicates()
        return self

    def __sub__(self, other):
        if not isinstance(other, Bibliography):
            raise NotBibliographyError()
        for k, v in other._data.items():
            if self._data[k] == v:
                self._data.pop(k, None)
        self.sort()
        self._handle_duplicates()
        return self

    def __iter__(self):
        raise NotImplementedError("Use entries or entries_dict")

    def __contains__(self, item):
        return any((entry == item) for entry in self.entries)

    @property
    def entries(self) -> list:
        return [e for e in self._data.values()]

    @property
    def entries_dict(self):
        return self._data

    @property
    def authors_years_dict(self) -> dict:
        return {k: entry.author_year for k, entry in self._data.items()}

    @property
    def authors_years(self) -> list:
        return [entry for entry in self.authors_years_dict.values()]

    @property
    def unique_authors_years(self) -> dict:
        unique_authors_years = {}
        for k, v in self.authors_years_dict.items():
            if v in unique_authors_years:
                unique_authors_years[v].append(k)
            else:
                unique_authors_years[v] = [k]
        return unique_authors_years

    def sort(self):
        self._data = OrderedDict(
            sorted(self._data.items(), key=lambda key_val: key_val[1]))

    def _handle_duplicates(self):
        self.sort()

        # delete duplicates
        duplicates = []
        last_entry = object()
        for k, entry in self._data.items():
            if entry == last_entry:
                duplicates.append(k)
            last_entry = entry
        for k in duplicates:
            self._data.pop(k, None)

        # set numbers/letters for pseudo-duplicates
        for k, v in self.unique_authors_years.items():
            if 1 < len(v):
                for i, id_ in enumerate(v):
                    self._data[id_]['letter_number'] = i + 1
            else:
                id_ = v[0]
                self._data[id_]['letter_number'] = None
Exemplo n.º 57
0
def make_editor_session(pad,
                        path,
                        is_attachment=None,
                        alt=PRIMARY_ALT,
                        datamodel=None):
    """Creates an editor session for the given path object."""
    if alt != PRIMARY_ALT and not pad.db.config.is_valid_alternative(alt):
        raise BadEdit("Attempted to edit an invalid alternative (%s)" % alt)

    raw_data = pad.db.load_raw_data(path,
                                    cls=OrderedDict,
                                    alt=alt,
                                    fallback=False)
    raw_data_fallback = None
    if alt != PRIMARY_ALT:
        raw_data_fallback = pad.db.load_raw_data(path, cls=OrderedDict)
        all_data = OrderedDict()
        all_data.update(raw_data_fallback or ())
        all_data.update(raw_data or ())
    else:
        all_data = raw_data

    id = posixpath.basename(path)
    if not is_valid_id(id):
        raise BadEdit("Invalid ID")

    record = None
    exists = raw_data is not None or raw_data_fallback is not None
    if raw_data is None:
        raw_data = OrderedDict()

    if is_attachment is None:
        if not exists:
            is_attachment = False
        else:
            is_attachment = bool(all_data.get("_attachment_for"))
    elif bool(all_data.get("_attachment_for")) != is_attachment:
        raise BadEdit("The attachment flag passed is conflicting with the "
                      "record's attachment flag.")

    if exists:
        # XXX: what about changing the datamodel after the fact?
        if datamodel is not None:
            raise BadEdit("When editing an existing record, a datamodel "
                          "must not be provided.")
        datamodel = pad.db.get_datamodel_for_raw_data(all_data, pad)
    else:
        if datamodel is None:
            datamodel = pad.db.get_implied_datamodel(path, is_attachment, pad)
        elif isinstance(datamodel, str):
            datamodel = pad.db.datamodels[datamodel]

    if exists:
        record = pad.instance_from_data(dict(all_data), datamodel)

    for key in implied_keys:
        raw_data.pop(key, None)
        if raw_data_fallback:
            raw_data_fallback.pop(key, None)

    return EditorSession(
        pad,
        id,
        str(path),
        raw_data,
        raw_data_fallback,
        datamodel,
        record,
        exists,
        is_attachment,
        alt,
    )
Exemplo n.º 58
0
class EventListener(object):
    """``Session``, ``Call`` and ``Job`` tracking through a default set of
    event handlers.

    Tracks various session entities by wrapping received event data in local
    ``models`` APIs and/or data structures. Serves as a higher level API on
    top of the underlying event loop.
    """
    def __init__(
            self,
            event_loop,
            call_tracking_header='variable_call_uuid',
            max_limit=float('inf'),
    ):
        """
        :param str call_tracking_header:
            Name of the freeswitch variable (including the 'variable_' prefix)
            to use for associating sessions into tracked calls
            (see `_handle_create`).

            It is common to set this to an Xheader variable if attempting
            to track calls "through" an intermediary device (i.e. the first
            hop receiving requests) such as a B2BUA.

            NOTE: in order for this association mechanism to work the
            intermediary device must be configured to forward the Xheaders
            it receives.
        """
        self.event_loop = event_loop
        self.sessions = OrderedDict()
        self.log = utils.get_logger(utils.pstr(self))
        # store last 1k of each type of failed session
        self.failed_sessions = OrderedDict()
        self.bg_jobs = OrderedDict()
        self.calls = OrderedDict()  # maps aleg uuids to Sessions instances
        self.hangup_causes = Counter()  # record of causes by category
        self.sessions_per_app = Counter()
        self.max_limit = max_limit
        self.call_tracking_header = call_tracking_header
        # state reset
        self.reset()

        # add default handlers
        for evname, cbtype, cb in get_callbacks(self, only='handler'):
            self.event_loop.add_handler(evname, cb)

    def register_job(self, future, **kwargs):
        '''Register for a job to be handled when the appropriate event arrives.
        Once an event corresponding to the job is received, the bgjob event
        handler will 'consume' it and invoke its callback.

        Parameters
        ----------
        event : ESL.ESLevent
            as returned from an ESLConnection.bgapi call
        kwargs : dict
            same signatures as for Job.__init__

        Returns
        -------
        bj : an instance of Job (a background job)
        '''
        bj = Job(future, **kwargs)
        self.bg_jobs[bj.uuid] = bj
        return bj

    def count_jobs(self):
        return len(self.bg_jobs)

    def count_sessions(self):
        return len(self.sessions)

    def count_calls(self):
        '''Count the number of active calls hosted by the slave process
        '''
        return len(self.calls)

    def count_failed(self):
        '''Return the failed session count
        '''
        return sum(self.hangup_causes.values()
                   ) - self.hangup_causes['NORMAL_CLEARING']

    def reset(self):
        '''Clear all internal stats and counters
        '''
        self.log.debug('resetting all stats...')
        self.hangup_causes.clear()
        self.failed_jobs = Counter()
        self.total_answered_sessions = 0

    @handler('CHANNEL_HANGUP')
    @handler('CHANNEL_PARK')
    @handler('CALL_UPDATE')
    def lookup_sess(self, e):
        """The most basic handler template which looks up the locally tracked
        session corresponding to event `e` and updates it with event data
        """
        sess = self.sessions.get(e.get('Unique-ID'), False)
        if sess:
            sess.update(e)
            return True, sess
        return False, None

    def lookup_sess_and_job(self, e):
        """Look up and return the session and any corresponding background job.
        """
        consumed, sess = self.lookup_sess(e)
        if consumed:
            return True, sess, sess.bg_job
        return False, None, None

    @handler('LOG')
    def _handle_log(self, e):
        self.log.info(e.get('Body'))
        return True, None

    @handler('SERVER_DISCONNECTED')
    def _handle_disconnect(self, e):
        """Log disconnects.
        """
        self.log.warning("Received DISCONNECT from server '{}'".format(
            self.host))
        return True, None

    @handler('BACKGROUND_JOB')
    def _handle_bj(self, e):
        '''Handle bjs and report failures.
        If a job is found in the local cache then update the instance
        with event data.
        This handler returns 'None' on error (i.e. failed bj)
        which must be handled by any callbacks.
        '''
        error = False
        consumed = False
        resp = None
        sess = None
        ok = '+OK '
        err = '-ERR'
        job_uuid = e.get('Job-UUID')
        body = e.get('Body')

        # always report errors even for jobs which we aren't tracking
        if err in body:
            resp = body.strip(err).strip()
            error = True
            self.log.debug("job '{}' failed with:\n{}".format(
                job_uuid, str(body)))
        elif ok in body:
            resp = body.strip(ok + '\n')

        job = self.bg_jobs.get(job_uuid, None)
        job_just_created = False
        if not job:
            job = Job(event=e)
            job_just_created = True
        else:
            job.events.update(e)

        # attempt to lookup an associated session
        sess = self.sessions.get(job.sess_uuid or resp, None)

        if error:
            # if the job returned an error, report it and remove the job
            self.log.error("Job '{}' corresponding to session '{}'"
                           " failed with:\n{}".format(job_uuid, job.sess_uuid,
                                                      str(body)))
            job.fail(resp)  # fail the job
            if not job_just_created:
                # always pop failed jobs
                self.bg_jobs.pop(job_uuid)
            # append the id for later lookup and discard?
            self.failed_jobs[resp] += 1
            consumed = True

        else:  # OK case
            if sess:
                # special case: the bg job event returns a known originated
                # session's (i.e. pre-registered) uuid in its body
                if job.sess_uuid:

                    assert str(job.sess_uuid) == str(resp), \
                        ("""Session uuid '{}' <-> BgJob uuid '{}' mismatch!?
                         """.format(job.sess_uuid, resp))

                # reference this job in the corresponding session
                # self.sessions[resp].bg_job = job
                sess.bg_job = job
                self.log.debug("Job '{}' was sucessful".format(job_uuid))
                consumed = True
            else:
                self.log.warn(
                    "No session corresponding to bj '{}'".format(job_uuid))

            # run the job's callback
            job(resp)

        return consumed, sess, job

    @handler('CHANNEL_CREATE')
    @handler('CHANNEL_ORIGINATE')
    def _handle_initial_event(self, e):
        '''Handle channel create events by building local
        `Session` and `Call` objects for state tracking.
        '''
        uuid = e.get('Unique-ID')
        # Record the newly activated session
        # TODO: pass con as weakref?
        con = self.event_loop._con

        # short circuit if we have already allocated a session since FS is
        # indeterminate about which event create|originate will arrive first
        sess = self.sessions.get(uuid)
        if sess:
            return True, sess

        # allocate a session model
        sess = Session(e, event_loop=self.event_loop, uuid=uuid, con=con)
        direction = sess['Call-Direction']
        self.log.debug("{} session created with uuid '{}'".format(
            direction, uuid))
        sess.cid = self.event_loop.get_id(e, 'default')

        # Use our specified "call identification variable" to try and associate
        # sessions into calls. By default the 'variable_call_uuid' channel
        # variable is used for tracking locally bridged calls
        call_uuid = e.get(self.call_tracking_header)  # could be 'None'
        if not call_uuid:
            self.log.warn(
                "Unable to associate {} session '{}' with a call using "
                "variable '{}'".format(direction, sess.uuid,
                                       self.call_tracking_header))
            call_uuid = uuid

        # associate sessions into a call
        # (i.e. set the relevant sessions to reference each other)
        if call_uuid in self.calls:
            call = self.calls[call_uuid]
            self.log.debug("session '{}' is bridged to call '{}'".format(
                uuid, call.uuid))
            # append this session to the call's set
            call.append(sess)

        else:  # this sess is not yet tracked so use its id as the 'call' id
            call = Call(call_uuid, sess)
            self.calls[call_uuid] = call
            self.log.debug("call created for session '{}'".format(call_uuid))
        sess.call = call
        self.sessions[uuid] = sess
        self.sessions_per_app[sess.cid] += 1
        return True, sess

    @handler('CHANNEL_ANSWER')
    def _handle_answer(self, e):
        '''Handle answer events

        Returns
        -------
        sess : session instance corresponding to uuid
        '''
        uuid = e.get('Unique-ID')
        sess = self.sessions.get(uuid, None)
        if sess:
            self.log.debug("answered {} session '{}'".format(
                e.get('Call-Direction'), uuid))
            sess.answered = True
            self.total_answered_sessions += 1
            sess.update(e)
            return True, sess
        else:
            self.log.warn('Skipping answer of {}'.format(uuid))
            return False, None

    @handler('CHANNEL_DESTROY')
    # @handler('CHANNEL_HANGUP_COMPLETE')  # XXX: a race between these two...
    def _handle_destroy(self, e):
        '''Handle channel destroy events.

        Returns
        -------
        sess : session instance corresponding to uuid
        job  : corresponding bj for a session if exists, ow None
        '''
        uuid = e.get('Unique-ID')
        sess = self.sessions.pop(uuid, None)
        direction = sess['Call-Direction'] if sess else 'unknown'
        if not sess:
            return False, None
        sess.update(e)
        sess.hungup = True
        cause = e.get('Hangup-Cause')
        self.hangup_causes[cause] += 1  # count session causes
        self.sessions_per_app[sess.cid] -= 1

        # if possible lookup the relevant call
        call_uuid = e.get(self.call_tracking_header)
        if not call_uuid:
            self.log.warn(
                "handling HANGUP for {} session '{}' which can not be "
                "associated with an active call using {}?".format(
                    direction, sess.uuid, self.call_tracking_header))
            call_uuid = uuid

        # XXX seems like sometimes FS changes the `call_uuid`
        # between create and hangup oddly enough
        call = self.calls.get(call_uuid, sess.call)
        if call:
            if sess in call.sessions:
                self.log.debug("hungup {} session '{}' for Call '{}'".format(
                    direction, uuid, call.uuid))
                call.sessions.remove(sess)
            else:
                # session was somehow tracked by the wrong call
                self.log.err("session '{}' mismatched with call '{}'?".format(
                    sess.uuid, call.uuid))

            # all sessions hungup
            if len(call.sessions) == 0:
                self.log.debug(
                    "all sessions for call '{}' were hung up".format(
                        call_uuid))
                # remove call from our set
                call = self.calls.pop(call.uuid, None)
                if not call:
                    self.log.warn(
                        "Call with id '{}' containing Session '{}' was "
                        "already removed".format(call.uuid, sess.uuid))
        else:
            # we should never get hangups for calls we never saw created
            self.log.err("no call found for '{}'".format(call_uuid))

        # pop any corresponding job
        job = sess.bg_job
        # may have been popped by the partner
        self.bg_jobs.pop(job.uuid if job else None, None)
        sess.bg_job = None  # deref job - avoid mem leaks

        if not sess.answered or cause != 'NORMAL_CLEARING':
            self.log.debug("'{}' was not successful??".format(sess.uuid))
            self.failed_sessions.setdefault(cause,
                                            deque(maxlen=1000)).append(sess)

        self.log.debug("hungup Session '{}'".format(uuid))

        # hangups are always consumed
        return True, sess, job

    @property
    def host(self):
        return self.event_loop.host

    @property
    def port(self):
        return self.event_loop.port

    def is_alive(self):
        return self.event_loop.is_alive()

    def is_running(self):
        return self.event_loop.is_running()

    def connect(self, **kwargs):
        return self.event_loop.connect(**kwargs)

    def connected(self):
        return self.event_loop.connected()

    def start(self):
        return self.event_loop.start()

    def disconnect(self, **kwargs):
        return self.event_loop.disconnect(**kwargs)

    def unsubscribe(self, evname):
        return self.event_loop.unsubscribe(evname)
Exemplo n.º 59
0
class SkillInterface(SkillCore):
    """
    """

    #--------Class functions--------

    def __init__(self, children_processor=Sequential()):
        super(SkillInterface, self).__init__()
        # Params
        self._remaps = OrderedDict()
        # Connections
        self._parent = None
        self._children = []
        self._children_processor = children_processor
        # Caches
        self._was_simulated = False
        self._max_cache = 2
        self._params_cache = []
        self._input_cache = []
        self._remaps_cache = defaultdict(list)

    def __call__(self, *children):
        """
        @brief Add a set of children skills
        @return self for nested children declarations
        """
        for c in children:
            self.addChild(c)
        return self

    @property
    def parent(self):
        """
        @brief      Returns the parent skill
        """
        return self._parent

    def getLightCopy(self):
        """
        @brief      Makes a light copy (only description, params and state)

        @return     A light copy
        """
        p = self.__class__(self._children_processor)
        p._children_processor = deepcopy(self._children_processor)
        p._type = copy(self._type)
        p._label = copy(self._label)
        p._params = deepcopy(self._params)
        p._remaps = deepcopy(self._remaps)
        p._description = deepcopy(self._description)
        p._pre_conditions = deepcopy(self._pre_conditions)
        p._hold_conditions = deepcopy(self._hold_conditions)
        p._post_conditions = deepcopy(self._post_conditions)
        if self._state != State.Uninitialized:
            p._state = copy(self._state)
            p._wmi = self._wmi
        return p

    def _setState(self, state):
        self._state = state
        self._state_change.set()

    def _clearRemaps(self):
        """
        @brief      Clear remaps
        """
        for r1, r2 in reversed(self._remaps.items()):
            self.remap(r2, r1)
        self._remaps = OrderedDict()
        self._remaps_cache = defaultdict(list)

    def _copyRemaps(self, skill):
        """
        @brief      Copy the remaps of another skill. Called automatically when
                    the skill is added as a child
        """
        for r1, r2 in skill._remaps.iteritems():
            self.remap(r1, r2)

    def _revertRemaps(self):
        """
        @brief      Revert remaps. Just used in revertInput
        """
        remapid = len(self._params_cache)
        try:
            if self._remaps_cache[remapid]:
                for remap in self._remaps_cache[remapid]:
                    log.warn("REMAP",
                             "Revert {} {}".format(remap[0], remap[1]))
                    print self._remaps_cache
                    self._remaps.pop(remap[0])
                    self.remap(remap[1], remap[0], False)
                del self._remaps_cache[remapid]
        except BaseException:
            print self.printInfo(True)
            print self._remaps_cache
            raise

    def getParamsNoRemaps(self):
        """
        @brief Get the skill's parameters without key remappings
        """
        ph = params.ParamHandler()
        ph.reset(self._params.getCopy())
        for r1, r2 in reversed(self._remaps.items()):
            ph.remap(r2, r1)
        return ph

    def get_remap(self, key):
        """
        @brief Return the key remapping if existing or otherwise the key unchanged
        """
        return self._remaps[key] if key in self._remaps else key

    def remap(self, initial_key, target_key, record=True):
        """
        @brief Remap a parameter with initial_key to a new target_key

        All skill's children are remapped too.
        """
        # Ignore harmful remappings
        if initial_key in self._remaps:
            if self._remaps[initial_key] == target_key:  # Redundant
                #log.warn(self.type, "Ignoring redundant remap {}->{}".format(initial_key, target_key))
                return
            else:  # Already remapped
                log.warn(
                    self.type,
                    "Key {} already remapped to {}. Can t remap to {}".format(
                        initial_key, self._remaps[initial_key], target_key))
                return

        #log.warn("Remap", "{} {} {}. Existing: {}".format(self.type, initial_key, target_key, self._remaps))
        # The current 2->3 is related to an existing remap 1->2


#        if initial_key in self._remaps.values():
#            log.info("ChainRemap", "{}: {}->{}->{}".format(self.type, self._remaps.keys()[self._remaps.values().index(initial_key)], initial_key, target_key))

        for c in self._children:
            c.remap(initial_key, target_key)

        if target_key in self._remaps:
            # Asking remap 1->2 but exists a remap 2->3. Records a remap 1->3
            target_key = self.get_remap(target_key)

        if self.params.hasParam(target_key):
            log.error(
                "Remap",
                "{}: Invalid remapping {}->{}, target key is already present.".
                format(self.type, initial_key, target_key))
            return

        if self.params.hasParam(initial_key):
            #log.warn("{}".format(self.type), "REMAPPING: {} {}".format(initial_key, target_key))
            # Remaps
            self._params.remap(initial_key, target_key)
            for c in self._pre_conditions:
                c.remap(initial_key, target_key)
            for c in self._hold_conditions:
                c.remap(initial_key, target_key)
            for c in self._post_conditions:
                c.remap(initial_key, target_key)
        # Records
        if record:
            self._remaps[initial_key] = target_key
            remapid = len(self._params_cache)
            if remapid > 0:  # If I have params cached, I cache also the remap to revert it afterwards
                self._remaps_cache[remapid].append((initial_key, target_key))

    def init(self, wmi):
        self._wmi = wmi
        self.createDescription()
        self.modifyDescription(self)
        self._setState(State.Idle)

    def inSubtreeOf(self, skill):
        if self in skill._children:
            return True
        for c in skill._children:
            if self.inSubtreeOf(c):
                return True
        return False

    def hold(self):
        for c in self._hold_conditions:
            if not c.setTrue(self._params, self._wmi):
                log.error(c.getDescription(), "Hold failed.")
                return False
        return True

    def revertHold(self):
        for c in reversed(self._hold_conditions):
            #print c.getDescription()
            if not c.revert(self._params, self._wmi):
                log.error(c.getDescription(), "Revert hold failed.")
                return False
        self._was_simulated = False
        return True

    def simulate(self):
        self._was_simulated = True
        #print "Simulate: {}".format(self.printInfo(True))
        for c in self._post_conditions:
            if not c.setTrue(self._params, self._wmi):
                log.error(c.getDescription(), "Simulation failed.")
                return State.Failure
        return State.Success

    def revertSimulation(self):
        if not self._was_simulated:
            log.warn("revert", "No simulation was made, can't revert.")
            return False
        for c in reversed(self._post_conditions):
            if not c.revert(self._params, self._wmi):
                log.error(c.getDescription(), "Revert failed.")
                return False
        self._was_simulated = False
        return True

    def visit(self, visitor):
        return visitor.process(self)

    def visitPreempt(self, visitor):
        return visitor.processPreempt(self)

    def getParent(self):
        return self._parent

    def hasChildren(self):
        return len(self._children) > 0

    def addChild(self, p, latch=False):
        p._parent = self
        self._children.append(p)
        p._copyRemaps(self)
        if latch and len(self._children) > 1:
            for c in self._children[-2]._post_conditions:
                for key in c.getKeys():
                    if not p.params.hasParam(key):
                        p.params[key] = deepcopy(
                            self._children[-2]._params._params[key])
            p._pre_conditions += deepcopy(self._children[-2]._post_conditions)
        return self

    def last(self):
        return self._children[-1]

    def popChild(self):
        child = self._children.pop()
        child._parent = None

    def specifyParam(self, key, values):
        """
        Specify a parameter and update the input cache
        """
        super(SkillInterface, self).specifyParam(key, values)
        if self._input_cache:
            if key in self._input_cache[-1]:
                self._input_cache[-1][key].setValues(values)

    def specifyParamsDefault(self, input_params):
        """
        Set the parameters and makes them default (they will no more be overwritten by setInput)
        """
        self._params_cache.append(self._params.getCopy())
        self._input_cache.append(input_params.getCopy())
        super(SkillInterface, self).specifyParamsDefault(input_params)

    def specifyParams(self, input_params, keep_default=True):
        """
        Set the parameters. Params already specified are preserved
        """
        self._params_cache.append(self._params.getCopy())
        self._input_cache.append(input_params.getCopy())
        super(SkillInterface, self).specifyParams(input_params, keep_default)
        # Erase the oldest cache if the max lenght is reached
        if len(self._params_cache) > self._max_cache:
            self._popCache()

    def _popCache(self):
        self._params_cache.pop(0)
        for i in range(self._max_cache + 1):
            if i not in self._remaps_cache:
                continue
            del self._remaps_cache[i]
            if i + 1 in self._remaps_cache:
                self._remaps_cache[i] = self._remaps_cache[i + 1]

    def revertInput(self):
        if not self._params_cache:
            log.warn("revertInput", "No cache available, can't revert input.")
            return None
        self._revertRemaps()
        self._params.reset(deepcopy(self._params_cache.pop()))
        return deepcopy(self._input_cache.pop())

    def start(self, params=None):
        self._children_processor.reset()
        return SkillCore.start(self, params)

    def tick(self):
        res = self.execute()
        #        print "{} {}".format(self, res)
        if res is not None:
            self._setState(res)
        if not self.onEnd():
            self._setState(State.Failure)
            res = State.Failure
        return res

    #--------User functions--------
    def resetDescription(self, other=None):
        if other:
            self._params.reset(self._description._params.merge(other._params))
        else:
            self._params = deepcopy(self._description._params)
        self._pre_conditions = list()
        self._hold_conditions = list()
        self._post_conditions = list()
        for p in self._description._pre_conditions:
            self.addPreCondition(deepcopy(p))
        for p in self._description._hold_conditions:
            self.addHoldCondition(deepcopy(p))
        for p in self._description._post_conditions:
            self.addPostCondition(deepcopy(p))

    def mergeDescription(self, other):
        self.resetDescription(other)
        for c in other._pre_conditions:
            if not c in self._pre_conditions:
                self.addPreCondition(c)
        for c in other._hold_conditions:
            if not c in self._hold_conditions:
                self.addHoldCondition(c)
        for c in other._post_conditions:
            if not c in self._post_conditions:
                self.addPostCondition(c)

    def addParam(self, key, value, param_type, options=[], description=""):
        """
        @brief Check for remaps before adding a parameter.

        See SkillDescription for more details
        """
        if key in self._remaps:
            key = self._remaps[key]
        SkillDescription.addParam(self, key, value, param_type, options,
                                  description)

    def addPreCondition(self, condition, modify_description=False):
        if modify_description:
            self._description.addPreCondition(deepcopy(condition))
        self._pre_conditions.append(condition)
        for r1, r2 in self._remaps.iteritems():
            self._pre_conditions[-1].remap(r1, r2)

    def addHoldCondition(self, condition, modify_description=False):
        if modify_description:
            self._description.addHoldCondition(deepcopy(condition))
        self._hold_conditions.append(condition)
        for r1, r2 in self._remaps.iteritems():
            self._hold_conditions[-1].remap(r1, r2)

    def addPostCondition(self, condition, modify_description=False):
        if modify_description:
            self._description.addPostCondition(deepcopy(condition))
        self._post_conditions.append(condition)
        for r1, r2 in self._remaps.iteritems():
            self._post_conditions[-1].remap(r1, r2)

    def processChildren(self, visitor):
        """
        TODO: this function has to be removed and embedded into the tick
        """
        self._setState(
            self._children_processor.processChildren(self._children, visitor))
        return self._state

    def setProcessor(self, processor_type):
        """
        Set the children processor
        """
        self._children_processor = processor_type

    #--------Virtual functions--------

    def hasInstance(self):
        """ Wrapper skills override this """
        return True

    def execute(self):
        """
            Optional
        """
        self._setProgress("End", 1)
        return None

    def wrapper_expand(self):
        pass
Exemplo n.º 60
0
class DebugVariableViewer(object):
    """
    Class that implements a generic viewer that display a list of variable values
    This class has to be inherited to effectively display variable values
    """
    def __init__(self, window, items=None):
        """
        Constructor
        @param window: Reference to the Debug Variable Panel
        @param items: List of DebugVariableItem displayed by Viewer
        """
        self.ParentWindow = window
        items = [] if items is None else items
        self.ItemsDict = OrderedDict([(item.GetVariable(), item)
                                      for item in items])
        self.Items = self.ItemsDict.viewvalues()

        # Variable storing current highlight displayed in Viewer
        self.Highlight = HIGHLIGHT_NONE
        # List of buttons
        self.Buttons = []
        self.InitHighlightPensBrushes()

    def __del__(self):
        """
        Destructor
        """
        # Remove reference to Debug Variable Panel
        self.ParentWindow = None

    def InitHighlightPensBrushes(self):
        """
        Init global pens and brushes
        """
        if not HIGHLIGHT:
            HIGHLIGHT['DROP_PEN'] = wx.Pen(wx.Colour(0, 128, 255))
            HIGHLIGHT['DROP_BRUSH'] = wx.Brush(wx.Colour(0, 128, 255, 128))
            HIGHLIGHT['RESIZE_PEN'] = wx.Pen(wx.Colour(200, 200, 200))
            HIGHLIGHT['RESIZE_BRUSH'] = wx.Brush(wx.Colour(200, 200, 200))

    def GetIndex(self):
        """
        Return position of Viewer in Debug Variable Panel
        @return: Position of Viewer
        """
        return self.ParentWindow.GetViewerIndex(self)

    def GetItem(self, variable):
        """
        Return item storing values of a variable
        @param variable: Variable path
        @return: Item storing values of this variable
        """
        return self.ItemsDict.get(variable, None)

    def GetItems(self):
        """
        Return items displayed by Viewer
        @return: List of items displayed in Viewer
        """
        return self.ItemsDict.values()

    def AddItem(self, item):
        """
        Add an item to the list of items displayed by Viewer
        @param item: Item to add to the list
        """
        self.ItemsDict[item.GetVariable()] = item

    def RemoveItem(self, item):
        """
        Remove an item from the list of items displayed by Viewer
        @param item: Item to remove from the list
        """
        self.ItemsDict.pop(item.GetVariable(), None)

    def ClearItems(self):
        """
        Clear list of items displayed by Viewer
        """
        # Unsubscribe every items of the list
        for item in self.Items:
            self.ParentWindow.RemoveDataConsumer(item)

        # Clear list
        self.ItemsDict.clear()

    def ItemsIsEmpty(self):
        """
        Return if list of items displayed by Viewer is empty
        @return: True if list is empty
        """
        return len(self.Items) == 0

    def SubscribeAllDataConsumers(self):
        """
        Function that unsubscribe and remove every item that store values of
        a variable that doesn't exist in PLC anymore
        """
        for item in self.ItemsDict.values()[:]:
            iec_path = item.GetVariable()

            # Check that variablepath exist in PLC
            if self.ParentWindow.GetDataType(iec_path) is None:
                # If not, unsubscribe and remove it
                self.ParentWindow.RemoveDataConsumer(item)
                self.RemoveItem(item)
            else:
                # If it exist, resubscribe and refresh data type
                self.ParentWindow.AddDataConsumer(iec_path.upper(), item, True)
                item.RefreshVariableType()

    def ResetItemsData(self):
        """
        Reset data stored in every items displayed in Viewer
        """
        for item in self.Items:
            item.ResetData()

    def GetItemsMinCommonTick(self):
        """
        Return the minimum tick common to all iems displayed in Viewer
        @return: Minimum common tick between items
        """
        return reduce(max, [
            item.GetData()[0, 0]
            for item in self.Items if len(item.GetData()) > 0
        ], 0)

    def RefreshViewer(self):
        """
        Method that refresh the content displayed by Viewer
        Need to be overridden by inherited classes
        """
        pass

    def SetHighlight(self, highlight):
        """
        Set Highlight type displayed in Viewer
        @return: True if highlight has changed
        """
        # Return immediately if highlight don't change
        if self.Highlight == highlight:
            return False

        self.Highlight = highlight
        return True

    def GetButtons(self):
        """
        Return list of buttons defined in Viewer
        @return: List of buttons
        """
        return self.Buttons

    def IsOverButton(self, x, y):
        """
        Return if point is over one button of Viewer
        @param x: X coordinate of point
        @param y: Y coordinate of point
        @return: button where point is over
        """
        for button in self.GetButtons():
            if button.HitTest(x, y):
                return button
        return None

    def HandleButton(self, x, y):
        """
        Search for the button under point and if found execute associated
        callback
        @param x: X coordinate of point
        @param y: Y coordinate of point
        @return: True if a button was found and callback executed
        """
        button = self.IsOverButton(x, y)
        if button is None:
            return False

        button.ProcessCallback()
        return True

    def ShowButtons(self, show):
        """
        Set display state of buttons in Viewer
        @param show: Display state (True if buttons must be displayed)
        """
        # Change display of every buttons
        for button in self.Buttons:
            button.Show(show)

        # Refresh button positions
        self.RefreshButtonsPosition()
        self.RefreshViewer()

    def RefreshButtonsPosition(self):
        """
        Function that refresh buttons position in Viewer
        """
        # Get Viewer size
        width, _height = self.GetSize()

        # Buttons are align right so we calculate buttons positions in
        # reverse order
        buttons = self.Buttons[:]
        buttons.reverse()

        # Position offset on x coordinate
        x_offset = 0
        for button in buttons:
            # Buttons are stacked right, removing those that are not active
            if button.IsEnabled():
                # Update button position according to button width and offset
                # on x coordinate
                w, _h = button.GetSize()
                button.SetPosition(width - 5 - w - x_offset, 5)
                # Update offset on x coordinate
                x_offset += w + 2

    def DrawCommonElements(self, dc, buttons=None):
        """
        Function that draw common graphics for every Viewers
        @param dc: wx.DC object corresponding to Device context where drawing
        common graphics
        @param buttons: List of buttons to draw if different from default
        (default None)
        """
        # Get Viewer size
        width, height = self.GetSize()

        # Set dc styling for drop before or drop after highlight
        dc.SetPen(HIGHLIGHT['DROP_PEN'])
        dc.SetBrush(HIGHLIGHT['DROP_BRUSH'])

        # Draw line at upper side of Viewer if highlight is drop before
        if self.Highlight == HIGHLIGHT_BEFORE:
            dc.DrawLine(0, 1, width - 1, 1)

        # Draw line at lower side of Viewer if highlight is drop before
        elif self.Highlight == HIGHLIGHT_AFTER:
            dc.DrawLine(0, height - 1, width - 1, height - 1)

        # If no specific buttons are defined, get default buttons
        if buttons is None:
            buttons = self.Buttons
        # Draw buttons
        for button in buttons:
            button.Draw(dc)

        # If graph dragging is processing
        if self.ParentWindow.IsDragging():
            destBBox = self.ParentWindow.GetDraggingAxesClippingRegion(self)
            srcPos = self.ParentWindow.GetDraggingAxesPosition(self)
            if destBBox.width > 0 and destBBox.height > 0:
                srcPanel = self.ParentWindow.DraggingAxesPanel
                srcBBox = srcPanel.GetAxesBoundingBox()

                srcX = srcBBox.x - (srcPos.x if destBBox.x == 0 else 0)
                srcY = srcBBox.y - (srcPos.y if destBBox.y == 0 else 0)

                srcBmp = _convert_agg_to_wx_bitmap(srcPanel.get_renderer(),
                                                   None)
                srcDC = wx.MemoryDC()
                srcDC.SelectObject(srcBmp)

                dc.Blit(destBBox.x, destBBox.y, int(destBBox.width),
                        int(destBBox.height), srcDC, srcX, srcY)

    def OnEnter(self, event):
        """
        Function called when entering Viewer
        @param event: wx.MouseEvent
        """
        # Display buttons
        self.ShowButtons(True)
        event.Skip()

    def OnLeave(self, event):
        """
        Function called when leaving Viewer
        @param event: wx.MouseEvent
        """
        # Hide buttons
        self.ShowButtons(False)
        event.Skip()

    def OnCloseButton(self):
        """
        Function called when Close button is pressed
        """
        wx.CallAfter(self.ParentWindow.DeleteValue, self)

    def OnForceButton(self):
        """
        Function called when Force button is pressed
        """
        self.ForceValue(self.ItemsDict.values()[0])

    def OnReleaseButton(self):
        """
        Function called when Release button is pressed
        """
        self.ReleaseValue(self.ItemsDict.values()[0])

    def OnMouseDragging(self, x, y):
        """
        Function called when mouse is dragged over Viewer
        @param x: X coordinate of mouse pointer
        @param y: Y coordinate of mouse pointer
        """
        xw, yw = self.GetPosition()
        # Refresh highlight in Debug Variable Panel (highlight can be displayed
        # in another Viewer
        self.ParentWindow.RefreshHighlight(x + xw, y + yw)

    def RefreshHighlight(self, x, y):
        """
        Function called by Debug Variable Panel asking Viewer to refresh
        highlight according to mouse position
        @param x: X coordinate of mouse pointer
        @param y: Y coordinate of mouse pointer
        """
        # Get Viewer size
        _width, height = self.GetSize()

        # Mouse is in the first half of Viewer
        if y < height // 2:
            # If Viewer is the upper one, draw drop before highlight
            if self.ParentWindow.IsViewerFirst(self):
                self.SetHighlight(HIGHLIGHT_BEFORE)

            # Else draw drop after highlight in previous Viewer
            else:
                self.SetHighlight(HIGHLIGHT_NONE)
                self.ParentWindow.HighlightPreviousViewer(self)

        # Mouse is in the second half of Viewer, draw drop after highlight
        else:
            self.SetHighlight(HIGHLIGHT_AFTER)

    def OnEraseBackground(self, event):
        """
        Function called when Viewer background is going to be erase
        @param event: wx.EraseEvent
        """
        # Prevent flicker on Windows
        pass

    def OnResize(self, event):
        """
        Function called when Viewer size changed
        @param event: wx.ResizeEvent
        """
        # Refresh button positions
        self.RefreshButtonsPosition()
        self.ParentWindow.ForceRefresh()
        event.Skip()

    def ForceValue(self, item):
        """
        Force value of item given
        @param item: Item to force value
        """
        # Check variable data type
        iec_path = item.GetVariable()
        iec_type = self.ParentWindow.GetDataType(iec_path)
        # Return immediately if not found
        if iec_type is None:
            return

        # Open a dialog to enter varaible forced value
        dialog = ForceVariableDialog(self, iec_type, str(item.GetValue()))
        if dialog.ShowModal() == wx.ID_OK:
            self.ParentWindow.ForceDataValue(iec_path.upper(),
                                             dialog.GetValue())

    def ReleaseValue(self, item):
        """
        Release value of item given
        @param item: Item to release value
        """
        self.ParentWindow.ReleaseDataValue(item.GetVariable().upper())