Example #1
1
class ForgetfulStorage(object):
    implements(IStorage)

    def __init__(self, ttl=604800):
        """
        By default, max age is a week.
        """
        self.data = OrderedDict()
        self.ttl = ttl

    def __setitem__(self, key, value):
        if key in self.data:
            del self.data[key]
        self.data[key] = (time.time(), value)
        self.cull()

    def cull(self):
        for k, v in self.iteritemsOlderThan(self.ttl):
            self.data.popitem(last=False)

    def get(self, key, default=None):
        self.cull()
        if key in self.data:
            return False, self[key]
        return False, default

    def __getitem__(self, key):
        self.cull()
        return self.data[key][1]

    def __iter__(self):
        self.cull()
        return iter(self.data)

    def __repr__(self):
        self.cull()
        return repr(self.data)

    def iteritemsOlderThan(self, secondsOld):
        minBirthday = time.time() - secondsOld
        zipped = self._tripleIterable()
        matches = takewhile(lambda r: minBirthday >= r[1], zipped)
        return imap(operator.itemgetter(0, 2), matches)

    def _tripleIterable(self):
        ikeys = self.data.iterkeys()
        ibirthday = imap(operator.itemgetter(0), self.data.itervalues())
        ivalues = imap(operator.itemgetter(1), self.data.itervalues())
        return izip(ikeys, ibirthday, ivalues)

    def iteritems(self):
        self.cull()
        ikeys = self.data.iterkeys()
        ivalues = imap(operator.itemgetter(1), self.data.itervalues())
        return izip(ikeys, ivalues)
Example #2
0
 def run_cloners(cls, old_event, new_event):
     selected = {x[7:] for x in request.values.getlist('cloners') if x.startswith('cloner_')}
     all_cloners = OrderedDict((name, cloner_cls(old_event))
                               for name, cloner_cls in get_event_cloners().iteritems())
     if any(cloner.is_internal for name, cloner in all_cloners.iteritems() if name in selected):
         raise Exception('An internal cloner was selected')
     # enable internal cloners that are enabled by default or required by another cloner
     selected |= {c.name
                  for c in all_cloners.itervalues()
                  if c.is_internal and (c.is_default or c.required_by_deep & selected)}
     # enable unavailable cloners that may be pulled in as a dependency nonetheless
     extra = {c.name
              for c in all_cloners.itervalues()
              if not c.is_available and c.always_available_dep and c.required_by_deep & selected}
     selected |= extra
     active_cloners = OrderedDict((name, cloner) for name, cloner in all_cloners.iteritems() if name in selected)
     if not all((c.is_internal or c.is_visible) and c.is_available
                for c in active_cloners.itervalues()
                if c.name not in extra):
         raise Exception('An invisible/unavailable cloner was selected')
     for name, cloner in active_cloners.iteritems():
         if not (selected >= cloner.requires_deep):
             raise Exception('Cloner {} requires {}'.format(name, ', '.join(cloner.requires_deep - selected)))
     shared_data = {}
     cloner_names = set(active_cloners)
     for name, cloner in active_cloners.iteritems():
         shared_data[name] = cloner.run(new_event, cloner_names, cloner._prepare_shared_data(shared_data))
Example #3
0
    def by(self, name, index=None, field=None):
        """ facet by given function

            Examples:

            .by(index=lambda x: x)
            .by(field=lambda x: ('T_high' if x['T'] > 1000 else 'T_low'))
        """
        if index:
            ret = OrderedDict(
                   [(index(val),self[self.index.get_level_values(name) == val])
                        for val in self.index.get_level_values(name)]
                  )
            for _ in ret.itervalues():
                _.properties = self.properties
            return MultiFrame(ret)
        else:
            self['cat'] = map(field, self[name])
            cats = self.groupby('cat').groups.keys()
            ret =  OrderedDict(
                    [(cat,self[self['cat'] == cat]) for cat in cats]
                )
            for _ in ret.itervalues():
                _.properties = self.properties
            return MultiFrame(ret)
Example #4
0
    def process_static_data(cls, data):
        # create instances
        ret = OrderedDict()
        for t in sorted(data, key=lambda x: x["name"]):
            name = str(t["name"])
            ret[name] = Type(name, t["effectiveAgainst"], t["weakAgainst"])

        # additional manipulations
        size = len(ret)
        by_effectiveness = {}
        by_resistance = {}
        for t in ret.itervalues():  # type: Type
            t.attack_effective_against = [ret[name] for name in t.attack_effective_against]
            t.attack_weak_against = [ret[name] for name in t.attack_weak_against]

            # group types effective against, weak against specific types
            for l, d in (t.attack_effective_against, by_effectiveness), \
                        (t.attack_weak_against, by_resistance):
                for tt in l:
                    if tt not in d:
                        d[tt] = set()
                    d[tt].add(t)

            # calc average factor for damage of this type relative to all types
            t.rate = (size
                      + ((EFFECTIVENESS_FACTOR-1) * len(t.attack_effective_against))
                      - ((1-RESISTANCE_FACTOR) * len(t.attack_weak_against))) / size

        # set pokemon type resistance/weakness info
        for t in ret.itervalues():  # type: Type
            t.pokemon_resistant_to = by_resistance[t]
            t.pokemon_vulnerable_to = by_effectiveness[t]

        return ret
Example #5
0
class ResourceLoader(object):
	def __init__(self):
		self.resources = OrderedDict()

	def require(self, name):
		if name in self.resources:
			return

		resource = resources.get(name)
		if not resource:
			raise LoaderError("There is no resource by the name %s." % name)
		for requirement in resource.requirements:
			self.require(requirement)
		self.resources[resource.name] = resource

	@property
	def styles(self):
		s = HtmlList()
		for resource in self.resources.itervalues():
			for style in resource.styles:
				s.append(Style(**style))
		return s

	@property
	def scripts(self):
		s = HtmlList()
		for resource in self.resources.itervalues():
			for script in resource.scripts:
				s.append(Script(**script))
		return s
Example #6
0
    def add_leveled_toc_items(self):
        added = OrderedDict()
        added2 = OrderedDict()
        counter = 1

        def find_matches(expr, doc):
            try:
                ans = XPath(expr)(doc)
                len(ans)
                return ans
            except:
                self.log.warn("Invalid ToC expression, ignoring: %s" % expr)
                return []

        for document in self.oeb.spine:
            previous_level1 = list(added.itervalues())[-1] if added else None
            previous_level2 = list(added2.itervalues())[-1] if added2 else None

            for elem in find_matches(self.opts.level1_toc, document.data):
                text, _href = self.elem_to_link(document, elem, counter)
                counter += 1
                if text:
                    node = self.oeb.toc.add(text, _href, play_order=self.oeb.toc.next_play_order())
                    added[elem] = node
                    # node.add(_('Top'), _href)

            if self.opts.level2_toc is not None and added:
                for elem in find_matches(self.opts.level2_toc, document.data):
                    level1 = None
                    for item in document.data.iterdescendants():
                        if item in added:
                            level1 = added[item]
                        elif item == elem:
                            if level1 is None:
                                if previous_level1 is None:
                                    break
                                level1 = previous_level1
                            text, _href = self.elem_to_link(document, elem, counter)
                            counter += 1
                            if text:
                                added2[elem] = level1.add(text, _href, play_order=self.oeb.toc.next_play_order())
                            break

                if self.opts.level3_toc is not None and added2:
                    for elem in find_matches(self.opts.level3_toc, document.data):
                        level2 = None
                        for item in document.data.iterdescendants():
                            if item in added2:
                                level2 = added2[item]
                            elif item == elem:
                                if level2 is None:
                                    if previous_level2 is None:
                                        break
                                    level2 = previous_level2
                                text, _href = self.elem_to_link(document, elem, counter)
                                counter += 1
                                if text:
                                    level2.add(text, _href, play_order=self.oeb.toc.next_play_order())
                                break
    class View(object):
        """ Presents a view of the set of `Option` arguments to a search command

        TODO: Description

        """
        def __init__(self, command):
            self._items = OrderedDict([
                (option.name, Option.Item(command, option))
                for member_name, option in type(command).option_definitions])
            return

        def __contains__(self, name):
            return name in self._items

        def __getitem__(self, name):
            return self._items[name]

        def __iter__(self):
            return self._items.__iter__()

        def __len__(self):
            return len(self._items)

        def __repr__(self):
            text = ''.join([
                'Option.View(',
                ','.join([repr(item) for item in self.itervalues()]),
                ')'])
            return text

        def __str__(self):
            text = ' '.join(
                [str(item) for item in self.itervalues() if item.is_set])
            return text

        #region Methods

        def get_missing(self):
            missing = [
                item.name for item in self._items.itervalues()
                if item.is_required and not item.is_set]
            return missing if len(missing) > 0 else None

        def iteritems(self):
            return self._items.iteritems()

        def iterkeys(self):
            return self.__iter__()

        def itervalues(self):
            return self._items.itervalues()

        def reset(self):
            for value in self.itervalues():
                value.reset()
            return
Example #8
0
class ForgetfulStorage(object):
	def __init__(self, ttl=604800):
		"""
		Implements a storage class for distributed log messages.
		Default max age is a week.
		"""
		self.data = OrderedDict()
		self.ttl  = ttl

	def __setitem__(self, key, value):
#		if key in self.data:
#			self.data[key] = (time.time(),value)
		self.data[key] = (time.time(), value)
		self.cull()

	def __getitem__(self, key):
		self.cull()
		return self.data[key][1]

	def __iter__(self):
		self.cull()
		return iter(self.data)

	def __repr__(self):
		self.cull()
		return repr(self.data)

	def get(self, key, default=None):
		self.cull()
		if key in self.data:
			return self[key]
		return default

	def cull(self):
		"""
		Note that it may be useful to track what we evict.
		"""
		for k, v in self.iteritems_older_than(self.ttl):
			self.data.popitem(last=False)

	def iteritems_older_than(self, seconds_old):
		min_birthday = time.time() - seconds_old
		zipped = self._triple_iterable()
		matches = takewhile(lambda r: min_birthday >= r[1], zipped)
		return imap(operator.itemgetter(0, 2), matches)

	def _triple_iterable(self):
		ikeys = self.data.iterkeys()
		ibirthday = imap(operator.itemgetter(0), self.data.itervalues())
		ivalues = imap(operator.itemgetter(1), self.data.itervalues())
		return izip(ikeys, ibirthday, ivalues)

	def iteritems(self):
		self.cull()
		ikeys = self.data.iterkeys()
		ivalues = imap(operator.itemgetter(1), self.data.itervalues())
		return izip(ikeys, ivalues)
    def get(self):
        archive = memcache.get('archive')
        if archive is not None:
            self.response.out.headers['Content-Type'] = 'application/json'
            self.response.out.write(json.dumps({'articles':archive}))

        else:
            more = True
            archive = OrderedDict()
            curs = None
            
            while more:
                articles, curs, more = Article.query().order(
                    -Article.when).fetch_page(
                        50,
                        start_cursor=curs,
                        projection=[Article.title, 
                                    Article.when,
                                    Article.keywords])
                for article in articles:
                    article_key = article.key.urlsafe()
                    
                    if article_key in archive:
                        archive[article_key]['keywords'].extend(
                            article.keywords)
                    else:
                        archive[article_key] = {
                            'title': article.title,
                            'when': article.when.strftime("%d/%m/%Y %H:%M:%S"),
                            'keywords': article.keywords,
                            'key': article_key,
                            'ncomments': 0}
                        

            more = True
            curs = None

            while more:
                page, curs, more = Comment.query().fetch_page(
                    100,
                    start_cursor = curs)

                for comment in page:
                    parent_key = comment.key.parent().urlsafe()
                    if parent_key in archive:
                        archive[parent_key]['ncomments'] += 1
                     

            memcache.add('archive',
                         [a for a in archive.itervalues()],
                         time=86400)
            self.response.out.headers['Content-Type'] = 'application/json'
            self.response.out.write(json.dumps(
                {'articles':[a for a in archive.itervalues()]}
            ))
Example #10
0
    def add_leveled_toc_items(self):
        added = OrderedDict()
        added2 = OrderedDict()
        counter = 1
        for document in self.oeb.spine:
            previous_level1 = list(added.itervalues())[-1] if added else None
            previous_level2 = list(added2.itervalues())[-1] if added2 else None

            for elem in XPath(self.opts.level1_toc)(document.data):
                text, _href = self.elem_to_link(document, elem, counter)
                counter += 1
                if text:
                    node = self.oeb.toc.add(text, _href,
                            play_order=self.oeb.toc.next_play_order())
                    added[elem] = node
                    #node.add(_('Top'), _href)

            if self.opts.level2_toc is not None and added:
                for elem in XPath(self.opts.level2_toc)(document.data):
                    level1 = None
                    for item in document.data.iterdescendants():
                        if item in added:
                            level1 = added[item]
                        elif item == elem:
                            if level1 is None:
                                if previous_level1 is None:
                                    break
                                level1 = previous_level1
                            text, _href = self.elem_to_link(document, elem, counter)
                            counter += 1
                            if text:
                                added2[elem] = level1.add(text, _href,
                                    play_order=self.oeb.toc.next_play_order())
                            break

                if self.opts.level3_toc is not None and added2:
                    for elem in XPath(self.opts.level3_toc)(document.data):
                        level2 = None
                        for item in document.data.iterdescendants():
                            if item in added2:
                                level2 = added2[item]
                            elif item == elem:
                                if level2 is None:
                                    if previous_level2 is None:
                                        break
                                    level2 = previous_level2
                                text, _href = \
                                        self.elem_to_link(document, elem, counter)
                                counter += 1
                                if text:
                                    level2.add(text, _href,
                                        play_order=self.oeb.toc.next_play_order())
                                break
Example #11
0
class Atributos(OrderedDictBase):
    """Atributos da classe."""

    def __init__(self, xmlclasse=None, data=None, class_associations=None):
        if xmlclasse is not None:
            self.__atributos = OrderedDict()
            xmlatributos = xmlclasse.iterdescendants(tag="Attribute")

            if xmlatributos is not None:
                for xmlatributo in xmlatributos:
                    # Atribuição dos parâmetros de construção do objeto classe.
                    xml_attributes = xmlatributo.attrib
                    tagged_values = TaggedValues(xmlatributo)
                    stereotypes = Stereotypes(xmlatributo)

                    # Contrução do atributo e inclusão na lista de atributos de classe.
                    if stereotypes.find('name', 'association_attribute') is not None:
                        atributo = AssociationAttribute(xml_attributes, tagged_values, stereotypes=stereotypes)
                    else:
                        atributo = Attribute(xml_attributes, tagged_values, stereotypes=stereotypes)

                    self.__atributos[atributo.name] = atributo
            else:
                logger.debug(u'Nenhum atributo encontrado.')

            # Cria os atributos associativos.
            if class_associations is not None:
                for association in class_associations:
                    attribute = AssociationAttribute(association.xml_attributes, association.tagged_values)
                    self.__atributos[attribute.name] = attribute
        elif data is not None:
            self.__atributos = data
        else:
            self.__atributos = OrderedDict()

        # Instancia a classe superior.
        super(Atributos, self).__init__(self.__atributos, Atributos)

    def __str__(self):
        return_msg = ''
        for attr in self.__atributos.itervalues():
            return_msg += '\n  %s' % str(attr)
        return return_msg

    @property
    def association_attributes(self):
        """Lista de classes principais."""
        association_attributes = OrderedDict()
        for attribute in self.__atributos.itervalues():
            if attribute.is_association_attribute:
                association_attributes[attribute.id] = attribute
        return Atributos(data=association_attributes)
Example #12
0
class Region(object):
    """ Holds a region's layers along the blade.

    :param layers: Dictionary of Layer3D objects
    :type layers: dict
    """

    def __init__(self):
        self.layers = OrderedDict()

    def add_layer(self, name):
        ''' Inserts a layer into layers dict.

        :param name: Name of the material
        :return: The layer added to the region
        '''
        dubl = 0
        for k in self.layers.iterkeys():
            if name == k[:-2]:
                dubl += 1

        lname = '%s%02d' % (name, dubl)

        layer = Layer()
        self.layers[lname] = layer
        return layer

    def init_stack(self):
        ''' Initiates thickness and angle matrices of the region plus the max
            thickness of each layer
        :return: thick_matrix: 2d np.array containing thicknesses of all layers (size: no_layers,s)
        :return: thick_max: 1d np.array max thickness per layer
        :return: angle_matrix: 2d np.array containing angles of all layers (size: no_layers,s)
        '''
        thickmatdata = []
        for v in self.layers.itervalues():
            thickmatdata.append(v.thickness)
        self.thick_matrix = np.fliplr(np.rot90(np.array(thickmatdata), -1))

        thickmaxdata = []
        for l in range(self.thick_matrix.shape[1]):
            thickmaxdata.append(np.max(self.thick_matrix[:, l]))
        self.thick_max = np.array(thickmaxdata)

        anglematdata = []
        for v in self.layers.itervalues():
            anglematdata.append(v.angle)
        self.angle_matrix = np.fliplr(np.rot90(np.array(anglematdata), -1))

        return self.thick_matrix, self.thick_max, self.angle_matrix
Example #13
0
 def _setupnames(self, names, vlst, sort=None, **dims):
     """Sets up the names dictionary for the plot"""
     if isinstance(names, dict):
         return OrderedDict(names.items())
     if isinstance(names, (str, unicode)):
         names = [names]
     if isinstance(vlst, (str, unicode)):
         vlst = [vlst]
     iter_dims = OrderedDict()
     for key, val in sorted(dims.items()):
         # try if iterable
         try:
             iter(val)
             iter_dims[key] = dims.pop(key)
         except TypeError:
             pass
     if vlst is None:
         raise ValueError(
             "vlst must not be None if names is not a dictionary!")
     if sort is None:
         zipped_keys = ['var'] + iter_dims.keys()
         zipped_dims = product(vlst, *iter_dims.itervalues())
     else:
         zipped_keys = list(sort)
         if not set(zipped_keys) == set(iter_dims.keys() + ['var']):
             raise ValueError(
                 "Sorting parameter (%s) differs from iterable "
                 "dimensions (%s)!" % (
                     ', '.join(zipped_keys),
                     ', '.join(iter_dims.keys() + ['var'])))
         zipped_dims = product(*[
             vlst if key == 'var' else iter_dims[key]
             for key in sort])
     if names is None:
         names = OrderedDict([
             ('line%i' % i, {
                 dim: val for dim, val in zip(zipped_keys, dimstuple)})
             for i, dimstuple in enumerate(zipped_dims)])
     else:  # assume a list of strings
         names = OrderedDict([
             (str(name).format(i), {
                 dim: val for dim, val in zip(zipped_keys, dimstuple)})
             for i, (name, dimstuple) in enumerate(izip(cycle(names),
                                                        zipped_dims))])
     # update for non-iterable dimensions
     for settings in names.itervalues():
         for dim, val in dims.items():
             settings[dim] = val
     return names
Example #14
0
    def _make_raw_brackets_from_divisions(self):
        """Returns an OrderedDict mapping bracket names (normally numbers)
        to lists."""
        brackets = OrderedDict()
        teams = list(self.teams)
        for team in teams:
            # Converting from bracket's name to a float (so it can pretend to be a Bracket)
            division = float(team.division.name)
            if division in brackets:
                brackets[division].append(team)
            else:
                brackets[division] = [team]

        print "------"

        # Assigning bye teams as needed
        for bracket in brackets.itervalues():
            if len(bracket) % 2 != 0:
                from debate.models import Institution,Team
                bye_tournament = bracket[0].tournament
                bye_institution, created = Institution.objects.get_or_create(
                    name="Byes"
                )
                bye_reference = "Bye %s" % bracket[0].division
                bye_division = bracket[0].division
                bye_team = Team(
                    institution = bye_institution,
                    reference = bye_reference,
                    short_reference = "Bye",
                    tournament= bye_tournament,
                    type = "B",
                    use_institution_prefix = False,
                    division = bye_division,
                    cannot_break = True
                )
                bye_team.aff_count = 0
                bye_team.neg_count = 0
                bye_team.save()
                bracket.append(bye_team)
                print "\t Created a bye team for divison %s" % bracket[0].division

        # Assigning subranks - fixed based on alphabetical
        for bracket in brackets.itervalues():
            bracket.sort(key=lambda x: x.short_name, reverse=False)
            for i, team in enumerate(bracket):
                i += 1
                team.subrank = i

        return brackets
Example #15
0
  def Run(self, runner):
    """Iterates over several runs and handles the output."""
    traces = OrderedDict()
    for stdout in runner():
      for line in stdout.strip().splitlines():
        match = GENERIC_RESULTS_RE.match(line)
        if match:
          stddev = ""
          graph = match.group(1)
          trace = match.group(2)
          body = match.group(3)
          units = match.group(4)
          match_stddev = RESULT_STDDEV_RE.match(body)
          match_list = RESULT_LIST_RE.match(body)
          if match_stddev:
            result, stddev = map(str.strip, match_stddev.group(1).split(","))
            results = [result]
          elif match_list:
            results = map(str.strip, match_list.group(1).split(","))
          else:
            results = [body.strip()]

          trace_result = traces.setdefault(trace, Results([{
            "graphs": self.graphs + [graph, trace],
            "units": (units or self.units).strip(),
            "results": [],
            "stddev": "",
          }], []))
          trace_result.traces[0]["results"].extend(results)
          trace_result.traces[0]["stddev"] = stddev

    return reduce(lambda r, t: r + t, traces.itervalues(), Results())
def uniquify_points_and_return_input_index_to_unique_index_map( pts, threshold = 0 ):
	'''
	Given a sequence of N points 'pts',
	and an optional 'threshold' indicating how many decimal places of accuracy (default: 0)
	returns two items:
	   a sequence of all the unique elements in 'pts'
	   and
	   a list of length N where the i-th item in the list tells you where
	   pts[i] can be found in the unique elements.
	'''
	
	from collections import OrderedDict
	unique_pts = OrderedDict()
	pts_map = []
	## Add rounded points to a dictionary and set the key to
	## ( the index into the ordered dictionary, the non-rounded point )
	for i, ( pt, rounded_pt ) in enumerate( zip( pts, map( tuple, asarray( pts ).round( threshold ) ) ) ):
		index = unique_pts.setdefault( rounded_pt, ( len( unique_pts ), pt ) )[0]
		## For fancier schemes:
		# index = unique_pts.setdefault( rounded_pt, ( len( unique_pts ), [] ) )[0]
		# unique_pts[ rounded_pt ][1].append( pt )
		pts_map.append( index )
	
	## Return the original resolution points.
	## The average of all points that round:
	# return [ tuple( average( pt, axis = 0 ) ) for i, pt in unique_pts.itervalues() ], pts_map
	## The closest point to the rounded point:
	# return [ tuple( pt[ abs( asarray( pt ).round( threshold ) - asarray( pt ) ).sum(axis=1).argmin() ] ) for i, pt in unique_pts.itervalues() ], pts_map
	## Simplest, the first rounded point:
	return [ tuple( pt ) for i, pt in unique_pts.itervalues() ], pts_map
Example #17
0
def find_desktop_files(dirs=DESKTOP_DIRS):
    """
    :param list dirs:
    :rtype: list
    """

    all_files = chain.from_iterable(
        map(lambda f: os.path.join(f_path, f), find_files(f_path, '*.desktop')) for f_path in dirs)

    # dedup desktop file according to folow XDG data dir order
    # specifically the first file name (i.e. firefox.desktop) take precedence
    # and other files with the same name shoudl be ignored
    deduped_file_dict = OrderedDict()
    for file_path in all_files:
        file_name = os.path.basename(file_path)
        if file_name not in deduped_file_dict:
            deduped_file_dict[file_name] = file_path
    deduped_files = deduped_file_dict.itervalues()

    blacklisted_dirs_srt = Settings.get_instance().get_property('blacklisted-desktop-dirs')
    blacklisted_dirs = blacklisted_dirs_srt.split(':') if blacklisted_dirs_srt else []
    for file in deduped_files:
        try:
            if any([force_unicode(file).startswith(dir) for dir in blacklisted_dirs]):
                continue
        except UnicodeDecodeError:
            continue

        yield file
Example #18
0
class FieldSet(object):
    def __init__(self, delegate, schema_type):
        from pygtkhelpers.proxy import ProxyGroup, proxy_for

        self.delegate = delegate
        self.schema = schema_type()
        self.proxies = ProxyGroup()
        self.fields = OrderedDict()
        self.proxies.connect('changed', self._on_proxies_changed)
        for name, element in self.schema.items():
            self._setup_widget(name, element)

    def _setup_widget(self, name, element):
        widget = getattr(self.delegate, name, None)
        #XXX (AA) this will always be the case, we are running too soon
        if widget is None:
            widget = widget_for(element)
            setattr(self.delegate, name, widget)
        field = self.fields[name] = Field(element, widget=widget)
        field_name = element.properties.get('label', name)
        field.set_label(field_name.capitalize())
        self.proxies.add_proxy(name, field.proxy)

    def _on_proxies_changed(self, group, proxy, name, value):
        self.schema[name].set(value)

    def layout_as_table(self):
        # XXX: turn to utility function
        table = gtk.Table(len(self.fields), 2)
        table.set_row_spacings(6)
        table.set_col_spacings(6)
        table.set_border_width(6)
        for i, field_i in enumerate(self.fields.itervalues()):
            field_i.layout_as_table(table, i)
        return table
def smallest_subarray_covering_set(A, Q):
    """Find smallest range (i,j) s.t. all q in Q are in A[i..j]."""

    # handle 0-length covering
    if not Q:
        return 0, -1

    # start with no best and an empty candidate covering range
    Q = set(Q)  # want O(1) membership test
    min_size = None
    min_covering_range = None
    cand_locs = OrderedDict()

    # stream elements of A, maintaining the shortest covering range
    # that ends at the current element
    for j, elem in enumerate(A):

        # skip elements that can't contribute to a covering
        if elem not in Q:
            continue

        # extend the candidate range with the current elem,
        # removing any previous instance of the same elem
        cand_locs.pop(elem, None)
        cand_locs[elem] = j  # will be added in final position

        # if the new candidate is legal and better, make it the current best
        if len(cand_locs) == len(Q):
            i = cand_locs.itervalues().next()  # get front position, O(1) time
            if min_size is None or j - i < min_size:
                min_size = j - i
                min_covering_range = i, j

    return min_covering_range
def overwrite_asdc_authors(string_authors,  asdc_auth):
    """
    takes the authors string and update regarding what the admins decide
    param: take a long string of authors
           authors to underline
    return: string with updated information for asdc author
    """

    repls = {'<b>': '', '</b>': '', '...': ''}
    string_clean = reduce(lambda a, kv: a.replace(*kv), repls.iteritems(), string_authors)
    author_list = string_clean.split(";")
    asdc_auth = OrderedDict((auth.split("_")[0], auth.split("_")[1]) for auth in asdc_auth)
    auth_dict = OrderedDict((auth.split(",")[0].lower().strip().replace(" ", ""), auth) for auth in author_list)


    append_last_element = False
    if next(reversed(auth_dict)).strip() == 'et al.':
        auth_dict.popitem()
        append_last_element = True

    for auth in asdc_auth.keys():
        auth_dict[auth] = '<b> %s </b>' % (asdc_auth[auth].replace("%2C", ", ")).replace("+", " ")


    if append_last_element:
        auth_dict.update({'  et al.': '  et al.'})

    return ';'.join(auth_dict.itervalues())
Example #21
0
class MemoryTypeAttributes(MemoryBase, Mapping):

    def __init__(self, owner):
        self._owner = owner
        # NOTE: OrderedDict used to make debug more convenient
        self._items = OrderedDict()  # {}

    owner = roproperty("_owner")

    def on_complete(self, item):
        self._items[item.name] = item

    def new_sketch(self, restore=False):
        return MemoryTypeAttributeSketch(self)

    # unsafe
    def compose(self, ident=u"", file=None):
        if self.__dict__.get("_items"):
            file.write(u"%s<Attributes>\n" % ident)
            for attribute in self._items.itervalues():
                attribute.compose(ident=ident + u"\t", file=file)
            file.write(u"%s</Attributes>\n" % ident)

    def __getitem__(self, name):
        return self._items[name]

    def __iter__(self):
        return iter(self._items)

    def __len__(self):
        return len(self._items)

    def __str__(self):
        return "attributes of %s" % self._owner
Example #22
0
def box_data(keys, data=None):
    """
    Box Data into appropiate pandas object

    Parameters
    ----------
    keys : dict or iterable
        Either keys list or Result Dict
    data : iterable
        list of data
    """
    if isinstance(keys, dict):
        rdict = keys
    else:
        rdict = OrderedDict(zip(keys, data))

    test = next(rdict.itervalues())
    if np.isscalar(test):
        return Series(rdict)

    if isinstance(test, Series):
        return DataFrame(rdict)

    if isinstance(test, DataFrame):
        return Panel(rdict)

    if isinstance(test, ColumnPanel):
        data = OrderedDict([(k, v.to_panel()) for k, v  in rdict.iteritems()])
        return Panel4D(data)

    if isinstance(test, Panel):
        return Panel4D(data)

    return rdict
Example #23
0
class DocumentLengthTable:

	def __init__(self):
		self.table = OrderedDict()

	def __len__(self):
		return len(self.table)

	def write(self, filename='default.dlt'):
		with open(filename, 'w') as f:
			for docid in self.table:
				line = '%s %d\n' % (docid, self.table[docid])
				f.write(line)


	def add(self, docid, length):
		self.table[docid] = length

	def get_length(self, docid):
		if docid in self.table:
			return self.table[docid]
		else:
			raise LookupError('%s not found in table' % str(docid))

	def get_average_length(self):
		sum = 0
		for length in self.table.itervalues():
			sum += length
		return float(sum) / float(len(self.table))
Example #24
0
class SVDPeripheralRegister:
	def __init__(self, svd_elem, parent):
		self.parent = parent
		self.name = str(svd_elem.name)
		self.description = str(svd_elem.description)
		self.offset = int(str(svd_elem.addressOffset),0)
		self.size = int(str(svd_elem.size),0)
		fields = svd_elem.fields.getchildren()
		self.fields = OrderedDict()
		for f in fields:
			self.fields[str(f.name)] = SVDPeripheralRegisterField(f, self)
	
	def refactor_parent(self, parent):
		self.parent = parent
		try:
			fields = self.fields.itervalues()
		except AttributeError:
			fields = self.fields.values()
		for f in fields:
			f.refactor_parent(self)

	def address(self):
		return self.parent.base_address + self.offset
	
	def __unicode__(self):
		return str(self.name)
Example #25
0
    def check_for_add_to_toolbars(self, plugin):
        from calibre.gui2.preferences.toolbar import ConfigWidget
        from calibre.customize import InterfaceActionBase

        if not isinstance(plugin, InterfaceActionBase):
            return

        all_locations = OrderedDict(ConfigWidget.LOCATIONS)
        plugin_action = plugin.load_actual_plugin(self.gui)
        installed_actions = OrderedDict([
            (key, list(gprefs.get('action-layout-'+key, [])))
            for key in all_locations])

        # If already installed in a GUI container, do nothing
        for action_names in installed_actions.itervalues():
            if plugin_action.name in action_names:
                return

        allowed_locations = [(key, text) for key, text in
                all_locations.iteritems() if key
                not in plugin_action.dont_add_to]
        if not allowed_locations:
            return # This plugin doesn't want to live in the GUI

        from calibre.gui2.dialogs.choose_plugin_toolbars import ChoosePluginToolbarsDialog
        d = ChoosePluginToolbarsDialog(self, plugin_action, allowed_locations)
        if d.exec_() == d.Accepted:
            for key, text in d.selected_locations():
                installed_actions = list(gprefs.get('action-layout-'+key, []))
                installed_actions.append(plugin_action.name)
                gprefs['action-layout-'+key] = tuple(installed_actions)
def tx_modes_plot(consensus_data, ordered_genomes, tx_mode_plot_tgt):
    ordered_groups = ['transMap', 'transMap+TM', 'transMap+TMR', 'transMap+TM+TMR', 'TM', 'TMR', 'TM+TMR', 'CGP', 'PB',
                      'Other']
    ordered_groups = OrderedDict([[frozenset(x.split('+')), x] for x in ordered_groups])

    def split_fn(s):
        return ordered_groups.get(frozenset(s['Transcript Modes'].replace('aug', '').split(',')), 'Other')

    modes_df = json_biotype_counter_to_df(consensus_data, 'Transcript Modes')
    df = modes_df.pivot(index='genome', columns='Transcript Modes').transpose().reset_index()
    df['Modes'] = df.apply(split_fn, axis=1)
    df = df[['Modes'] + ordered_genomes]
    ordered_values = [x for x in ordered_groups.itervalues() if x in set(df['Modes'])]
    with tx_mode_plot_tgt.open('w') as outf, PdfPages(outf) as pdf:
        title_string = 'Transcript modes in protein coding consensus gene set'
        ylabel = 'Number of transcripts'
        if len(ordered_genomes) > 1:
            df['Ordered Modes'] = pd.Categorical(df['Modes'], ordered_values, ordered=True)
            df = df.sort_values('Ordered Modes')
            df = df[['Ordered Modes'] + ordered_genomes].set_index('Ordered Modes')
            df = df.fillna(0)
            generic_stacked_barplot(df, pdf, title_string, df.index, ylabel, ordered_genomes, 'Transcript mode(s)',
                                    bbox_to_anchor=(1.25, 0.7))

        else:
            generic_barplot(pd.melt(df, id_vars='Modes'), pdf, 'Transcript mode(s)', ylabel, title_string, x='Modes',
                            y='value', order=ordered_values)
Example #27
0
class TreeItem(ItemModel):
    def __init__(self, data, parent=None):
        ItemModel.__init__(self, data, parent)

        self.children = OrderedDictionary()

    def remove_child(self, key_value):
        if key_value not in self.children:
            raise KeyError('No such child exists')

        del self.children[key_value]

    def row(self):
        """ This method is necessary because of the parent-child node structure of the model, where there is no simple
            way to find the overall relationship of all the items in the database, rather just one items' relationship
            with those surrounding it.
        :return: int
        """
        if not self.parent:
            return 0

        return self.parent.children.values().index(self)

    def __iter__(self):
        return self.children.itervalues()
Example #28
0
class InterfaceSection (object):
	def __init__ (self, properties = None, traces = None, controls = None, title = "", name = None):
		self.name = name or title
		self.title = title
		self.properties = properties or []
		self.controls = controls or []
		self.traces = OrderedDict()

		if traces is not None:
			for t in traces:
				try:
					self.traces[t["name"]] = t
				except KeyError:
					self.traces[t["title"]] = t

	def __getitem__ (self, key):
		if key in ("name", "title", "properties", "controls", "traces"):
			return getattr(self, key)
		else:
			raise KeyError

	def output (self):
		return {
			"name": self.name,
			"title": self.title,
			"traces": [_trace(t) for t in self.traces.itervalues()],
			"properties": [_prop(p) for p in self.properties if p.type not in ("Image",)],
			"images": [_img(p) for p in self.properties if p.type == "Image"],
			"controls": [_control(c) for c in self.controls]
		}
Example #29
0
    def getDeviceShortNames(self, devicepaths, pciids):
        shortnames = OrderedDict()
        namecount = {}
        try:
        # Initialise PciIdsParser, throws customExceptions.PciIdsFileNotFound
        # if fail
            pciIdsParser = parseutil.PciIdsParser(pciids)
        except customExceptions.PciIdsFileNotFound:
            message.addError("pci.ids file could not be located in tool "
                             "directory: %s. " % paths.CURRENTDIR +
                             "Device "
                             "names could not be obtained. Please ensure that "
                             "the file is in the directory.",
                             False)
        else:

            for devicepath in devicepaths:
                # Add entry to dictionary shortnames
                shortnames[devicepath] = self.getClassName(
                    devicepath, pciIdsParser)

            namelist = []
            for value in shortnames.itervalues():
                namelist.append(value)

            listnumberer = util.ListNumberer(namelist)
            for devicepath in shortnames.iterkeys():
                shortnames[devicepath] = listnumberer.getName(
                    shortnames[devicepath])

        return shortnames
Example #30
0
class DictVal(Collapsible):
	def __init__(self, expanded=True):
		super(Dict, self).__init__(expanded)
		self.items = OrderedDict()
		
	def render_items(self):
		r = []
		for key, item in self.items.iteritems():
			r += [t(key), t(":"), indent(), nl()]
			r += [ElementTag(item)]
			r += [dedent(), nl()]
		return r

	def __getitem__(self, i):
		return self.items[i]

	def __setitem__(self, i, v):
		self.items[i] = v
		v.parent = self

	def fix_parents(self):
		super(Dict, self).fix_parents()
		self._fix_parents(self.items.values())

	def flatten(self):
		return [self] + [v.flatten() for v in self.items.itervalues() if isinstance(v, Node)]
		#skip Widgets, as in settings

	def add(self, (key, val)):
		assert(not self.items.has_key(key))
		self.items[key] = val
		assert(isinstance(key, str))
		assert(isinstance(val, element.Element))
		val.parent = self
Example #31
0
class Environment(object):
    """Environment within which all agents operate."""

    valid_actions = [None, 'forward', 'left', 'right']
    valid_inputs = {
        'light': TrafficLight.valid_states,
        'oncoming': valid_actions,
        'left': valid_actions,
        'right': valid_actions
    }
    valid_headings = [(1, 0), (0, -1), (-1, 0), (0, 1)]  # E, N, W, S
    hard_time_limit = -100  # Set a hard time limit even if deadline is not enforced.

    def __init__(self, verbose=False, num_dummies=100, grid_size=(8, 6)):
        self.num_dummies = num_dummies  # Number of dummy driver agents in the environment
        self.verbose = verbose  # If debug output should be given

        # Initialize simulation variables
        self.done = False
        self.t = 0
        self.agent_states = OrderedDict()
        self.step_data = {}
        self.success = None

        # Road network
        self.grid_size = grid_size  # (columns, rows)
        self.bounds = (1, 2, self.grid_size[0], self.grid_size[1] + 1)
        self.block_size = 100
        self.hang = 0.6
        self.intersections = OrderedDict()
        self.roads = []
        for x in xrange(self.bounds[0], self.bounds[2] + 1):
            for y in xrange(self.bounds[1], self.bounds[3] + 1):
                # A traffic light at each intersection
                self.intersections[(x, y)] = TrafficLight()

        for a in self.intersections:
            for b in self.intersections:
                if a == b:
                    continue
                if (abs(a[0] - b[0]) +
                        abs(a[1] - b[1])) == 1:  # L1 distance = 1
                    self.roads.append((a, b))

        # Add environment boundaries
        for x in xrange(self.bounds[0], self.bounds[2] + 1):
            self.roads.append(
                ((x, self.bounds[1] - self.hang), (x, self.bounds[1])))
            self.roads.append(
                ((x, self.bounds[3] + self.hang), (x, self.bounds[3])))
        for y in xrange(self.bounds[1], self.bounds[3] + 1):
            self.roads.append(
                ((self.bounds[0] - self.hang, y), (self.bounds[0], y)))
            self.roads.append(
                ((self.bounds[2] + self.hang, y), (self.bounds[2], y)))

        # Create dummy agents
        for i in xrange(self.num_dummies):
            self.create_agent(DummyAgent)

        # Primary agent and associated parameters
        self.primary_agent = None  # to be set explicitly
        self.enforce_deadline = False

        # Trial data (updated at the end of each trial)
        self.trial_data = {
            'testing': False,  # if the trial is for testing a learned policy
            'initial_distance': 0,  # L1 distance from start to destination
            'initial_deadline': 0,  # given deadline (time steps) to start with
            'net_reward': 0.0,  # total reward earned in current trial
            'final_deadline':
            None,  # deadline value (time remaining) at the end
            'actions': {
                0: 0,
                1: 0,
                2: 0,
                3: 0,
                4: 0
            },  # violations and accidents
            'success': 0  # whether the agent reached the destination in time
        }

    def create_agent(self, agent_class, *args, **kwargs):
        """ When called, create_agent creates an agent in the environment. """

        agent = agent_class(self, *args, **kwargs)
        self.agent_states[agent] = {
            'location': random.choice(self.intersections.keys()),
            'heading': (0, 1)
        }
        return agent

    def set_primary_agent(self, agent, enforce_deadline=False):
        """ When called, set_primary_agent sets 'agent' as the primary agent.
            The primary agent is the smartcab that is followed in the environment. """

        self.primary_agent = agent
        agent.primary_agent = True
        self.enforce_deadline = enforce_deadline

    def reset(self, testing=False):
        """ This function is called at the beginning of a new trial. """

        self.done = False
        self.t = 0

        # Reset status text
        self.step_data = {}

        # Reset traffic lights
        for traffic_light in self.intersections.itervalues():
            traffic_light.reset()

        # Pick a start and a destination
        start = random.choice(self.intersections.keys())
        destination = random.choice(self.intersections.keys())

        # Ensure starting location and destination are not too close
        while self.compute_dist(start, destination) < 4:
            start = random.choice(self.intersections.keys())
            destination = random.choice(self.intersections.keys())

        start_heading = random.choice(self.valid_headings)
        distance = self.compute_dist(start, destination)
        deadline = distance * 5  # 5 time steps per intersection away
        if (self.verbose == True):  # Debugging
            print "Environment.reset(): Trial set up with start = {}, destination = {}, deadline = {}".format(
                start, destination, deadline)

        # Create a map of all possible initial positions
        positions = dict()
        for location in self.intersections:
            positions[location] = list()
            for heading in self.valid_headings:
                positions[location].append(heading)

        # Initialize agent(s)
        for agent in self.agent_states.iterkeys():

            if agent is self.primary_agent:
                self.agent_states[agent] = {
                    'location': start,
                    'heading': start_heading,
                    'destination': destination,
                    'deadline': deadline
                }
            # For dummy agents, make them choose one of the available
            # intersections and headings still in 'positions'
            else:
                intersection = random.choice(positions.keys())
                heading = random.choice(positions[intersection])
                self.agent_states[agent] = {
                    'location': intersection,
                    'heading': heading,
                    'destination': None,
                    'deadline': None
                }
                # Now delete the taken location and heading from 'positions'
                positions[intersection] = list(
                    set(positions[intersection]) - {heading})
                if positions[intersection] == list(
                ):  # No headings available for intersection
                    del positions[
                        intersection]  # Delete the intersection altogether

            agent.reset(destination=(destination
                                     if agent is self.primary_agent else None),
                        testing=testing)
            if agent is self.primary_agent:
                # Reset metrics for this trial (step data will be set during the step)
                self.trial_data['testing'] = testing
                self.trial_data['initial_deadline'] = deadline
                self.trial_data['final_deadline'] = deadline
                self.trial_data['net_reward'] = 0.0
                self.trial_data['actions'] = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}
                self.trial_data['parameters'] = {
                    'e': agent.epsilon,
                    'a': agent.alpha
                }
                self.trial_data['success'] = 0

    def step(self):
        """ This function is called when a time step is taken turing a trial. """

        # Pretty print to terminal
        print ""
        print "/-------------------"
        print "| Step {} Results".format(self.t)
        print "\-------------------"
        print ""

        if (self.verbose == True):  # Debugging
            print "Environment.step(): t = {}".format(self.t)

        # Update agents, primary first
        if self.primary_agent is not None:
            self.primary_agent.update()

        for agent in self.agent_states.iterkeys():
            if agent is not self.primary_agent:
                agent.update()

        # Update traffic lights
        for intersection, traffic_light in self.intersections.iteritems():
            traffic_light.update(self.t)

        if self.primary_agent is not None:
            # Agent has taken an action: reduce the deadline by 1
            agent_deadline = self.agent_states[
                self.primary_agent]['deadline'] - 1
            self.agent_states[self.primary_agent]['deadline'] = agent_deadline

            if agent_deadline <= self.hard_time_limit:
                self.done = True
                self.success = False
                if self.verbose:  # Debugging
                    print "Environment.step(): Primary agent hit hard time limit ({})! Trial aborted.".format(
                        self.hard_time_limit)
            elif self.enforce_deadline and agent_deadline <= 0:
                self.done = True
                self.success = False
                if self.verbose:  # Debugging
                    print "Environment.step(): Primary agent ran out of time! Trial aborted."

        self.t += 1

    def sense(self, agent):
        """ This function is called when information is requested about the sensor
            inputs from an 'agent' in the environment. """

        assert agent in self.agent_states, "Unknown agent!"

        state = self.agent_states[agent]
        location = state['location']
        heading = state['heading']
        if (self.intersections[location].state and heading[1] != 0) or (
            (not self.intersections[location].state) and heading[0] != 0):
            light = 'green'
        else:
            light = 'red'

        # Populate oncoming, left, right
        oncoming = None
        left = None
        right = None
        for other_agent, other_state in self.agent_states.iteritems():
            if agent == other_agent or location != other_state['location'] or \
                (heading[0] == other_state['heading'][0] and heading[1] == other_state['heading'][1]):
                continue
            # For dummy agents, ignore the primary agent
            # This is because the primary agent is not required to follow the waypoint
            if other_agent == self.primary_agent:
                continue
            other_heading = other_agent.get_next_waypoint()
            if (heading[0] * other_state['heading'][0] +
                    heading[1] * other_state['heading'][1]) == -1:
                if oncoming != 'left':  # we don't want to override oncoming == 'left'
                    oncoming = other_heading
            elif (heading[1] == other_state['heading'][0]
                  and -heading[0] == other_state['heading'][1]):
                if right != 'forward' and right != 'left':  # we don't want to override right == 'forward or 'left'
                    right = other_heading
            else:
                if left != 'forward':  # we don't want to override left == 'forward'
                    left = other_heading

        return {
            'light': light,
            'oncoming': oncoming,
            'left': left,
            'right': right
        }

    def get_deadline(self, agent):
        """ Returns the deadline remaining for an agent. """

        return self.agent_states[agent][
            'deadline'] if agent is self.primary_agent else None

    def act(self, agent, action):
        """ Consider an action and perform the action if it is legal.
            Receive a reward for the agent based on traffic laws. """

        assert agent in self.agent_states, "Unknown agent!"

        if action not in self.valid_actions:
            print("Illegal action: ")
            print(action)

        assert action in self.valid_actions, "Invalid action!"

        state = self.agent_states[agent]
        location = state['location']
        heading = state['heading']
        if (self.intersections[location].state and heading[1] != 0) or (
            (not self.intersections[location].state) and heading[0] != 0):
            light = 'green'
        else:
            light = 'red'
        inputs = self.sense(agent)

        # Assess whether the agent can move based on the action chosen.
        # Either the action is okay to perform, or falls under 4 types of violations:
        # 0: Action okay
        # 1: Minor traffic violation
        # 2: Major traffic violation
        # 3: Minor traffic violation causing an accident
        # 4: Major traffic violation causing an accident
        violation = 0

        # Reward scheme
        # First initialize reward uniformly random from [-1, 1]
        reward = 2 * random.random() - 1

        # Create a penalty factor as a function of remaining deadline
        # Scales reward multiplicatively from [0, 1]
        fnc = self.t * 1.0 / (
            self.t + state['deadline']) if agent.primary_agent else 0.0
        gradient = 10

        # No penalty given to an agent that has no enforced deadline
        penalty = 0

        # If the deadline is enforced, give a penalty based on time remaining
        if self.enforce_deadline:
            penalty = (math.pow(gradient, fnc) - 1) / (gradient - 1)

        # Agent wants to drive forward:
        if action == 'forward':
            if light != 'green':  # Running red light
                violation = 2  # Major violation
                if inputs['left'] == 'forward' or inputs[
                        'right'] == 'forward':  # Cross traffic
                    violation = 4  # Accident

        # Agent wants to drive left:
        elif action == 'left':
            if light != 'green':  # Running a red light
                violation = 2  # Major violation
                if inputs['left'] == 'forward' or inputs[
                        'right'] == 'forward':  # Cross traffic
                    violation = 4  # Accident
                elif inputs[
                        'oncoming'] == 'right':  # Oncoming car turning right
                    violation = 4  # Accident
            else:  # Green light
                if inputs['oncoming'] == 'right' or inputs[
                        'oncoming'] == 'forward':  # Incoming traffic
                    violation = 3  # Accident
                else:  # Valid move!
                    heading = (heading[1], -heading[0])

        # Agent wants to drive right:
        elif action == 'right':
            if light != 'green' and inputs[
                    'left'] == 'forward':  # Cross traffic
                violation = 3  # Accident
            else:  # Valid move!
                heading = (-heading[1], heading[0])

        # Agent wants to perform no action:
        elif action == None:
            if light == 'green':
                violation = 1  # Minor violation

        # Did the agent attempt a valid move?
        if violation == 0:
            if action == agent.get_next_waypoint(
            ):  # Was it the correct action?
                reward += 2 - penalty  # (2, 1)
            elif action == None and light != 'green' and agent.get_next_waypoint(
            ) == 'right':
                # valid action but incorrect (idling at red light, when we should have gone right on red)
                reward += 1 - penalty  # (1, 0)
            elif action == None and light != 'green':  # Was the agent stuck at a red light?
                reward += 2 - penalty  # (2, 1)
            else:  # Valid but incorrect
                reward += 1 - penalty  # (1, 0)

            # Move the agent
            if action is not None:
                location = (
                    (location[0] + heading[0] - self.bounds[0]) %
                    (self.bounds[2] - self.bounds[0] + 1) + self.bounds[0],
                    (location[1] + heading[1] - self.bounds[1]) %
                    (self.bounds[3] - self.bounds[1] + 1) + self.bounds[1]
                )  # wrap-around
                state['location'] = location
                state['heading'] = heading
        # Agent attempted invalid move
        else:
            if violation == 1:  # Minor violation
                reward += -5
            elif violation == 2:  # Major violation
                reward += -10
            elif violation == 3:  # Minor accident
                reward += -20
            elif violation == 4:  # Major accident
                reward += -40

        # Did agent reach the goal after a valid move?
        if agent is self.primary_agent:
            if state['location'] == state['destination']:
                # Did agent get to destination before deadline?
                if state['deadline'] >= 0:
                    self.trial_data['success'] = 1

                # Stop the trial
                self.done = True
                self.success = True

                if (self.verbose == True):  # Debugging
                    print "Environment.act(): Primary agent has reached destination!"

            if (self.verbose == True):  # Debugging
                print "Environment.act() [POST]: location: {}, heading: {}, action: {}, reward: {}".format(
                    location, heading, action, reward)

            # Update metrics
            self.step_data['t'] = self.t
            self.step_data['violation'] = violation
            self.step_data['state'] = agent.get_state()
            self.step_data['deadline'] = state['deadline']
            self.step_data['waypoint'] = agent.get_next_waypoint()
            self.step_data['inputs'] = inputs
            self.step_data['light'] = light
            self.step_data['action'] = action
            self.step_data['reward'] = reward

            self.trial_data['final_deadline'] = state['deadline'] - 1
            self.trial_data['net_reward'] += reward
            self.trial_data['actions'][violation] += 1

            if (self.verbose == True):  # Debugging
                print "Environment.act(): Step data: {}".format(self.step_data)
        return reward

    def compute_dist(self, a, b):
        """ Compute the Manhattan (L1) distance of a spherical world. """

        dx1 = abs(b[0] - a[0])
        dx2 = abs(self.grid_size[0] - dx1)
        dx = dx1 if dx1 < dx2 else dx2

        dy1 = abs(b[1] - a[1])
        dy2 = abs(self.grid_size[1] - dy1)
        dy = dy1 if dy1 < dy2 else dy2

        return dx + dy
Example #32
0
class Environment(object):
    """Environment within which all agents operate."""

    valid_actions = [None, 'forward', 'left', 'right']
    valid_inputs = {
        'light': TrafficLight.valid_states,
        'oncoming': valid_actions,
        'left': valid_actions,
        'right': valid_actions
    }
    valid_headings = [(1, 0), (0, -1), (-1, 0), (0, 1)]  # ENWS
    hard_time_limit = -100  # even if enforce_deadline is False, end trial when deadline reaches this value (to avoid deadlocks)

    def __init__(self):
        self.done = False
        self.results = []
        self.t = 0
        self.agent_states = OrderedDict()
        self.status_text = ""
        self.count = 1

        # Road network
        self.grid_size = (8, 6)  # (cols, rows)
        self.bounds = (1, 1, self.grid_size[0], self.grid_size[1])
        self.block_size = 100
        self.intersections = OrderedDict()
        self.roads = []
        for x in xrange(self.bounds[0], self.bounds[2] + 1):
            for y in xrange(self.bounds[1], self.bounds[3] + 1):
                self.intersections[(x, y)] = TrafficLight(
                )  # a traffic light at each intersection

        for a in self.intersections:
            for b in self.intersections:
                if a == b:
                    continue
                if (abs(a[0] - b[0]) +
                        abs(a[1] - b[1])) == 1:  # L1 distance = 1
                    self.roads.append((a, b))

        # Dummy agents
        self.num_dummies = 3  # no. of dummy agents
        for i in xrange(self.num_dummies):
            self.create_agent(DummyAgent)

        # Primary agent
        self.primary_agent = None  # to be set explicitly
        self.enforce_deadline = False

    def create_agent(self, agent_class, *args, **kwargs):
        agent = agent_class(self, *args, **kwargs)
        self.agent_states[agent] = {
            'location': random.choice(self.intersections.keys()),
            'heading': (0, 1)
        }
        return agent

    def set_primary_agent(self, agent, enforce_deadline=False):
        self.primary_agent = agent
        self.enforce_deadline = enforce_deadline

    def reset(self):
        self.done = False
        self.t = 0

        # Reset traffic lights
        for traffic_light in self.intersections.itervalues():
            traffic_light.reset()

        # Pick a start and a destination
        start = random.choice(self.intersections.keys())
        destination = random.choice(self.intersections.keys())

        # Ensure starting location and destination are not too close
        while self.compute_dist(start, destination) < 4:
            start = random.choice(self.intersections.keys())
            destination = random.choice(self.intersections.keys())

        start_heading = random.choice(self.valid_headings)
        deadline = self.compute_dist(start, destination) * 5
        # print "Environment.reset(): Trial set up with start = {}, destination = {}, deadline = {}".format(start, destination, deadline)

        # Initialize agent(s)
        for agent in self.agent_states.iterkeys():
            self.agent_states[agent] = {
                'location':
                start if agent is self.primary_agent else random.choice(
                    self.intersections.keys()),
                'heading':
                start_heading if agent is self.primary_agent else
                random.choice(self.valid_headings),
                'destination':
                destination if agent is self.primary_agent else None,
                'deadline':
                deadline if agent is self.primary_agent else None
            }
            agent.reset(destination=(
                destination if agent is self.primary_agent else None))

    def step(self):
        #print "Environment.step(): t = {}".format(self.t)  # [debug]

        # Update traffic lights
        for intersection, traffic_light in self.intersections.iteritems():
            traffic_light.update(self.t)

        # Update agents
        for agent in self.agent_states.iterkeys():
            agent.update(self.t)

        self.t += 1
        if self.primary_agent is not None and self.done is not True:
            agent_deadline = self.agent_states[self.primary_agent]['deadline']
            if agent_deadline <= self.hard_time_limit:
                self.done = True
                self.results.append((0, self.primary_agent.score,
                                     self.primary_agent.penalties))
                print "Environment.step(): Primary agent hit hard time limit ({})! Trial aborted.".format(
                    self.hard_time_limit)
            elif self.enforce_deadline and agent_deadline <= 0:
                self.done = True
                # print "0"
                self.results.append((0, self.primary_agent.score,
                                     self.primary_agent.penalties))
                print "Environment.step(): Primary agent ran out of time! Trial aborted."
            self.agent_states[
                self.primary_agent]['deadline'] = agent_deadline - 1

    def sense(self, agent):
        assert agent in self.agent_states, "Unknown agent!"

        state = self.agent_states[agent]
        location = state['location']
        heading = state['heading']
        light = 'green' if (self.intersections[location].state
                            and heading[1] != 0) or (
                                (not self.intersections[location].state)
                                and heading[0] != 0) else 'red'

        # Populate oncoming, left, right
        oncoming = None
        left = None
        right = None
        for other_agent, other_state in self.agent_states.iteritems():
            if agent == other_agent or location != other_state['location'] or (
                    heading[0] == other_state['heading'][0]
                    and heading[1] == other_state['heading'][1]):
                continue
            other_heading = other_agent.get_next_waypoint()
            if (heading[0] * other_state['heading'][0] +
                    heading[1] * other_state['heading'][1]) == -1:
                if oncoming != 'left':  # we don't want to override oncoming == 'left'
                    oncoming = other_heading
            elif (heading[1] == other_state['heading'][0]
                  and -heading[0] == other_state['heading'][1]):
                if right != 'forward' and right != 'left':  # we don't want to override right == 'forward or 'left'
                    right = other_heading
            else:
                if left != 'forward':  # we don't want to override left == 'forward'
                    left = other_heading

        return {
            'light': light,
            'oncoming': oncoming,
            'left': left,
            'right': right
        }  # TODO: make this a namedtuple

    def get_deadline(self, agent):
        return self.agent_states[agent][
            'deadline'] if agent is self.primary_agent else None

    def act(self, agent, action):
        assert agent in self.agent_states, "Unknown agent!"
        assert action in self.valid_actions, "Invalid action!"

        state = self.agent_states[agent]
        location = state['location']
        heading = state['heading']
        light = 'green' if (self.intersections[location].state
                            and heading[1] != 0) or (
                                (not self.intersections[location].state)
                                and heading[0] != 0) else 'red'
        sense = self.sense(agent)

        # Move agent if within bounds and obeys traffic rules
        reward = 0  # reward/penalty
        move_okay = True
        if action == 'forward':
            if light != 'green':
                move_okay = False
        elif action == 'left':
            if light == 'green' and (sense['oncoming'] == None
                                     or sense['oncoming'] == 'left'):
                heading = (heading[1], -heading[0])
            else:
                move_okay = False
        elif action == 'right':
            if light == 'green' or sense['left'] != 'straight':
                heading = (-heading[1], heading[0])
            else:
                move_okay = False

        if move_okay:
            # Valid move (could be null)
            if action is not None:
                # Valid non-null move
                location = (
                    (location[0] + heading[0] - self.bounds[0]) %
                    (self.bounds[2] - self.bounds[0] + 1) + self.bounds[0],
                    (location[1] + heading[1] - self.bounds[1]) %
                    (self.bounds[3] - self.bounds[1] + 1) + self.bounds[1]
                )  # wrap-around
                #if self.bounds[0] <= location[0] <= self.bounds[2] and self.bounds[1] <= location[1] <= self.bounds[3]:  # bounded
                state['location'] = location
                state['heading'] = heading
                reward = 2.0 if action == agent.get_next_waypoint(
                ) else -0.5  # valid, but is it correct? (as per waypoint)
            else:
                # Valid null move
                reward = 0.0
        else:
            # Invalid move
            reward = -1.0

        if agent is self.primary_agent:
            if state['location'] == state['destination']:
                if state['deadline'] >= 0:
                    reward += 10  # bonus
                self.done = True
                self.results.append((1, self.primary_agent.score,
                                     self.primary_agent.penalties))
                print "Environment.act(): Primary agent has reached destination!"  # [debug]
            self.status_text = "state: {}\naction: {}\nreward: {}".format(
                agent.get_state(), action, reward)
            # print "Environment.act() [POST]: location: {}, heading: {}, action: {}, reward: {}".format(location, heading, action, reward)  # [debug]

        return reward

    def compute_dist(self, a, b):
        """L1 distance between two points."""
        return abs(b[0] - a[0]) + abs(b[1] - a[1])
Example #33
0
class ResponseHeaders(object):
    """Dictionary-like object holding the headers for the response"""
    def __init__(self):
        self.data = OrderedDict()

    def set(self, key, value):
        """Set a header to a specific value, overwriting any previous header
        with the same name

        :param key: Name of the header to set
        :param value: Value to set the header to
        """
        self.data[key.lower()] = (key, [value])

    def append(self, key, value):
        """Add a new header with a given name, not overwriting any existing
        headers with the same name

        :param key: Name of the header to add
        :param value: Value to set for the header
        """
        if key.lower() in self.data:
            self.data[key.lower()][1].append(value)
        else:
            self.set(key, value)

    def get(self, key, default=missing):
        """Get the set values for a particular header."""
        try:
            return self[key]
        except KeyError:
            if default is missing:
                return []
            return default

    def __getitem__(self, key):
        """Get a list of values for a particular header

        """
        return self.data[key.lower()][1]

    def __delitem__(self, key):
        del self.data[key.lower()]

    def __contains__(self, key):
        return key.lower() in self.data

    def __setitem__(self, key, value):
        self.set(key, value)

    def __iter__(self):
        for key, values in self.data.itervalues():
            for value in values:
                yield key, value

    def items(self):
        return list(self)

    def update(self, items_iter):
        for name, value in items_iter:
            self.set(name, value)

    def __repr__(self):
        return repr(self.data)
Example #34
0
class RecipeContainer(Observable, Configurable, Validatable):
    """Base class for organizing pieces of a FitRecipe.

    RecipeContainers are hierarchical organizations of Parameters and other
    RecipeContainers. This class provides attribute-access to these contained
    objects.  Parameters and other RecipeContainers can be found within the
    hierarchy with the _locateManagedObject method.

    A RecipeContainer can manage dictionaries for that store various objects.
    These dictionaries can be added to the RecipeContainer using the _manage
    method. RecipeContainer methods that add, remove or retrieve objects will
    work with any managed dictionary. This makes it easy to add new types of
    objects to be contained by a RecipeContainer. By default, the
    RecipeContainer is configured to manage an OrderedDict of Parameter
    objects.

    RecipeContainer is an Observable, and observes its managed objects and
    Parameters. This allows hierarchical calculation elements, such as
    ProfileGenerator, to detect changes in Parameters and Restraints on which
    it may depend.

    Attributes
    name            --  A name for this RecipeContainer. Names should be unique
                        within a RecipeContainer and should be valid attribute
                        names.
    _parameters     --  A managed OrderedDict of contained Parameters.
    __managed       --  A list of managed dictionaries. This is used for
                        attribute access, addition and removal.
    _configobjs     --  A set of configurable objects that must know of
                        configuration changes within this object.

    Properties
    names           --  Variable names (read only). See getNames.
    values          --  Variable values (read only). See getValues.
    """

    names = property(lambda self: self.getNames())
    values = property(lambda self: self.getValues())

    def __init__(self, name):
        Observable.__init__(self)
        Configurable.__init__(self)
        validateName(name)
        self.name = name
        self._parameters = OrderedDict()

        self.__managed = []
        self._manage(self._parameters)

        return

    def _manage(self, d):
        """Manage a dictionary of objects.

        This adds the dictionary to the __managed list. Dictionaries in
        __managed are used for attribute access, addition, and removal.
        """
        self.__managed.append(d)
        return

    def _iterManaged(self):
        """Get iterator over managed objects."""
        return chain(*(d.values() for d in self.__managed))

    def iterPars(self, pattern="", recurse=True):
        """Iterate over the Parameters contained in this object.

        Parameters
        ----------
        pattern : str
            Iterate over parameters with names matching this regular
            expression (all parameters by default).
        recurse : bool
            Recurse into managed objects when True (default).
        """
        regexp = re.compile(pattern)
        for par in list(self._parameters.values()):
            if regexp.search(par.name):
                yield par
        if not recurse:
            return
        # Iterate over objects within the managed dictionaries.
        managed = self.__managed[:]
        managed.remove(self._parameters)
        for m in managed:
            for obj in m.values():
                if hasattr(obj, "iterPars"):
                    for par in obj.iterPars(pattern=pattern):
                        yield par
        return

    def __iter__(self):
        """Iterate over top-level parameters."""
        return self._parameters.itervalues()

    def __len__(self):
        """Get number of top-level parameters."""
        return len(self._parameters)

    def __getitem__(self, idx):
        """Get top-level parameters by index."""
        return self._parameters.values()[idx]

    def __getattr__(self, name):
        """Gives access to the contained objects as attributes."""
        arg = self.get(name)
        if arg is None:
            raise AttributeError(name)
        return arg

    # Ensure there is no __dir__ override in the base class.
    assert (getattr(Observable, '__dir__', None) is getattr(
        Configurable, '__dir__', None) is getattr(Validatable, '__dir__', None)
            is getattr(object, '__dir__', None))

    def __dir__(self):
        "Return sorted list of attributes for this object."
        rv = set(dir(type(self)))
        rv.update(self.__dict__)
        # self.get fetches looks up for items in all managed dictionaries.
        # Add keys from each dictionary in self.__managed.
        rv.update(*self.__managed)
        rv = sorted(rv)
        return rv

    # Needed by __setattr__
    _parameters = OrderedDict()
    __managed = []

    def __setattr__(self, name, value):
        """Parameter access and object checking."""
        if name in self._parameters:
            par = self._parameters[name]
            if isinstance(value, Parameter):
                par.value = value.value
            else:
                par.value = value
            return

        m = self.get(name)
        if m is not None:
            raise AttributeError("Cannot set '%s'" % name)

        super(RecipeContainer, self).__setattr__(name, value)
        return

    def __delattr__(self, name):
        """Delete parameters with del.

        This does not allow deletion of non-parameters, as this may require
        configuration changes that are not yet handled in a general way.
        """
        if name in self._parameters:
            self._removeParameter(self._parameters[name])
            return

        m = self.get(name)
        if m is not None:
            raise AttributeError("Cannot delete '%s'" % name)

        super(RecipeContainer, self).__delattr__(name)
        return

    def get(self, name, default=None):
        """Get a managed object."""
        for d in self.__managed:
            arg = d.get(name)
            if arg is not None:
                return arg

        return default

    def getNames(self):
        """Get the names of managed parameters."""
        return [p.name for p in self._parameters.values()]

    def getValues(self):
        """Get the values of managed parameters."""
        return [p.value for p in self._parameters.values()]

    def _addObject(self, obj, d, check=True):
        """Add an object to a managed dictionary.

        obj     --  The object to be stored.
        d       --  The managed dictionary to store the object in.
        check   --  If True (default), a ValueError is raised an object of the
                    given name already exists.

        Raises ValueError if the object has no name.
        Raises ValueError if the object has the same name as some other managed
        object.
        """

        # Check name
        if not obj.name:
            message = "%s has no name" % obj.__class__.__name__
            raise ValueError(message)

        # Check for extant object in d with same name
        oldobj = d.get(obj.name)
        if check and oldobj is not None:
            message = "%s with name '%s' already exists"%\
                    (obj.__class__.__name__, obj.name)
            raise ValueError(message)

        # Check for object with same name in other dictionary.
        if oldobj is None and self.get(obj.name) is not None:
            message = "Non-%s with name '%s' already exists"%\
                    (obj.__class__.__name__, obj.name)
            raise ValueError(message)

        # Detach the old object, if there is one
        if oldobj is not None:
            oldobj.removeObserver(self._flush)

        # Add the object
        d[obj.name] = obj

        # Observe the object
        obj.addObserver(self._flush)

        # Store this as a configurable object
        self._storeConfigurable(obj)
        return

    def _removeObject(self, obj, d):
        """Remove an object from a managed dictionary.

        Raises ValueError if obj is not part of the dictionary.
        """
        if obj not in d.values():
            m = "'%s' is not part of the %s" % (obj, self.__class__.__name__)
            raise ValueError(m)

        del d[obj.name]
        obj.removeObserver(self._flush)

        return

    def _locateManagedObject(self, obj):
        """Find the location a managed object within the hierarchy.

        obj     --  The object to find.

        Returns a list of objects. The first member of the list is this object,
        and each subsequent member is a sub-object of the previous one.  The
        last entry in the list is obj. If obj cannot be found, the list is
        empty.
        """
        loc = [self]

        # This handles the case that an object is asked to locate itself.
        if obj is self:
            return loc

        for m in self._iterManaged():

            # Check locally for the object
            if m is obj:
                loc.append(obj)
                return loc

            # Check within managed objects
            if hasattr(m, "_locateManagedObject"):

                subloc = m._locateManagedObject(obj)
                if subloc:
                    return loc + subloc

        return []

    def _flush(self, other):
        """Invalidate cached state.

        This will force any observer to invalidate its state. By default this
        does nothing.
        """
        self.notify(other)
        return

    def _validate(self):
        """Validate my state.

        This validates that contained Parameters and managed objects are valid.

        Raises AttributeError if validation fails.
        """
        iterable = chain(self.__iter__(), self._iterManaged())
        self._validateOthers(iterable)
        return
Example #35
0
class OCRRecord(object):
    """
    A composite object of containing recognition results for a single scanned
    page.

    A page is divided into lines which may contain segments and graphemes. For
    practical purposes this means that the appropriate line or segment (the
    later overriding the first) has to be brought into scope first before
    adding any characters to it.

    Each element may be associated with a responsibility statement, identifying
    the origin of each alteration if the final serialization supports it.
    """

    # automatically generated properties on the class
    fields = [
        'title', 'author', 'editor', 'funder', 'principal', 'sponsor',
        'meeting', 'edition', 'publisher', 'distributor', 'authority', 'idno',
        'pub_place', 'licence', 'series_title', 'note', 'source_desc', 'img',
        'dimensions'
    ]

    xml_ns = '{http://www.w3.org/XML/1998/namespace}'
    tei_ns = '{http://www.tei-c.org/ns/1.0}'
    abbyy_ns = '{http://www.abbyy.com/FineReader_xml/FineReader10-schema-v1.xml}'

    # automatically generated properties in the fileDesc element and xpath to their location
    _tei_fields = [
        ('titleStmt', [(
            'title',
            '/' + tei_ns + 'title',
        ), ('author', '/' + tei_ns + 'author', 'ref'),
                       ('editor', '/' + tei_ns + 'editor', 'ref'),
                       ('funder', '/' + tei_ns + 'funder', 'ref'),
                       ('principal', '/' + tei_ns + 'principal', 'ref'),
                       ('sponsor', '/' + tei_ns + 'sponsor', 'ref'),
                       ('meeting', '/' + tei_ns + 'meeting')]),
        ('editionStmt', [('edition', '/' + tei_ns + 'edition')]),
        ('publicationStmt', [
            ('publisher', '/' + tei_ns + 'publisher', 'target'),
            ('distributor', '/' + tei_ns + 'distributor', 'target'),
            ('authority', '/' + tei_ns + 'authority', 'target'),
            ('idno', '/' + tei_ns + 'idno', 'type'),
            ('pub_place', '/' + tei_ns + 'pubPlace'),
            ('licence', '/{0}availability/{0}licence'.format(tei_ns), 'target')
        ]), ('seriesStmt', [('series_title', '/' + tei_ns + 'p')]),
        ('notesStmt', [('note', '/' + tei_ns + 'note')]),
        ('sourceDesc', [('source_desc', '/' + tei_ns + 'p')])
    ]

    def __init__(self):
        self.meta = {}
        self.img = None
        self.respstmt = OrderedDict()
        self.resp_scope = None
        self.line_scope = None
        self.segment_scope = None

        self.lines = OrderedDict()

    # generic setter/getter for metadata
    def _generic_getter(self, field):
        if field in self.meta:
            return self.meta[field]
        else:
            return None

    def _generic_setter(self, value, field):
        self.meta[field] = value

    # responsibility statement functionality
    def add_respstmt(self, resp, name, **kwargs):
        """
        Adds a responsibility statement and returns its identifier.

        The new responsibility statement is automatically scoped.
        Args:
            resp (unicode): Nature of the responsible process.
            name (unicode): Text describing the processing software.
            kwargs (dict): Additional data used be the final serialization.

        Returns:
            A string containing the respstmt ID.
        """
        kwargs['resp'] = resp
        kwargs['name'] = name
        id = u'resp_' + unicode(len(self.respstmt) + 1)
        self.respstmt[id] = kwargs
        self.resp_scope = id
        return id

    def scope_respstmt(self, id):
        """
        Scopes a responsibility statement.

        Args:
            id (unicode): String of targeted resposibility statement.

        Raises:
            NidabaRecordException if the responsibility statement couldn't be
            found.
        """
        if id not in self.respstmt:
            raise NidabaRecordException('No such responsibility statement')
        self.resp_scope = id

    def reset_respstmt_scope(self):
        """
        Clears the current responsibility scope.
        """
        self.resp_scope = None

    #  writer functions for topographical data
    def add_line(self, dim, **kwargs):
        """
        Marks the start of a new topographical line and scopes it.

        Args:
            dim (tuple): A tuple (x0, y0, x1, y1) denoting the bounding box.
            kwargs (dict): Additional data used by the final serialization.

        Returns:
            A string containing the line's identifier.
        """
        id = u'line_' + unicode(len(self.lines) + 1)
        kwargs['bbox'] = dim
        kwargs['content'] = OrderedDict()
        if self.resp_scope:
            kwargs['resp'] = self.resp_scope
        self.lines[id] = kwargs
        self.line_scope = id
        return id

    def add_segment(self, dim, language=None, confidence=None, **kwargs):
        """
        Marks the beginning of a new topographical segment in the current
        scope. Most often this correspond to a word recognized by an engine.

        Args:
            dim (tuple): A tuple containing the bounding box (x0, y0, x1, y1)
            lang (unicode): Optional identifier of the segment language.
            confidence (int): Optional confidence value between 0 and 100.
            kwargs (dict): Additional data used by the final serialization.

        Returns:
            A string containing the segment's indeitifier.

        Raises:
            NidabaRecordException if no line is scoped.
        """
        if not self.line_scope:
            raise NidabaRecordException('No line scoped.')
        id = u'seg_' + unicode(len(self.segments) + 1)
        kwargs['type'] = 'segment'
        kwargs['bbox'] = dim
        if language:
            kwargs['language'] = language
        if confidence:
            if confidence < 0 or confidence > 100:
                raise NidabaRecordException(
                    'Segmentconfidence {} outside valid '
                    'range'.format(confidence))
            kwargs['confidence'] = confidence
        if self.resp_scope:
            kwargs['resp'] = self.resp_scope

        kwargs['content'] = OrderedDict()
        self.lines[self.line_scope]['content'][id] = kwargs
        self.segment_scope = id
        return id

    # actual recognition result writer
    def add_graphemes(self, it):
        """
        Adds a number of graphemes to the current scope (either line or segment).

        A line and/or segment has to be created beforehand.

        Args:
            it (iterable): An iterable returning a dictionary which at least
                           contains a key 'grapheme' with the recognition
                           result. A bounding box has to be placed under the
                           key 'bbox'; a confidence value in the range 0-100
                           (int) is expected under 'confidence'. Additional
                           data (style etc.) will be retained for serializer
                           use.
        """
        if self.line_scope is None:
            raise NidabaRecordException('No element scoped.')
        if self.segment_scope is not None:
            target = self.lines[self.line_scope]['content'][
                self.segment_scope]['content']
        else:
            target = self.lines[self.line_scope]['content']
        gr_cnt = len(self.graphemes)
        ids = []
        for glyph in it:
            gr_cnt += 1
            id = u'grapheme_' + unicode(gr_cnt)
            ids.append(id)
            glyph['type'] = 'grapheme'
            if 'confidence' in glyph and (glyph['confidence'] < 0
                                          or glyph['confidence'] > 100):
                raise NidabaRecordException(
                    'Glyph confidence {} outside valid '
                    'range'.format(glyph['confidence']))
            if 'grapheme' not in glyph:
                raise NidabaRecordException(
                    'Mandatory field missing when adding graphemes.')
            if self.resp_scope:
                glyph['resp'] = self.resp_scope
            target[id] = glyph
        return ids

    def add_choices(self, id, it):
        """
        Adds alternative interpretations to an element.

        Args:
            id (unicode): ID of the element.
            it (iterable): An iterable returning a dictionary containing an
            alternative reading ('alternative') and an optional confidence
            value ('confidence') in the range between 0 and 100.

        Raises:
            NidabaRecordException if no element with the ID could be found.
        """
        if id in self.lines:
            target = self.lines[id]
        for line in self.lines.itervalues():
            if id in line['content']:
                target = line['content'][id]
                break
            for seg in line['content'].itervalues():
                if 'content' in seg and id in seg['content']:
                    target = seg['content'][id]
                    break
        alt = {'content': list(it)}
        if self.resp_scope:
            alt['resp'] = self.resp_scope
        target['alternatives'] = alt

    # scoping of topographical elements
    def scope_line(self, id):
        """
        Scopes a line.

        Args:
            id (unicode): ID of the line to scope.

        Raises:
            NidabaRecordException if no line with the ID could be found.
        """
        if id not in self.lines:
            raise NidabaRecordException('Invalid line ID.')
        self.line_scope = id

    def scope_segment(self, id):
        """
        Scopes a segment (and by association its line).

        Args:
            id (unicode): ID of the segment to scope.

        Raises
        """
        for line_id, line in self.lines.iteritems():
            if id in line['content']:
                self.line_scope = line_id
                self.segment_scope = id
                return
        raise NidabaRecordException('Invalid segment ID.')

    def reset_line_scope(self):
        """
        Resets line scope.
        """
        self.line_scope = None
        self.segment_scope = None

    def reset_segment_scope(self):
        """
        Resets segment scope.
        """
        self.segment_scope = None

    # clearing topographical data
    def clear_lines(self):
        """
        Deletes all lines and their content from the record.
        """
        self.reset_line_scope()
        self.lines = OrderedDict()

    def clear_segments(self):
        """
        Deletes all segments and their content from the record.
        """
        self.reset_segment_scope()
        self.segment_scope = None
        for line in self.lines.itervalues():
            line['content'] = OrderedDict()

    def clear_graphemes(self):
        """
        Deletes all graphemes from the record.
        """
        for line in self.lines.itervalues():
            for seg in line['content'].itervalues():
                if seg['type'] == 'grapheme':
                    line['content'] = OrderedDict()
                    break
                else:
                    seg['content'] = OrderedDict()

    # properties offering short cuts (line are already top-level records)
    @property
    def segments(self):
        """
        Returns an OrderedDict of segments with each segment being a dictionary.
        """
        seg = OrderedDict()
        for line in self.lines.itervalues():
            for seg_id, el in line['content'].iteritems():
                if el['type'] == 'segment':
                    seg[seg_id] = el
        return seg

    @property
    def graphemes(self):
        """
        Returns a list of graphemes with each grapheme being a dictionary.
        """
        g = OrderedDict()
        for line_id, line in self.lines.iteritems():
            for seg_id, el in line['content'].iteritems():
                if el['type'] == 'segment':
                    for g_id, gr in el['content'].iteritems():
                        g[g_id] = gr
                elif el['type'] == 'grapheme':
                    g[seg_id] = el
        return g

    # de-/serializers
    def load_tei(self, fp):
        """
        Reads in a TEI facsimile and populates the record.

        Args:
            fp (File): Source file descriptor.
        """

        doc = etree.parse(fp)

        self.respstmt = OrderedDict()
        self.resp_scope = None

        for stmt, fields in self._tei_fields:
            stmt_el = doc.find('{0}teiHeader/{0}fileDesc/{0}{1}'.format(
                self.tei_ns, stmt))
            if stmt_el is None:
                continue
            for field in fields:
                f_el = stmt_el.find('./' + field[1])
                if f_el is not None:
                    if len(field) == 3 and f_el.get(field[2]):
                        self.meta[field[0]] = [f_el.text, f_el.get(field[2])]
                    else:
                        self.meta[field[0]] = f_el.text

        for resp in doc.iter(self.tei_ns + 'respStmt'):
            id = resp.get(self.xml_ns + 'id')
            r = resp.find('.//{}resp'.format(self.tei_ns)).text
            n = resp.find('.//{}name'.format(self.tei_ns)).text
            self.respstmt[id] = {'resp': r, 'name': n}

        surface = doc.find('{0}sourceDoc/{0}surface'.format(self.tei_ns))
        if surface.get('lrx') is not None and surface.get('lry') is not None:
            self.dimensions = (int(surface.get('lrx')),
                               int(surface.get('lry')))
        graphic = surface.find('{}graphic'.format(self.tei_ns))
        if graphic is not None:
            self.img = graphic.get('url')

        root_zone = doc.find('{0}sourceDoc/{0}surface/{0}zone'.format(
            self.tei_ns))

        corr_flag = False
        alts = []
        sic = None
        last_el = None

        def _get_dict_from_key(id):
            if id in self.lines:
                return self.lines[id]
            for line in self.lines.itervalues():
                if id in line['content']:
                    return line['content'][id]
                for seg in line['content'].itervalues():
                    if 'content' in seg and id in seg['content']:
                        return seg['content'][id]

        for el in islice(root_zone.iter(), 1, None):
            if el.tag != self.tei_ns + 'corr' and corr_flag:
                corr_flag = False
                # flush alternatives
                self.add_choices(sic, alts)
                alts = []
            elif el.tag == self.tei_ns + 'sic':
                sic = None
            elif el.tag == self.tei_ns + 'corr':
                corr_flag = True
                alts.append({'alternative': ''.join(el.text)})
                last_el = alts[-1]
            elif el.tag == self.tei_ns + 'line':
                if el.get('resp') is not None:
                    self.scope_respstmt(el.get('resp')[1:])
                id = self.add_line((int(el.get('ulx')), int(el.get('uly')),
                                    int(el.get('lrx')), int(el.get('lry'))))
                last_el = _get_dict_from_key(id)
                sic = id if not sic else None
            elif el.tag == self.tei_ns + 'zone' and el.get(
                    'type') == 'segment':
                if el.get('resp') is not None:
                    self.scope_respstmt(el.get('resp')[1:])
                id = self.add_segment((int(el.get('ulx')), int(el.get('uly')),
                                       int(el.get('lrx')), int(el.get('lry'))))
                last_el = _get_dict_from_key(id)
                sic = id if not sic else None
            elif el.tag == self.tei_ns + 'zone' and el.get(
                    'type') == 'grapheme':
                gr = {
                    'bbox': (int(el.get('ulx')), int(el.get('uly')),
                             int(el.get('lrx')), int(el.get('lry'))),
                    'grapheme':
                    el.findtext('./{0}seg/{0}g'.format(self.tei_ns))
                }
                id = self.add_graphemes([gr])[0]
                last_el = _get_dict_from_key(id)
                sic = id if not sic else None
            elif el.tag == self.tei_ns + 'certainty':
                last_el['confidence'] = float(el.get('degree')) * 100
            elif el.tag in [
                    self.tei_ns + 'seg', self.tei_ns + 'g',
                    self.tei_ns + 'choice'
            ]:
                pass
            else:
                raise NidabaRecordException(
                    'Unknown tag {} encountered'.format(el.tag))

    def write_tei(self, fp):
        """
        Serializes the record to a TEI facsimile.

        Args:
            fp (File): Target file descriptor.
        """
        doc = Element('TEI',
                      nsmap={None: 'http://www.tei-c.org/ns/1.0'},
                      version='5.0')
        header = SubElement(doc, self.tei_ns + 'teiHeader')
        fileDesc = SubElement(header, self.tei_ns + 'fileDesc')

        sourceDoc = SubElement(doc, self.tei_ns + 'sourceDoc')
        kwargs = {}
        if self.dimensions:
            kwargs = {
                'ulx': '0',
                'uly': '0',
                'lrx': str(self.dimensions[0]),
                'lry': str(self.dimensions[1])
            }
        surface = SubElement(sourceDoc, self.tei_ns + 'surface', **kwargs)
        if self.img:
            SubElement(surface, self.tei_ns + 'graphic', url=self.img)

        surface_zone = SubElement(surface, self.tei_ns + 'zone')

        for stmt, fields in self._tei_fields:
            # create *Stmt in correct order
            parent = Element(self.tei_ns + stmt)
            for field in fields:
                if field[0] in self.meta:
                    el = parent
                    for node in field[1].split('/{')[1:]:
                        el = SubElement(el, '{' + node)
                    value = self.meta[field[0]]
                    if isinstance(value, list):
                        el.set(field[2], value[1])
                        value = value[0]
                    el.text = value
            # insert *Stmt only when needed
            if list(parent):
                fileDesc.append(parent)

        titleStmt = doc.find('{0}teiHeader/{0}fileDesc/{0}titleStmt'.format(
            self.tei_ns))

        if titleStmt is None:
            titleStmt = Element(self.tei_ns + 'titleStmt')
            fileDesc.insert(0, titleStmt)

        for id, resp in self.respstmt.iteritems():
            r = SubElement(titleStmt, self.tei_ns + 'respStmt')
            r.set(self.xml_ns + 'id', id)
            SubElement(r, self.tei_ns + 'resp').text = resp['resp']
            SubElement(r, self.tei_ns + 'name').text = resp['name']

        def _set_confidence(el, up, dic):
            cert = None
            if 'confidence' in dic:
                cert = SubElement(el,
                                  self.tei_ns + 'certainty',
                                  degree=u'{0:.2f}'.format(dic['confidence'] /
                                                           100.0),
                                  locus='value')
                if el.get(self.xml_ns + 'id'):
                    cert.set('target', '#' + el.get(self.xml_ns + 'id'))
            if 'resp' in up:
                el.set('resp', '#' + up['resp'])
                if cert is not None:
                    cert.set('resp', '#' + up['resp'])

        def _wrap_choices(alternatives, sic, parent):
            choice = SubElement(parent, self.tei_ns + 'choice')
            sic_el = SubElement(choice, self.tei_ns + 'sic')
            sic_el.append(sic)
            for alt in alternatives['content']:
                corr = SubElement(choice, self.tei_ns + 'corr')
                corr.text = alt['alternative']
                _set_confidence(corr, alternatives, alt)

        def _add_grapheme(grapheme_id, grapheme, parent):
            g_el = Element(self.tei_ns + 'zone', type='grapheme')
            g_el.set(self.xml_ns + 'id', grapheme_id)
            if 'bbox' in grapheme:
                g_el.set('ulx', str(grapheme['bbox'][0]))
                g_el.set('uly', str(grapheme['bbox'][1]))
                g_el.set('lrx', str(grapheme['bbox'][2]))
                g_el.set('lry', str(grapheme['bbox'][3]))

            if 'alternatives' in grapheme:
                _wrap_choices(grapheme['alternatives'], g_el, parent)
            else:
                parent.append(g_el)
            glyph = SubElement(SubElement(g_el, self.tei_ns + 'seg'),
                               self.tei_ns + 'g')
            glyph.text = grapheme['grapheme']
            _set_confidence(g_el, grapheme, grapheme)

        for line_id, line in self.lines.iteritems():
            line_el = Element(self.tei_ns + 'line',
                              ulx=str(line['bbox'][0]),
                              uly=str(line['bbox'][1]),
                              lrx=str(line['bbox'][2]),
                              lry=str(line['bbox'][3]))
            line_el.set(self.xml_ns + 'id', line_id)
            _set_confidence(line_el, line, line)
            if 'alternatives' in line:
                _wrap_choices(line['alternatives'], line_el, surface_zone)
            else:
                surface_zone.append(line_el)
            for seg_id, seg in line['content'].iteritems():
                if seg['type'] == 'segment':
                    seg_el = Element(self.tei_ns + 'zone',
                                     ulx=str(seg['bbox'][0]),
                                     uly=str(seg['bbox'][1]),
                                     lrx=str(seg['bbox'][2]),
                                     lry=str(seg['bbox'][3]),
                                     type=seg['type'])
                    seg_el.set(self.xml_ns + 'id', seg_id)
                    _set_confidence(seg_el, seg, seg)
                    for grapheme_id, grapheme in seg['content'].iteritems():
                        _add_grapheme(grapheme_id, grapheme, seg_el)
                    if 'alternatives' in seg:
                        _wrap_choices(seg['alternatives'], seg_el, line_el)
                    else:
                        line_el.append(seg_el)
                elif seg['type'] == 'grapheme':
                    _add_grapheme(seg_id, seg, line_el)
                else:
                    raise NidabaRecordException(
                        'Unknown nodes beneath line records')
        fp.write(
            etree.tostring(doc,
                           xml_declaration=True,
                           encoding='utf-8',
                           pretty_print=True))
        fp.flush()

    def write_abbyyxml(self, fp):
        """
        Writes the TEI document in a format reminiscent of Abbyy FineReader's
        XML output. Its basic format is:

        <document>
        <page>
        <text>
        <line l="0" r="111" t="6" b="89">
        <charParams l="0" r="78" t="6" b="89" charConfidence="76" wordStart="true">D</charParams>
        <charParams l="86" r="111" t="24" b="89" charConfidence="76" wordStart="false">e</charParams>
        </line>
        ....
        </text>
        </page>
        </document>

        Please note that alternative readings as produced for example by spell
        checking are dropped from the output. Responsibility statements,
        metadata, and source image information is likewise lost.

        Args:
            fp (file): File descriptor to write to.
        """
        page = Element(
            'document',
            xmlns=
            'http://www.abbyy.com/FineReader_xml/FineReader10-schema-v1.xml',
            version='1.0',
            producer='nidaba')
        p = SubElement(page, 'page')
        p.set('width', str(self.dimensions[0]))
        p.set('height', str(self.dimensions[1]))
        p.set('resolution', '0')
        p.set('originalCoords', '1')
        b = SubElement(p, 'block', blockType='Text')
        text = SubElement(b, 'text')
        par = SubElement(text, 'par')
        for line in self.lines.itervalues():
            lel = SubElement(par, 'line')
            # XXX: meaning of baseline is nowere documented
            lel.set('baseline', '0')
            lel.set('l', str(line['bbox'][0]))
            lel.set('t', str(line['bbox'][1]))
            lel.set('r', str(line['bbox'][2]))
            lel.set('b', str(line['bbox'][3]))
            for seg in line['content'].itervalues():
                if seg['type'] == 'segment':
                    formatting = SubElement(lel, 'formatting')
                    if 'language' in seg:
                        formatting.set('lang', seg['language'])
                    word_start = True
                    for g in seg['content'].itervalues():
                        if 'bbox' not in g:
                            raise NidabaRecordException(
                                'No bounding box for grapheme')
                        el = SubElement(formatting, 'charParams')
                        if word_start:
                            el.set('wordStart', 'true')
                            word_start = False
                        else:
                            el.set('wordStart', 'false')
                        el.text = g['grapheme']
                        el.set('l', str(g['bbox'][0]))
                        el.set('t', str(g['bbox'][1]))
                        el.set('r', str(g['bbox'][2]))
                        el.set('b', str(g['bbox'][3]))
                        if 'confidence' in g:
                            el.set('charConfidence', str(g['confidence']))
                elif seg['type'] == 'grapheme':
                    formatting = SubElement(lel, 'formatting')
                    if 'language' in seg:
                        formatting.set('lang', seg['language'])
                    el = SubElement(formatting, 'charParams')
                    el.text = g['grapheme']
                    el.set('l', str(g['bbox'][0]))
                    el.set('t', str(g['bbox'][1]))
                    el.set('r', str(g['bbox'][2]))
                    el.set('b', str(g['bbox'][3]))
                    if 'confidence' in g:
                        el.set('charConfidence', str(g['confidence']))
                else:
                    raise NidabaRecordException(
                        'Unknown nodes beneath line records')
        fp.write(etree.tostring(page, xml_declaration=True, encoding='utf-8'))
        fp.flush()

    def write_alto(self, fp):
        """
        Serializes the record as an ALTO XML document.

        See [0] for further information and schemata. Output will conform to
        version 3.1.

        Please note that the output will not be split into a series of
        "paragraphs" as segmentation algorithms don't produce them and they are
        dependent on typographic convention. Scores for alternatives are
        dropped as the standard does not provide for a way to encode them.
        Character confidences are rounded to the next lower confidence value
        (.98 -> 0, 0.05 -> 9).

        Alternatives are only serialized for segments. Line and grapheme
        alternatives are discarded.

        Args:
            fp (file): File descriptor to write to.

        [0] http://www.loc.gov/standards/alto/
        """
        alto = etree.Element('alto',
                             xmlns="http://www.loc.gov/standards/alto/ns-v3#")
        description = SubElement(alto, 'Description')
        SubElement(description, 'MeasurementUnit').text = 'pixel'

        # use the image url as source image file name
        source_img = SubElement(description, 'sourceImageInformation')
        if self.img is not None:
            SubElement(source_img, 'fileName').text = self.img
        # convert responsibility statements to ocrProcessingSteps. As TEI
        # offers no way to distinguish between pre-, OCR, and postprocessing
        # for responsibility statements everything before a respStmt containing
        # 'recognition' is converted into preprocessing and everything after it
        # to postprocessing.
        ocr_proc = SubElement(description, 'OCRProcessing')
        ocr_proc.set('ID', 'OCR_0')
        mode = 'preProcessingStep'
        for id, respstmt in self.respstmt.iteritems():
            if 'recognition' in respstmt[
                    'resp'] and mode == 'preProcessingStep':
                mode = 'ocrProcessingStep'
            proc = SubElement(ocr_proc, mode)
            SubElement(proc,
                       'processingStepDescription').text = respstmt['resp']
            ps = SubElement(proc, 'processingSoftware')
            SubElement(ps, 'softwareName').text = respstmt['name']
            if mode == 'ocrProcessingStep':
                mode = 'postProcessingStep'

        layout = SubElement(alto, 'Layout')
        page = SubElement(layout, 'Page')
        if self.dimensions is not None:
            page.set('WIDTH', str(self.dimensions[0]))
            page.set('HEIGHT', str(self.dimensions[1]))
        page.set('PHYSICAL_IMG_NR', '0')
        page.set('ID', 'page_0')

        # why do all OCR formats insist on creating 'paragraph' containers? As
        # paragraphs are highly variable and dependent on typographic
        # conventions all text on the page is wrapped into a single paragraph.
        print_space = SubElement(page, 'PrintSpace')
        print_space.set('HPOS', str(0.0))
        print_space.set('VPOS', str(0.0))
        if self.dimensions is not None:
            print_space.set('WIDTH', str(self.dimensions[0]))
            print_space.set('HEIGHT', str(self.dimensions[1]))
        text_block = SubElement(print_space, 'TextBlock')
        text_block.set('HPOS', str(0.0))
        text_block.set('VPOS', str(0.0))
        text_block.set('ID', 'textblock_0')
        if self.dimensions is not None:
            text_block.set('WIDTH', str(self.dimensions[0]))
            text_block.set('HEIGHT', str(self.dimensions[1]))

        for line_id, line in self.lines.iteritems():
            text_line = SubElement(text_block, 'TextLine')
            text_line.set('HPOS', str(line['bbox'][0]))
            text_line.set('VPOS', str(line['bbox'][1]))
            text_line.set('WIDTH', str(line['bbox'][2] - line['bbox'][0]))
            text_line.set('HEIGHT', str(line['bbox'][3] - line['bbox'][1]))
            text_line.set('ID', line_id)

            # There are 3 cases of content beneath a line: a list of graphemes
            # (which are outright dumped into a SINGLE String element), some
            # segments (each is converted to a String or SP node depending on
            # content), or some corr node which are converted to String nodes
            # containing ALTERNATIVE nodes.
            for seg_id, seg in line['content'].iteritems():
                if seg['type'] == 'grapheme':
                    text = u''
                    certs = []
                    for g in line['content'].itervalues():
                        text += g['grapheme']
                        # confidences for graphemes are integers between 0 and
                        # 9 with 0 (wtf?) representing highest confidence
                        if 'confidence' in g:
                            certs.append(
                                str(int(10 - 10 * (g['confidence'] / 100.0))))
                    text_el = SubElement(text_line, 'String')
                    text_el.set('CONTENT', text)
                    text_el.set('CC', ' '.join(certs))
                    break
                elif seg['type'] == 'segment':
                    text = ''.join(x['grapheme']
                                   for x in seg['content'].itervalues())
                    if text.isspace():
                        text_el = SubElement(text_line, 'SP')
                    else:
                        text_el = SubElement(text_line, 'String')
                        text_el.set('CONTENT', text)
                        # extract word confidences
                        if 'confidence' in seg:
                            text_el.set('WC', str(seg['confidence'] / 100.0))
                        certs = []
                        # extract character confidences
                        for g in seg['content'].itervalues():
                            if 'confidence' in g:
                                certs.append(
                                    str(
                                        int(10 - 10 *
                                            (g['confidence'] / 100.0))))
                        if certs:
                            text_el.set('CC', ' '.join(certs))

                    text_el.set('HPOS', str(seg['bbox'][0]))
                    text_el.set('VPOS', str(seg['bbox'][1]))
                    text_el.set('WIDTH', str(seg['bbox'][2] - seg['bbox'][0]))
                    text_el.set('HEIGHT', str(seg['bbox'][1] - seg['bbox'][3]))
                    text_el.set('ID', seg_id)
                    if 'alternatives' in seg:
                        for corr in seg['alternatives']['content']:
                            SubElement(
                                text_el,
                                'ALTERNATIVE').text = corr['alternative']
            # add an empty String element if no children exist
            if not list(text_line):
                text_el = SubElement(text_line, 'String')
                text_el.set('CONTENT', '')
        fp.write(
            etree.tostring(alto,
                           pretty_print=True,
                           xml_declaration=True,
                           encoding='utf-8'))
        fp.flush()

    def load_hocr(self, fp):
        """
        Reads an hOCR file and populates the record.

        Args:
            fp (file): File descriptor to read from.
        """
        doc = etree.HTML(fp.read())
        el = doc.find(".//meta[@name='ocr-system']")
        if el is not None:
            self.add_respstmt(el.get('content'), 'ocr-system')
        page = doc.find('body/div[@class="ocr_page"]')
        o = _parse_hocr(page.get('title'))
        if 'bbox' in o:
            self.dimensions = o['bbox'][2:]
        if 'image' in o:
            self.img = o['image'][0]

        corr_flag = False
        sic = None
        alts = []
        cuts = []
        for el in page.iter('span', 'ins', 'del'):
            el_class = el.get('class')
            if el_class != 'alt' and corr_flag:
                corr_flag = False
                # flush alternatives
                self.add_choices(sic, alts)
                alts = []
            if el.tag == 'ins':
                sic = None
            elif el.tag == 'del':
                corr_flag = True
                o = _parse_hocr(el.get('title'))
                alts.append({
                    'confidence': 100 - o['x_cost'][0],
                    'alternative': ''.join(el.itertext())
                })
            elif el_class == 'ocr_line':
                o = _parse_hocr(el.get('title'))
                id = self.add_line(o['bbox'])
                if el.xpath('.//span[starts-with(@class, "ocrx")]') is None:
                    sym = ''.join(el.itertext())
                    self.add_graphemes([{'grapheme': x} for x in sym])
                sic = id if not sic else None
            elif 'ocrx' in el_class:
                o = _parse_hocr(el.get('title'))
                id = self.add_segment(o['bbox'],
                                      o['x_conf'] if 'x_conf' in o else None)
                sic = id if not sic else None
                sym = ''.join(el.itertext())
                self.add_graphemes([{'grapheme': x} for x in sym])

    def write_hocr(self, fp):
        """
        Serializes the OCR record in hOCR format.

        Metadata except image source and dimensions are lost, as are
        responsibility statements. Alternatives EXCEPT grapheme alternatives
        are inserted using the INS-DEL syntax described in section 10 of the
        hOCR standard [0]. Grapheme coordinates and confidences are added as
        cuts/x_confs to the ocr_line element.

        [0] https://docs.google.com/document/d/1QQnIQtvdAC_8n92-LhwPcjtAUFwBlzE8EWnKAxlgVf0/preview

        Args:
            fp (file): File descriptor to write to.
        """
        page = etree.Element('html', xmlns="http://www.w3.org/1999/xhtml")
        head = SubElement(page, 'head')
        if 'title' in self.meta:
            SubElement(head, 'title').text = self.meta['title']
        if self.respstmt:
            SubElement(head,
                       'meta',
                       name="ocr-system",
                       content=self.respstmt.values()[-1]['name'])
        capa = "ocr_page"
        if self.lines:
            capa += ", ocr_line"
        if self.segments:
            capa += ", ocrx_word"
        SubElement(head, 'meta', name='ocr-capabilities', content=capa)

        body = SubElement(page, 'body')
        p_hocr = _micro_hocr()
        if 'dimensions' in self.meta:
            p_hocr.add('bbox', 0, 0, *self.meta['dimensions'])
        if self.img is not None:
            p_hocr.add('image', self.img)
        ocr_page = SubElement(body, 'div', title=str(p_hocr))
        ocr_page.set('class', 'ocr_page')

        def _wrap_alternatives(alternatives, ins, parent):
            span = SubElement(parent, 'span')
            span.set('class', 'alternatives')
            ins_el = SubElement(span, 'ins')
            ins_el.set('class', 'alt')
            ins_el.append(ins)
            for alt in alternatives['content']:
                corr = SubElement(span, 'del')
                corr.set('class', 'alt')
                corr.text = alt['alternative']
                if 'confidence' in alt:
                    corr.set('title',
                             'x_cost {}'.format(100 - alt['confidence']))

        for line_id, line in self.lines.iteritems():
            ocr_line = Element('span', id=line_id)
            ocr_line.set('class', 'ocr_line')
            ocr_line.text = u''
            l_hocr = _micro_hocr()
            l_hocr.add('bbox', line['bbox'])
            gr_boxes = []
            gr_confidences = []
            if 'alternatives' in line:
                _wrap_alternatives(line['alternatives'], ocr_line, ocr_page)
            else:
                ocr_page.append(ocr_line)
            SubElement(ocr_page, 'br')

            for seg_id, seg in line['content'].iteritems():
                if seg['type'] == 'grapheme':
                    if 'bbox' in seg:
                        gr_boxes.append(seg['bbox'])
                    if 'confidence' in seg:
                        gr_confidences.append(seg['confidence'])
                    ocr_line.text += seg['grapheme']
                elif seg['type'] == 'segment':
                    ocrx_word = Element('span', id=seg_id)
                    ocrx_word.set('class', 'ocrx_word')
                    s_hocr = _micro_hocr()
                    if 'bbox' in seg:
                        s_hocr.add('bbox', seg['bbox'])
                    if 'confidence' in seg:
                        s_hocr.add('x_wconf', seg['confidence'])
                    ocrx_word.set('title', str(s_hocr))
                    ocrx_word.text = u''
                    if 'alternatives' in seg:
                        _wrap_alternatives(seg['alternatives'], ocrx_word,
                                           ocr_line)
                    else:
                        ocr_line.append(ocrx_word)
                    for g in seg['content'].itervalues():
                        if 'bbox' in g:
                            gr_boxes.append(g['bbox'])
                        if 'confidence' in g:
                            gr_confidences.append(g['confidence'])
                        ocrx_word.text += g['grapheme']
                else:
                    raise NidabaRecordException(
                        'Unknown nodes beneath line records')
            if gr_boxes:
                l_hocr.add('cuts', *list(_delta(line['bbox'], gr_boxes)))
            if gr_confidences:
                l_hocr.add('x_confs', *gr_confidences)
            ocr_line.set('title', str(l_hocr))
        fp.write(
            etree.tostring(page,
                           pretty_print=True,
                           xml_declaration=True,
                           encoding='utf-8'))
        fp.flush()

    def write_text(self, fp, header=True):
        """
        Writes the OCR record as plain text.

        Args:
            fp (file): File descriptor to write to.
            header (bool): Serialize metadata before the recognized text
                           between '+++'.
        """
        if self.meta and header is True:
            fp.write('+++\n')
            for key, val in self.meta.iteritems():
                fp.write(u'{} = {}\n'.format(key, val).encode('utf-8'))
            fp.write('+++\n')

        for line in self.lines.itervalues():
            fp.write('\n')
            for seg in line['content'].itervalues():
                if seg['type'] == 'grapheme':
                    fp.write(g['grapheme'].encode('utf-8'))
                else:
                    for g in seg['content'].itervalues():
                        fp.write(g['grapheme'].encode('utf-8'))
        fp.flush()
Example #36
0
class fftView(QtGui.QTabWidget):
    def __init__(self, *args, **kwargs):
        super(fftView, self).__init__(*args, **kwargs)
        self.img_dict = OrderedDict()

    def add_images(self, image_list, do_fft=True, loadingfactors=None):

        self.clear()

        if not image_list:
            return
        if type(image_list) == dict:
            for path, img in image_list.iteritems():
                try:
                    img = np.array(img)
                    len(img)
                    flag = True
                    if do_fft: img = self.do_fft(img)

                    for item in self.img_dict.itervalues():
                        if np.array_equal(img, item): flag = False
                    if flag:
                        self.img_dict[path.split('/')[-1]] = img
                except TypeError:
                    continue
        else:
            for img in image_list:
                try:
                    img = np.array(img)
                    len(img)
                    flag = True
                    if do_fft: img = self.do_fft(img)

                    for item in self.img_dict.itervalues():
                        if np.array_equal(img, item): flag = False
                    if flag:
                        self.img_dict[len(self.img_dict)] = img
                except TypeError:
                    continue

        # sort img dictionary by key
        img_dict = OrderedDict()
        for key in sorted(self.img_dict.keys()):
            img_dict[key] = self.img_dict[key]
        del self.img_dict
        self.img_dict = img_dict

        data = imagetimeline(self.img_dict.itervalues())
        sizemax = max(map(np.shape, data))[0]

        view = TimelineView(sizemax)
        view.setImage(data)

        scale = calcscale(view)  # Sets up the scale
        view.imageItem.resetTransform()
        view.imageItem.scale(scale, scale)
        view.autoRange()
        view.getHistogramWidget().setHidden(False)
        view.ui.roiBtn.setHidden(True)
        view.ui.menuBtn.setHidden(True)
        view.sigImageChanged.connect(self.imageNameChanged)

        if loadingfactors is None:
            self.addTab(view, u"Tile " + str(1))
        else:
            self.addTab(view, str(loadingfactors))
        self.tabBar().hide()

        view.sigImageChanged.emit()

    def do_fft(self, img):
        """
        Returns absolute value of 2-D FFT of image. Assumes 2-D image
        """

        return np.log(np.fft.fftshift(np.abs(np.fft.fft2(img)**2)) + 1e-10)


    def open_from_rmcView(self, image_list):
        images = {}
        for lst in image_list:
            path = "/"
            for item in lst:
                path = os.path.join(path, item)
            img = Image.open(path).convert('L')
            img = np.array(img)
            images[path] = img

        self.add_images(images)

    def imageNameChanged(self):
        view = self.currentWidget()
        for key, val in self.img_dict.iteritems():
            if np.array_equal(val, view.image[view.currentIndex]):
                self.currentWidget().label.setText(key)
Example #37
0
class OperatorsFactory(object):
    __metaclass__ = Singleton

    def __init__(self):
        self._entries = OrderedDict()

    @property
    def name(self):
        return self._name

    def register(self, operator_class):
        if not issubclass(operator_class, BaseOperator):
            raise InvalidClassException(
                "The given class is not a BaseOperator "
                "subclass.")

        name = operator_class.name()
        ver = operator_class.version()
        entry = self._entries.get(name)
        if entry:
            entry[int(ver)] = operator_class
        else:
            self._entries[name] = {int(ver): operator_class}

    def has_operator(self, name, version=None):
        if version is None:
            return name in self._entries
        return name in self._entries and int(version) in self._entries[name]

    def get_all_operator_names(self, app=None):
        if app is None:
            return self._entries.keys()

        res = set()
        for versions in self._entries.itervalues():
            for c in versions.itervalues():
                if c.is_app_supported(app):
                    res.add(c.name())
                break
        return list(res)

    def get_channel_operator_names(self, app=None):
        res = set()
        for versions in self._entries.itervalues():
            for c in versions.itervalues():
                if c.is_channel_operator() and (app is None
                                                or c.is_app_supported(app)):
                    res.add(c.name())
                break
        return list(res)

    def get_item_operator_names(self, app=None):
        res = set()
        for versions in self._entries.itervalues():
            for c in versions.itervalues():
                import os
                if not c.is_channel_operator() and (app is None or
                                                    c.is_app_supported(app)):
                    res.add(c.name())
                break
        return list(res)

    def get_operator(self, name, version):
        if name in self._entries and int(version) in self._entries[name]:
            return self._entries[name][int(version)]

    def create(self, name, version, facade, node, channel=None):
        if not isinstance(facade, BaseFacade):
            raise InvalidFacadeException('You need to provide a valid facade '
                                         'to create an operator')

        if name in self._entries and int(version) in self._entries[name]:
            return self._entries[name][int(version)](facade,
                                                     node,
                                                     channel=channel)

    def get_latest_version(self, name):
        if not name in self._entries:
            return None

        versions = self._entries[name].keys()
        versions.sort()
        return versions[-1]

    def remove_operator(self, name):
        del self._entries[name]

    def clear_operators(self):
        self._entries.clear()

    def reload_operators(self):
        for name, versions in self._entries.iteritems():
            for ver, constructor in versions.iteritems():
                module_name = constructor.__module__
                class_name = constructor.__name__

                del sys.modules[module_name]
                self._entries[name][ver] = __import__(module_name, globals(),
                                                      locals(), [class_name])
Example #38
0
def create_allele_dataframes(imgt_dat, fasta_gen, fasta_nuc, **kwargs):
    if kwargs.get("verbosity", 0):
        print now(), 'Loading IMGT allele dat file...'

    alleles = OrderedDict()

    with open(imgt_dat, 'rU') as handle:
        for i, record in enumerate(SeqIO.parse(handle, "imgt")):
            alleles[record.id] = record

    if kwargs.get("verbosity", 0):
        print now(), 'Initializing allele DataFrame...'
    '''
    id      HLA000001
    type    A*01:01:01:01
    locus   A
    class   I --- I, II, TAP, MIC, other
    flags   0 --- problems with allele stored here, see below
    len_gen  3503
    len_nuc  1098
    full_gen  1
    full_nuc  1

    flagging: sum of the below codes
    +1 if HLA type ends in a letter (N, Q, etc.)
    +2 if CDS annotation != exon annotation
    +4 if features don't add up to gen sequence
    +8 if exons don't add up to nuc sequence
    '''

    allele_info = 'id type 4digit locus flags len_dat len_gen len_nuc full_gen full_nuc'

    table = pd.DataFrame(index=alleles.keys(), columns=allele_info.split())
    sequences = []

    if kwargs.get("verbosity", 0):
        print now(), 'Filling DataFrame with allele data...'

    all_features = [
    ]  # contains tuples: (HLA id, feature type, feature number, feature start, feature end)

    for allele in alleles.itervalues():

        allele_type = allele.description.replace('HLA-', '').split(',')[0]

        table.ix[allele.id]['id'] = allele.id
        table.ix[allele.id]['type'] = allele_type
        table.ix[allele.id]['4digit'] = ':'.join(allele_type.split(':')[:2])
        table.ix[allele.id]['locus'] = allele_type.split('*')[0]
        table.ix[allele.id]['flags'] = 0 if allele_type[-1].isdigit() else 1
        table.ix[allele.id]['len_dat'] = len(str(allele.seq))
        table.ix[allele.id][
            'len_gen'] = 0  # TODO: IT STILL DOESNT SOLVE PERFORMANCEWARNING!
        table.ix[allele.id][
            'len_nuc'] = 0  # we initialize these nulls so that we don't get a
        table.ix[allele.id][
            'full_gen'] = 0  # PerformanceWarning + pickling when storing HDF
        table.ix[allele.id][
            'full_nuc'] = 0  # because of NaNs (they don't map to ctypes)
        sequences.append((allele.id, 'dat', str(allele.seq)))

        # number of features in 2013-04-30 hla.dat:
        # 9296 source (total # of alleles)
        # 9291 CDS, 5 gene (HLA-P pseudogenes)
        # 24493 exons, 3697 introns, 1027 UTRs
        # CDS matches exons in 99+% of cases, some have a single base-pair extension at the end for some
        # weird single-base exons or such (all on unimportant pseudogene loci)
        # so we extract exons, introns and UTRs
        features = [
            f for f in allele.features if f.type in ('exon', 'intron', 'UTR')
        ]

        for feature in features:
            if feature.type in ('exon', 'intron'):
                feature_num = int(feature.qualifiers['number'][0])
            else:  # UTR
                # UTR either starts at the beginning or ends at the end
                assert feature.location.start == 0 or feature.location.end == len(
                    allele.seq)
                feature_num = 1 if feature.location.start == 0 else 2  # 1 if 5' UTR, 2 if 3' UTR

            all_features.append((allele.id, feature.type, feature_num,
                                 int(feature.location.start),
                                 int(feature.location.end), len(feature)))

        # small sanity check, can be commented out
        cds = [f for f in allele.features if f.type == 'CDS']
        if cds:
            if sum(map(len, [f for f in features if f.type == 'exon'])) != len(
                    cds[0]):
                print "\tCDS length doesn't match sum of exons for", allele.id, allele_type
                table.ix[allele.id]['flags'] += 2
        else:
            print "\tNo CDS found for", allele.id, allele_type
            table.ix[allele.id]['flags'] += 2

    if kwargs.get("verbosity", 0):
        print now(), 'Loading gen and nuc files...'

    with open(fasta_gen, 'r') as fasta_gen:
        for record in SeqIO.parse(fasta_gen, 'fasta'):
            allele_id = record.id.replace('HLA:', '')
            table.ix[allele_id]['len_gen'] = len(record.seq)
            sequences.append((allele_id, 'gen', str(record.seq)))

    with open(fasta_nuc, 'r') as fasta_nuc:
        for record in SeqIO.parse(fasta_nuc, 'fasta'):
            allele_id = record.id.replace('HLA:', '')
            table.ix[allele_id]['len_nuc'] = len(record.seq)
            sequences.append((allele_id, 'nuc', str(record.seq)))

    # convert list of tuples into DataFrame for features and sequences
    all_features = pd.DataFrame(
        all_features,
        columns=['id', 'feature', 'number', 'start', 'end', 'length'])
    sequences = pd.DataFrame(sequences, columns=['id', 'source', 'sequence'])

    joined = pd.merge(table, all_features, how='inner', on='id')

    exons_for_locus = {}

    for i_locus, i_group in joined.groupby('locus'):
        exons_for_locus[i_locus] = i_group[i_group['feature'] ==
                                           'exon']['number'].max()

    # for i_id, i_group in joined.groupby('id'):
    #     max_exons = exons_for_locus[table.ix[i_id]['locus']]
    #     if len(i_group) >= 2*max_exons-1:
    #         print i_id, 'is fully annotated on all exons and introns and UTRs'

    if kwargs.get("verbosity", 0):
        print now(), 'Checking dat features vs gen/nuc sequences...'

    for allele, features in joined.groupby('id'):
        row = features.irow(
            0
        )  # first row of the features subtable. Contains all allele information because of the join
        sum_features_length = features['length'].sum()
        sum_exons_length = features.ix[features['feature'] ==
                                       'exon']['length'].sum()
        if row['len_gen'] > 0 and row['len_gen'] != sum_features_length:
            print "\tFeature lengths don't add up to gen sequence length", allele, row[
                'len_gen'], sum_features_length, row['type']
            table.ix[allele]['flags'] += 4
        if row['len_nuc'] > 0 and row['len_nuc'] != sum_exons_length:
            print "\tExon lengths don't add up to nuc sequence length", allele, row[
                'len_nuc'], sum_exons_length, row['type']
            table.ix[allele]['flags'] += 8

    if kwargs.get("verbosity", 0):
        print now(), 'Sanity check finished...'

    return table, all_features, sequences
Example #39
0
            _b.mu * 1000.**3,  # converting units: km^3/s^2 --> m^3/s^2

            # radius
            # 	body radius (SI units, i.e. meters)
            _b.R * 1000.,

            # safe_radius
            # 	body distance safe for a spacecraft fly-by
            ((2. * _b.R) if _b.body == 'jupiter' else (_b.R + 50.)) * 1000.,
            # "The spacecraft range to Jupiter cannot go below 2*R_J at any time"
            # "The flyby altitudes at the satellites (i.e. the range to the satellite centre at closest approach on the flyby minus the satellite radius) cannot be below 50 km"

            # name
            # 	body name
            _b.body,
        )) for _b in body_tuple.itervalues()
])

(jupiter, io, europa, ganymede, callisto) = body_obj.values()

# http://en.wikipedia.org/wiki/Orbital_elements
body_label = {
    #	'Object UID'    : ,

    # a specified point in time
    #	'Epoch (MJD)'   : ,

    # measure of the radius of an orbit taken from the points of that same orbit's two most distant points
    # http://en.wikipedia.org/wiki/Semimajor_axis
    'a (km)': r'$a$: semi major axis (km)',
Example #40
0
class CodeClass(object):
    def __init__(self,
                 generator,
                 name,
                 module=None,
                 base='object',
                 description=None):
        self.generator = generator
        self.name = name
        self.module = module
        self.base = base
        self.description = description
        self.properties = OrderedDict()
        self.methods = OrderedDict()

    @property
    def fullname(self):
        return '%s.%s' % (self.module.fullname, self.name)

    def make_names_unique(self):
        names = []

        def unique_name(name):
            if name not in names:
                names.append(name)
                return name
            else:
                return unique_name(name + '_')

        for m in self.methods.itervalues():
            m.name = unique_name(m.name)

    def __str__(self):
        self.make_names_unique()

        with Writer() as w:
            w.write('@has_validated_properties')
            w.write('@has_interfaces')
            base = self.base if isinstance(
                self.base, str) else (self.base.name if self.base.module
                                      == self.module else self.base.fullname)
            w.write('class %s(%s):' % (self.name, base))
            w.i()
            if self.description:
                w.write_docstring(self.description)
            w.write('def __init__(self, context):')
            w.i()
            w.write('self.context = context')
            w.o()
            for n, p in self.properties.iteritems():
                w.write()
                if p.default is not None:
                    w.write('@property_default(%s)' % safe_repr(p.default))
                if p.type:
                    w.write('@property_type(%s)' %
                            self.generator.get_classname(p.type))
                w.write('@validated_property')
                w.write('def %s():' % n)
                w.i()
                if p.description or p.type is not None:
                    w.write('"""')
                    if p.description:
                        w.write(p.description.strip())
                    if p.type is not None:
                        if p.description:
                            w.write()
                        w.write(':rtype: :class:`%s`' %
                                self.generator.get_classname(p.type))
                    w.write('"""')
                else:
                    w.write('pass')
                w.o()
            for n, m in self.methods.iteritems():
                w.write()
                w.write(m)
            return str(w)
Example #41
0
class TechnicalBase(PlotInterface):
    """
    指标基类。

    :ivar name: 指标对象名称
    :ivar series: 单值指标的序列变量或多值指标字典
    :ivar is_multiple: 是否是多值指标
    """
    def __init__(self, name='', widget=None):
        super(TechnicalBase, self).__init__(name, widget)
        self.name = name
        self.series = None
        self._args = None

    def _rolling_algo(self, data, n, i):
        """ 逐步运行函数。"""
        raise NotImplementedError

    def _vector_algo(self, data, n):
        """向量化运行, 结果必须赋值给self.values。

        Args:
            data (np.ndarray): 数据

            n (int): 时间窗口大小
        """
        raise NotImplementedError

    def compute(self):
        """
         构建时间序列变量,执行指标的向量算法。
        """
        if not hasattr(self, '_args'):
            raise Exception("每个指标都必须有_args属性,代表指标计算的参数!")
        self.data = self._args[0]
        # 数据转化成ta-lib能处理的格式
        self._args[0] = transform2ndarray(self._args[0])
        apply(self._vector_algo, tuple(self._args))
        if not hasattr(self, 'values'):
            raise Exception("每个指标都必须有value属性,代表指标计算结果!")
        if isinstance(self.values, dict):
            self.series = OrderedDict()
            for key, value in self.values.iteritems():
                self.series[key] = series.NumberSeries(value, self.name, self,
                                                       float('nan'))
            for key, value in self.series.iteritems():
                setattr(self, key, value)
            self.is_multiple = True
        else:
            self.series = [
                series.NumberSeries(self.values, self.name, self, float('nan'))
            ]
            self.is_multiple = False
        self._init_bound()

    def compute_element(self, cache_index, rolling_index):
        """ 计算一个回溯值, 被Series延迟调用。

        Args:
            cache_index (int): 缓存索引

            rolling_index (int): 回溯索引
        """
        #rolling_index = min(len(self.data)-1, self.curbar)
        #values = None
        #if self._cache[cache_index][0] == self.curbar:
        #values = self._cache[cache_index][1] # 缓存命中
        #else:
        #self._rolling_data = transform2ndarray(self.data)  # 输入
        ## 指标一次返回多个值
        #args =  (self._rolling_data, ) + self._args + (rolling_index,)
        #values = apply(self._rolling_algo, args)
        #self._cache[cache_index] = (self.curbar, values)

        #for i, v in enumerate(values):
        #if self.is_multiple:
        #self.series.values()[i].update(v)
        #else:
        #self.series[i].update(v)
        pass

    @property
    def curbar(self):
        if self.is_multiple:
            return self.series.itervalues().next().curbar
        return self.series[0].curbar

    def __size__(self):
        """"""
        if self.is_multiple:
            return len(self.series.itervalues().next())
        return len(self.series[0])

    #def debug_data(self):
    #""" 主要用于调试"""
    #return [s.data for s in self.series]

    def _added_to_tracker(self, tracker):
        if tracker:
            tracker.add_indicator(self)

    #def __tuple__(self):
    #""" 返回元组。某些指标,比如布林带有多个返回值。
    #这里以元组的形式返回多个序列变量。
    #"""
    #if isinstance(self.series, list):
    #return tuple(self.series)
    #else:
    #return (self.series,)

    #def __iter__(self):
    #return self

    #def next(self):
    #"""docstring for next"""
    #iter(self.series)

    def __call__(self, index):
        return self[index]

    def __getitem__(self, index):
        # 解析多元值, 返回series
        # python 3.x 有这种机制?
        # print self.name, index
        # print self.series[0].data
        if self.is_multiple:
            return self.series[index]
        # 返回单变量的值。
        if index >= 0:
            return self.series[0][index]
        else:
            raise SeriesIndexError

    def __float__(self):
        return self.series[0][0]

    def __str__(self):
        return str(self.series[0][0])

    #
    def __eq__(self, r):
        return float(self) == float(r)

    def __lt__(self, other):
        return float(self) < float(other)

    def __le__(self, other):
        return float(self) <= float(other)

    def __ne__(self, other):
        return float(self) != float(other)

    def __gt__(self, other):
        return float(self) > float(other)

    def __ge__(self, other):
        return float(self) >= float(other)

    #
    def __add__(self, r):
        return self.series[0][0] + float(r)

    def __sub__(self, r):
        return self.series[0][0] - float(r)

    def __mul__(self, r):
        return self.series[0][0] * float(r)

    def __div__(self, r):
        return self.series[0][0] / float(r)

    def __mod__(self, r):
        return self.series[0][0] % float(r)

    def __pow__(self, r):
        return self.series[0][0]**float(r)

    #
    def __radd__(self, r):
        return self.series[0][0] + float(r)

    def __rsub__(self, r):
        return self.series[0][0] - float(r)

    def __rmul__(self, r):
        return self.series[0][0] * float(r)

    def __rdiv__(self, r):
        return self.series[0][0] / float(r)

    def __rmod__(self, r):
        return self.series[0][0] % float(r)

    def __rpow__(self, r):
        return self.series[0][0]**float(r)
Example #42
0
class JarWriter(object):
    '''
    Class with methods to write Jar files. Can write more-or-less standard jar
    archives as well as jar archives optimized for Gecko. See the documentation
    for the close() member function for a description of both layouts.
    '''
    def __init__(self, file=None, fileobj=None, compress=True, optimize=True,
                 compress_level=9):
        '''
        Initialize a Jar archive in the given file. Use the given file-like
        object if one is given instead of opening the given file name.
        The compress option determines the default behavior for storing data
        in the jar archive. The optimize options determines whether the jar
        archive should be optimized for Gecko or not. ``compress_level``
        defines the zlib compression level. It must be a value between 0 and 9
        and defaults to 9, the highest and slowest level of compression.
        '''
        if fileobj:
            self._data = fileobj
        else:
            self._data = open(file, 'wb')
        if compress is True:
            compress = JAR_DEFLATED
        self._compress = compress
        self._compress_level = compress_level
        self._contents = OrderedDict()
        self._last_preloaded = None
        self._optimize = optimize

    def __enter__(self):
        '''
        Context manager __enter__ method for JarWriter.
        '''
        return self

    def __exit__(self, type, value, tb):
        '''
        Context manager __exit__ method for JarWriter.
        '''
        self.finish()

    def finish(self):
        '''
        Flush and close the Jar archive.

        Standard jar archives are laid out like the following:
            - Local file header 1
            - File data 1
            - Local file header 2
            - File data 2
            - (...)
            - Central directory entry pointing at Local file header 1
            - Central directory entry pointing at Local file header 2
            - (...)
            - End of central directory, pointing at first central directory
              entry.

        Jar archives optimized for Gecko are laid out like the following:
            - 32-bits unsigned integer giving the amount of data to preload.
            - Central directory entry pointing at Local file header 1
            - Central directory entry pointing at Local file header 2
            - (...)
            - End of central directory, pointing at first central directory
              entry.
            - Local file header 1
            - File data 1
            - Local file header 2
            - File data 2
            - (...)
            - End of central directory, pointing at first central directory
              entry.
        The duplication of the End of central directory is to accomodate some
        Zip reading tools that want an end of central directory structure to
        follow the central directory entries.
        '''
        offset = 0
        headers = {}
        preload_size = 0
        # Prepare central directory entries
        for entry, content in self._contents.itervalues():
            header = JarLocalFileHeader()
            for name in entry.STRUCT:
                if name in header:
                    header[name] = entry[name]
            entry['offset'] = offset
            offset += len(content) + header.size
            if entry['filename'] == self._last_preloaded:
                preload_size = offset
            headers[entry] = header
        # Prepare end of central directory
        end = JarCdirEnd()
        end['disk_entries'] = len(self._contents)
        end['cdir_entries'] = end['disk_entries']
        end['cdir_size'] = reduce(lambda x, y: x + y[0].size,
                                  self._contents.values(), 0)
        # On optimized archives, store the preloaded size and the central
        # directory entries, followed by the first end of central directory.
        if self._optimize:
            end['cdir_offset'] = 4
            offset = end['cdir_size'] + end['cdir_offset'] + end.size
            if preload_size:
                preload_size += offset
            self._data.write(struct.pack('<I', preload_size))
            for entry, _ in self._contents.itervalues():
                entry['offset'] += offset
                self._data.write(entry.serialize())
            self._data.write(end.serialize())
        # Store local file entries followed by compressed data
        for entry, content in self._contents.itervalues():
            self._data.write(headers[entry].serialize())
            self._data.write(content)
        # On non optimized archives, store the central directory entries.
        if not self._optimize:
            end['cdir_offset'] = offset
            for entry, _ in self._contents.itervalues():
                self._data.write(entry.serialize())
        # Store the end of central directory.
        self._data.write(end.serialize())
        self._data.close()

    def add(self, name, data, compress=None, mode=None, skip_duplicates=False):
        '''
        Add a new member to the jar archive, with the given name and the given
        data.
        The compress option indicates how the given data should be compressed
        (one of JAR_STORED, JAR_DEFLATE or JAR_BROTLI), or compressed according
        to the default defined when creating the JarWriter (None). True and
        False are allowed values for backwards compatibility, mapping,
        respectively, to JAR_DEFLATE and JAR_STORED.
        When the data should be compressed, it is only really compressed if
        the compressed size is smaller than the uncompressed size.
        The mode option gives the unix permissions that should be stored
        for the jar entry.
        If a duplicated member is found skip_duplicates will prevent raising
        an exception if set to True.
        The given data may be a buffer, a file-like instance, a Deflater or a
        JarFileReader instance. The latter two allow to avoid uncompressing
        data to recompress it.
        '''
        name = mozpath.normsep(name)

        if name in self._contents and not skip_duplicates:
            raise JarWriterError("File %s already in JarWriter" % name)
        if compress is None:
            compress = self._compress
        if compress is True:
            compress = JAR_DEFLATED
        if compress is False:
            compress = JAR_STORED
        if (isinstance(data, (JarFileReader, Deflater)) and \
                data.compress == compress):
            deflater = data
        else:
            deflater = Deflater(compress, compress_level=self._compress_level)
            if isinstance(data, basestring):
                deflater.write(data)
            elif hasattr(data, 'read'):
                if hasattr(data, 'seek'):
                    data.seek(0)
                deflater.write(data.read())
            else:
                raise JarWriterError("Don't know how to handle %s" %
                                     type(data))
        # Fill a central directory entry for this new member.
        entry = JarCdirEntry()
        entry['creator_version'] = 20
        if mode is not None:
            # Set creator host system (upper byte of creator_version)
            # to 3 (Unix) so mode is honored when there is one.
            entry['creator_version'] |= 3 << 8
            entry['external_attr'] = (mode & 0xFFFF) << 16
        if deflater.compressed:
            entry['min_version'] = 20  # Version 2.0 supports deflated streams
            entry['general_flag'] = 2  # Max compression
            entry['compression'] = deflater.compress
        else:
            entry['min_version'] = 10  # Version 1.0 for stored streams
            entry['general_flag'] = 0
            entry['compression'] = JAR_STORED
        # January 1st, 2010. See bug 592369.
        entry['lastmod_date'] = ((2010 - 1980) << 9) | (1 << 5) | 1
        entry['lastmod_time'] = 0
        entry['crc32'] = deflater.crc32
        entry['compressed_size'] = deflater.compressed_size
        entry['uncompressed_size'] = deflater.uncompressed_size
        entry['filename'] = name
        self._contents[name] = entry, deflater.compressed_data

    def preload(self, files):
        '''
        Set which members of the jar archive should be preloaded when opening
        the archive in Gecko. This reorders the members according to the order
        of given list.
        '''
        new_contents = OrderedDict()
        for f in files:
            if not f in self._contents:
                continue
            new_contents[f] = self._contents[f]
            self._last_preloaded = f
        for f in self._contents:
            if not f in new_contents:
                new_contents[f] = self._contents[f]
        self._contents = new_contents
Example #43
0
class alg:
    def __repr__(self):
        return "LIRS"

    def __init__(self, c, **kwargs):
        # c is cache size
        self.c = c  # Max
        self.lirs = 0
        self.hirs = 0
        self.maxhirs = max(1, 0.01*c)
        self.maxlirs = c - self.maxhirs
        self.S = OrderedDict()  # Recently accessed pages (LIR pages and non-resident HIR pages), newest element is last
        self.Q = OrderedDict()  # Resident HIR pages
        self.hitcount = 0
        self.count = 0

    def setup(self, reqlist):
        # I'm an online algorithm :-)
        pass

    def get(self, key, warm=0):
        #if self.count % 100 == 0:
        #       sys.stderr.write("S: %d\n" % len(self.S))
        #self.countResident()
        if not warm:
            self.count += 1
        if (key in self.S and self.S[key].resident) or key in self.Q:
            if key in self.S:
                e = self.S[key]
                del self.S[key]
                if e.flag == HIR:
                # A HIR block with lower reuse distance than the bottom LIR element
                    if e.resident:
                        del self.Q[key]
                        self.hirs -= 1
                    self.swap()   # Change one LIR page to a HIR page
                    self.evict()  # Keep the number of resident HIR pages <= self.maxhirs
                    self.lirs += 1
                e.resident = True
                e.flag = LIR
                self.S[key] = e
                self.prune()  #  If this was a LIR element at the bottom
            elif key in self.Q:
            # Resident HIR page which is not in S, reuse distance is large so we don't make it a LIR page
                e = self.Q[key]
                #assert e.resident and e.flag == HIR
                del self.Q[key]
                self.Q[key] = e
                self.S[key] = e
            #print "HIT for %s" % key
            if not warm:
                self.hitcount += 1
            return 1
        return 0
        #print "MISS for %s" % key

#old get is not right
#    def get(self, key):
#        #if self.count % 100 == 0:
#        #       sys.stderr.write("S: %d\n" % len(self.S))
#        self.countResident()
#        self.count += 1
#        if (key in self.S and self.S[key].resident) or key in self.Q:
#            #print "HIT for %s" % key
#            self.hitcount += 1
#            return 1
#        return 0
#        #print "MISS for %s" % key

    def put(self, key, val=1):
        if key in self.S:
            e = self.S[key]
            del self.S[key]
            if e.flag == HIR:
                # A HIR block with lower reuse distance than the bottom LIR element
                if e.resident:
                    del self.Q[key]
                    self.hirs -= 1
                #assert self.oldestS().flag == LIR
                #assert self.hirs <= self.maxhirs
                self.swap()   # Change one LIR page to a HIR page
                self.evict()  # Keep the number of resident HIR pages <= self.maxhirs
                self.lirs += 1
            e.resident = True
            e.flag = LIR
            self.S[key] = e
            self.prune()  #  If this was a LIR element at the bottom
        elif key in self.Q:
            # Resident HIR page which is not in S, reuse distance is large so we don't make it a LIR page
            e = self.Q[key]
            #assert e.resident and e.flag == HIR
            del self.Q[key]
            self.Q[key] = e
            self.S[key] = e
        else:
            # not in cache
            # When a miss occurs and a free block is needed for replacement, we choose an HIR block that is resident in the cache
            e = entry(key)
            if self.lirs < self.maxlirs:
                e.flag = LIR   # Not using all the cache, make it a LIR page
                self.lirs += 1
            else:
                # NOTE: Not changing the number of LIR blocks!
                self.Q[key] = e
                self.hirs += 1
                self.evict() # Evicting if we have max number of HIR blocks
            self.S[key] = e

        #assert self.oldestS().flag == LIR
        #self.countResident()

    def countResident(self):
        sm = 0
        for key, e in self.S.items():
            sm += 1 if e.resident else 0
        for key, e in self.Q.items():
            sm += 1 if key not in self.S else 0
        if sm != self.lirs + self.hirs:
            print "sm=%d, self.lirs=%d, self.hirs=%d" % (sm, self.lirs, self.hirs)
        assert sm == self.lirs + self.hirs

    def evict(self):
        if self.hirs > self.maxhirs:        # Setting the size of Q
            k, e_hir = self.Q.popitem(last=False)
            #assert e_hir.flag == HIR and e_hir.resident
            e_hir.resident = False                # It is maybe in S
            self.hirs -= 1
        if self.hirs > self.maxhirs:
            print "hirs: %d maxhirs: %d" % (self.hirs, self.maxhirs)
        #assert self.hirs <= self.maxhirs
        # TODO: If there are many nonresident entries in S, should we delete this from S?

    def swap(self):
        key, e = self.S.popitem(last=False)
        #assert e.flag == LIR and e.resident
        e.flag = HIR
        self.hirs += 1
        self.lirs -= 1
        self.Q[e.key] = e
        # Switched this LIR block to HIR and put it in Q
        # Important! We don't put it back in S


    def prune(self):
        while self.S:
            oldest = self.oldestS()
            if oldest.flag == LIR:
                break
            k, dele = self.S.popitem(last=False)
            #assert dele.flag == HIR
            #if dele.resident:
                #assert k in self.Q
            #else:
                #assert k not in self.Q

    def oldestS(self):
        return self.S.itervalues().next()

    def oldestQ(self):
        return self.Q.itervalues().next()
Example #44
0
class Environment(object):
    """
    Environment within which all agents operate.
    """

    hard_time_limit = -100  # Set a hard time limit even if deadline is not enforced.
    #. define these here?
    # num_dummies     = 100 # number of other agents on the roads
    # grid_size       = (8, 6) # columns, rows

    valid_actions = [None, 'forward', 'left', 'right']
    valid_inputs = {
        'light':    TrafficLight.valid_states,
        'oncoming': valid_actions,
        'left':     valid_actions,
        'right':    valid_actions
    }
    valid_headings = [(1, 0), (0, -1), (-1, 0), (0, 1)]  # E, N, W, S #. use symbols


    #. put verbose last
    def __init__(self, verbose=False, num_dummies=100, grid_size = (8, 6)): #. magic #s
        """
        Create an environment
        verbose     - set to True to display additional output from the simulation
        num_dummies - discrete number of dummy agents in the environment
        grid_size   - discrete number of intersections (columns, rows)
        """

        self.num_dummies = num_dummies  # Number of dummy driver agents in the environment
        self.verbose = verbose # If debug output should be given

        # Initialize simulation variables
        self.done = False
        self.t = 0 #? integer time step
        self.agent_states = OrderedDict()
        self.step_data = {}
        self.success = None #? did the agent reach the destination in time?

        # Road network
        self.grid_size = grid_size  # (columns, rows)
        self.bounds = (1, 2, self.grid_size[0], self.grid_size[1] + 1) # ? bad - storing info in unlabeled tuple
        self.block_size = 100  #?
        self.hang = 0.6  #?
        self.intersections = OrderedDict() #?
        self.roads = []  #?

        #? Add traffic lights to each intersection
        for x in xrange(self.bounds[0], self.bounds[2] + 1):
            for y in xrange(self.bounds[1], self.bounds[3] + 1):
                self.intersections[(x, y)] = TrafficLight()  # A traffic light at each intersection

        #? Add roads between adjacent traffic lights
        for a in self.intersections:
            for b in self.intersections:
                if a == b:
                    continue
                if (abs(a[0] - b[0]) + abs(a[1] - b[1])) == 1:  # L1 distance = 1
                    self.roads.append((a, b))

        # Add environment boundaries
        #? what is this? roads along edges?
        #. prefer range over xrange for upwards compatibility
        for x in range(self.bounds[0], self.bounds[2] + 1):
            self.roads.append(((x, self.bounds[1] - self.hang), (x, self.bounds[1])))
            self.roads.append(((x, self.bounds[3] + self.hang), (x, self.bounds[3])))
        for y in range(self.bounds[1], self.bounds[3] + 1):
            self.roads.append(((self.bounds[0] - self.hang, y), (self.bounds[0], y)))
            self.roads.append(((self.bounds[2] + self.hang, y), (self.bounds[2], y)))

        # Create dummy agents
        for i in range(self.num_dummies):
            self.create_agent(DummyAgent)

        # Primary agent and associated parameters
        self.primary_agent = None  # to be set explicitly
        self.enforce_deadline = False

        # Trial data (updated at the end of each trial)
        self.trial_data = {
            'testing': False,        # if the trial is for testing a learned policy
            'initial_distance': 0,   # L1 distance from start to destination
            'initial_deadline': 0,   # given deadline (time steps) to start with
            'net_reward': 0.0,       # total reward earned in current trial
            'final_deadline': None,  # deadline value (time remaining) at the end
            'actions': {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}, # violations and accidents #. unlabelled dict!
            'coverage': 0,           # percentage of state-actions seen
            'success': 0,            # whether the agent reached the destination in time
        }


    def create_agent(self, agent_class, *args, **kwargs):
        """
        Create an agent in the environment at a random location.
        """
        #. parameters
        #   learning - set to True to force the driving agent to use Q-learning
        #   epsilon  - continuous value for the exploration factor, default is 1
        #   alpha    - continuous value for the learning rate, default is 0.5

        agent = agent_class(self, *args, **kwargs)
        # ? what is heading? why a tuple?
        # self.agent_states[agent] = {'location': random.choice(self.intersections.keys()), 'heading': (0, 1)}
        intersection = random.choice(self.intersections.keys())
        self.agent_states[agent] = {'location': intersection, 'heading': (0, 1)}
        return agent


    def set_primary_agent(self, agent, enforce_deadline=False):
        """
        Set the given agent as the primary agent.
        The primary agent is the smartcab that is followed in the environment.
        """
        self.primary_agent = agent
        agent.primary_agent = True
        self.enforce_deadline = enforce_deadline


    def reset(self, testing=False):
        """
        Begin a new trial.
        """

        self.done = False
        self.t = 0

        # Reset status text
        self.step_data = {} #? this is status text?

        # Reset traffic lights
        for traffic_light in self.intersections.itervalues():
            traffic_light.reset()

        # Pick a random start and destination
        start = random.choice(self.intersections.keys())
        destination = random.choice(self.intersections.keys())

        # Ensure starting location and destination are not too close
        while self.compute_dist(start, destination) < 4: #. magic number
            start = random.choice(self.intersections.keys())
            destination = random.choice(self.intersections.keys())

        # # Pick a random start and destination that aren't too close
        # min_distance = 4
        # distance = 0
        # while distance < min_distance:
        #     start = random.choice(self.intersections.keys())
        #     destination = random.choice(self.intersections.keys())
        #     distance = self.compute_dist(start, destination)

        start_heading = random.choice(self.valid_headings) #. pick a direction
        distance = self.compute_dist(start, destination) #. would already have from above
        deadline = distance * 5 # 5 time steps per intersection away #. magic number

        if self.verbose: # Debugging
            print "Environment.reset(): Trial set up with start = {}, destination = {}, deadline = {}".format(start, destination, deadline)

        # Create a map of all possible initial positions
        #? what is this?
        positions = dict()
        for location in self.intersections:
            positions[location] = list()
            for heading in self.valid_headings:
                positions[location].append(heading)

        # Initialize agent(s)
        #? why is the set of agents called agent_states and not just agents?
        #. call it agent_key also
        for agent in self.agent_states.iterkeys():
            #. agent = self.agents[agent_key]
            # or
            # for agent in self.agents.values():

            if agent is self.primary_agent:
            # if agent.primary_agent: #. better?
                #. wait, the environment is storing the agents states? why?
                self.agent_states[agent] = {
                    'location': start,
                    'heading': start_heading,
                    'destination': destination,
                    'deadline': deadline
                }

            # For dummy agents, make them choose one of the available
            # intersections and headings still in 'positions'
            else:
                intersection = random.choice(positions.keys())
                heading = random.choice(positions[intersection])
                self.agent_states[agent] = {
                    'location': intersection,
                    'heading': heading,
                    'destination': None,
                    'deadline': None
                }
                # Now delete the taken location and heading from 'positions'
                positions[intersection] = list(set(positions[intersection]) - set([heading]))
                if positions[intersection] == list(): # No headings available for intersection
                    del positions[intersection] # Delete the intersection altogether


            #? whoa, so agent is an object, and it's used as the key in self.agent_states -
            # weird...
            agent.reset(destination=(destination if agent is self.primary_agent else None), testing=testing)

            if agent is self.primary_agent:
                # Reset metrics for this trial (step data will be set during the step)
                self.trial_data['testing'] = testing
                self.trial_data['initial_deadline'] = deadline
                self.trial_data['final_deadline'] = deadline
                self.trial_data['net_reward'] = 0.0
                self.trial_data['actions'] = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}
                self.trial_data['parameters'] = {'e': agent.epsilon, 'a': agent.alpha}
                self.trial_data['success'] = 0
                self.trial_data['coverage'] = 0


    def step(self):
        """
        This function is called when a time step is taken turing a trial.
        """

        # Pretty print to terminal
        # print ""
        # print "/-------------------"
        # print "| Step {} Results".format(self.t)
        # print "\-------------------"
        # print ""

        if self.verbose: # Debugging
            print "Environment.step(): t = {}".format(self.t)

        # Update agents, primary first
        if self.primary_agent is not None:
            self.primary_agent.update()
        for agent in self.agent_states.iterkeys():
            if agent is not self.primary_agent:
                agent.update()

        # Update traffic lights
        for intersection, traffic_light in self.intersections.iteritems():
            traffic_light.update(self.t)

        #? update primary agent's deadline #. rename to time_remaining
        if self.primary_agent is not None:
        #. if self.primary_agent:
            # Agent has taken an action: reduce the deadline by 1
            agent_deadline = self.agent_states[self.primary_agent]['deadline'] - 1
            self.agent_states[self.primary_agent]['deadline'] = agent_deadline
            #. Agent has taken an action: reduce the time remaining by 1
            #. self.agent_states[self.primary_agent]['time_remaining'] -= 1

            if agent_deadline <= self.hard_time_limit:
                self.done = True
                self.success = False
                if self.verbose: # Debugging
                    print "Environment.step(): Primary agent hit hard time limit ({})! Trial aborted.".format(self.hard_time_limit)
            elif self.enforce_deadline and agent_deadline <= 0:
                self.done = True
                self.success = False
                if self.verbose: # Debugging
                    print "Environment.step(): Primary agent ran out of time! Trial aborted."

        self.t += 1 # environment stores the global time state


    def sense(self, agent): #. rename - getSensorInformation?
        """
        Get sensor input information for an 'agent' in the environment.
        """

        assert (agent in self.agent_states), "Unknown agent!"

        #? Get location, heading, light status for given agent
        state = self.agent_states[agent]  #. should just get state directly from agent, not environment
        location = state['location']
        heading = state['heading']
        #. confusing... - use red/green instead of heading 0/1
        light = 'green' if (self.intersections[location].state and heading[1] != 0) or \
                ((not self.intersections[location].state) and heading[0] != 0) else 'red'

        # Populate oncoming, left, right
        #? which are...?
        oncoming = None
        left = None
        right = None
        for other_agent, other_state in self.agent_states.iteritems():
            #. confusing
            if agent == other_agent or location != other_state['location'] or \
               (heading[0] == other_state['heading'][0] and heading[1] == other_state['heading'][1]):
                continue
            # For dummy agents, ignore the primary agent
            # This is because the primary agent is not required to follow the waypoint
            if other_agent == self.primary_agent: #. should use 'is' ?
                continue
            other_heading = other_agent.get_next_waypoint()
            # ?
            #. confusing
            if (heading[0] * other_state['heading'][0] + heading[1] * other_state['heading'][1]) == -1:
                if oncoming != 'left':  # we don't want to override oncoming == 'left'
                    oncoming = other_heading
            elif (heading[1] == other_state['heading'][0] and -heading[0] == other_state['heading'][1]):
                if right != 'forward' and right != 'left':  # we don't want to override right == 'forward or 'left'
                    right = other_heading
            else:
                if left != 'forward':  # we don't want to override left == 'forward'
                    left = other_heading

        #. sensors = {light: light, ...}
        # return sensors
        return {'light': light, 'oncoming': oncoming, 'left': left, 'right': right}


    #. bad name - should be time remaining
    def get_deadline(self, agent):
        """
        Returns the deadline remaining for an agent.
        """

        return self.agent_states[agent]['deadline'] if agent is self.primary_agent else None


    def act(self, agent, action):
        """
        Consider an action and perform the action if it is legal.
        Returns a reward for the agent based on traffic laws.
        """
        #. long fn - break down a bit
        #. doesn't include reward for reaching destination?
        #. this belongs in Agent, yeah?

        # assert agent in self.agent_states, "Unknown agent!"
        # assert action in self.valid_actions, "Invalid action!"
        assert (agent in self.agent_states), "Unknown agent!"
        assert (action in self.valid_actions), "Invalid action!"

        state = self.agent_states[agent]
        location = state['location']
        heading = state['heading']
        #. confusing, duplicate code
        light = 'green' if (self.intersections[location].state and heading[1] != 0) or \
                ((not self.intersections[location].state) and heading[0] != 0) else 'red'
        inputs = self.sense(agent)

        # Assess whether the agent can move based on the action chosen.
        # Either the action is okay to perform, or falls under 4 types of violations:
        #. use strings or consts for violation types
        # 0: Action okay
        # 1: Minor traffic violation
        # 2: Major traffic violation
        # 3: Minor traffic violation causing an accident
        # 4: Major traffic violation causing an accident
        violation = 0

        # Reward scheme
        # First initialize reward uniformly random from [-1, 1]
        reward = 2 * random.random() - 1

        # Create a penalty factor as a function of remaining deadline
        # Scales reward multiplicatively from [0, 1]
        #? fnc means what?
        fnc = self.t * 1.0 / (self.t + state['deadline']) if agent.primary_agent else 0.0
        gradient = 10 #. magic number - gradient of what?

        # No penalty given to an agent that has no enforced deadline
        #. put this in an else clause below
        penalty = 0

        # If the deadline is enforced, give a penalty based on time remaining
        #? explain penalty fn
        if self.enforce_deadline:
            penalty = (math.pow(gradient, fnc) - 1) / (gradient - 1)

        # Agent wants to drive forward:
        if action == 'forward':
            if light != 'green': # Running red light
                violation = 2 # Major violation #. magic# (and several more)
                if inputs['left'] == 'forward' or inputs['right'] == 'forward': # Cross traffic
                    violation = 4 # Accident

        # Agent wants to drive left:
        elif action == 'left':
            if light != 'green': # Running a red light
                violation = 2 # Major violation
                if inputs['left'] == 'forward' or inputs['right'] == 'forward': # Cross traffic
                    violation = 4 # Accident
                elif inputs['oncoming'] == 'right': # Oncoming car turning right
                    violation = 4 # Accident
            else: # Green light
                if inputs['oncoming'] == 'right' or inputs['oncoming'] == 'forward': # Incoming traffic
                    violation = 3 # Accident
                else: # Valid move!
                    heading = (heading[1], -heading[0])

        # Agent wants to drive right:
        elif action == 'right':
            if light != 'green' and inputs['left'] == 'forward': # Cross traffic
                violation = 3 # Accident
            else: # Valid move!
                heading = (-heading[1], heading[0])

        # Agent wants to perform no action:
        elif action == None:
            if light == 'green' and inputs['oncoming'] != 'left': # No oncoming traffic
                violation = 1 # Minor violation


        # Did the agent attempt a valid move?
        if violation == 0:
            #. magic numbers
            if action == agent.get_next_waypoint(): # Was it the correct action?
                reward += 2 - penalty # (2, 1)
            elif action == None and light != 'green': # Was the agent stuck at a red light?
                reward += 2 - penalty # (2, 1)
            else: # Valid but incorrect
                reward += 1 - penalty # (1, 0)

            # Move the agent
            if action is not None:
                #. confusing - wrap around torus world?
                location = ((location[0] + heading[0] - self.bounds[0]) % (self.bounds[2] - self.bounds[0] + 1) + self.bounds[0],
                            (location[1] + heading[1] - self.bounds[1]) % (self.bounds[3] - self.bounds[1] + 1) + self.bounds[1])  # wrap-around
                state['location'] = location
                state['heading'] = heading
        # Agent attempted invalid move
        else:
            #. magic #s - keys and rewards - define as a dictionary at start
            if violation == 1: # Minor violation
                reward += -5
            elif violation == 2: # Major violation
                reward += -10
            elif violation == 3: # Minor accident
                reward += -20
            elif violation == 4: # Major accident
                reward += -40

        # Did agent reach the goal after a valid move?
        if agent is self.primary_agent:
            if state['location'] == state['destination']:
                # Did agent get to destination before deadline?
                if state['deadline'] >= 0:
                    self.trial_data['success'] = 1

                # Stop the trial
                self.done = True
                self.success = True

                if self.verbose: # Debugging
                    print "Environment.act(): Primary agent has reached destination!"

            if self.verbose: # Debugging
                print "Environment.act() [POST]: location: {}, heading: {}, action: {}, reward: {}".format(location, heading, action, reward)

            # Update metrics
            self.step_data['t'] = self.t
            self.step_data['violation'] = violation
            self.step_data['state'] = agent.get_state()
            self.step_data['deadline'] = state['deadline']
            self.step_data['waypoint'] = agent.get_next_waypoint()
            self.step_data['inputs'] = inputs
            self.step_data['light'] = light
            self.step_data['action'] = action
            self.step_data['reward'] = reward

            self.trial_data['final_deadline'] = state['deadline'] - 1
            self.trial_data['net_reward'] += reward
            self.trial_data['actions'][violation] += 1
            self.trial_data['coverage'] = agent.coverage

            if self.verbose: # Debugging
                print "Environment.act(): Step data: {}".format(self.step_data)

        return reward


    def compute_dist(self, a, b):
        """
        Compute the Manhattan (L1) distance of a toroidal world.
        """

        dx1 = abs(b[0] - a[0])
        dx2 = abs(self.grid_size[0] - dx1)
        dx = dx1 if dx1 < dx2 else dx2

        dy1 = abs(b[1] - a[1])
        dy2 = abs(self.grid_size[1] - dy1)
        dy = dy1 if dy1 < dy2 else dy2

        return dx + dy
Example #45
0
class CoverSettingsWidget(QWidget):

    changed = pyqtSignal()

    def __init__(self, mi=None, prefs=None, parent=None, for_global_prefs=False):
        QWidget.__init__(self, parent)
        self.ignore_changed = False
        self.for_global_prefs = for_global_prefs

        self.l = l = QHBoxLayout(self)
        l.setContentsMargins(0, 0, 0, 0)
        self.setLayout(l)
        self.settings_tabs = st = QTabWidget(self)
        l.addWidget(st)
        self.preview_label = la = Preview(self)
        l.addWidget(la)

        if prefs is None:
            prefs = cprefs
        self.original_prefs = prefs
        self.mi = mi or self.default_mi()

        self.colors_page = cp = QWidget(st)
        st.addTab(cp, _('&Colors'))
        cp.l = l = QGridLayout()
        cp.setLayout(l)
        if for_global_prefs:
            msg = _('When generating covers, a color scheme for the cover is chosen at random from the'
                    ' color schemes below. You can prevent an individual scheme from being selected by'
                    ' unchecking it. The preview on the right shows the currently selected color scheme.')
        else:
            msg = _('Choose a color scheme to be used for this generated cover.') + '<p>' + _(
                'In normal cover generation, the color scheme is chosen at random from the list of color schemes below. You'
                ' can prevent an individual color scheme from being chosen by unchecking it here.')
        cp.la = la = QLabel('<p>' + msg)
        la.setWordWrap(True)
        l.addWidget(la, 0, 0, 1, -1)
        self.colors_list = cl = QListWidget(cp)
        l.addWidget(cl, 1, 0, 1, -1)
        self.colors_map = OrderedDict()
        self.ncs = ncs = QPushButton(QIcon(I('plus.png')), _('&New color scheme'), cp)
        ncs.clicked.connect(self.create_color_scheme)
        l.addWidget(ncs)
        self.ecs = ecs = QPushButton(QIcon(I('format-fill-color.png')), _('&Edit color scheme'), cp)
        ecs.clicked.connect(self.edit_color_scheme)
        l.addWidget(ecs, l.rowCount()-1, 1)
        self.rcs = rcs = QPushButton(QIcon(I('minus.png')), _('&Remove color scheme'), cp)
        rcs.clicked.connect(self.remove_color_scheme)
        l.addWidget(rcs, l.rowCount()-1, 2)

        self.styles_page = sp = QWidget(st)
        st.addTab(sp, _('&Styles'))
        sp.l = l = QVBoxLayout()
        sp.setLayout(l)
        if for_global_prefs:
            msg = _('When generating covers, a style for the cover is chosen at random from the'
                    ' styles below. You can prevent an individual style from being selected by'
                    ' unchecking it. The preview on the right shows the currently selected style.')
        else:
            msg = _('Choose a style to be used for this generated cover.') + '<p>' + _(
                'In normal cover generation, the style is chosen at random from the list of styles below. You'
                ' can prevent an individual style from being chosen by unchecking it here.')
        sp.la = la = QLabel('<p>' + msg)
        la.setWordWrap(True)
        l.addWidget(la)
        self.styles_list = sl = QListWidget(sp)
        l.addWidget(sl)
        self.style_map = OrderedDict()

        self.font_page = fp = QWidget(st)
        st.addTab(fp, _('&Fonts and sizes'))
        fp.l = l = QFormLayout()
        fp.setLayout(l)
        fp.f = []

        def add_hline():
            f = QFrame()
            fp.f.append(f)
            f.setFrameShape(f.HLine)
            l.addRow(f)

        for x, label, size_label in (
                ('title', _('&Title font family:'), _('&Title font size:')),
                ('subtitle', _('&Subtitle font family'), _('&Subtitle font size:')),
                ('footer', _('&Footer font family'), _('&Footer font size')),
        ):
            attr = '%s_font_family' % x
            ff = FontFamilyChooser(fp)
            setattr(self, attr, ff)
            l.addRow(label, ff)
            ff.family_changed.connect(self.emit_changed)
            attr = '%s_font_size' % x
            fs = QSpinBox(fp)
            setattr(self, attr, fs)
            fs.setMinimum(8), fs.setMaximum(200), fs.setSuffix(' px')
            fs.setValue(prefs[attr])
            fs.valueChanged.connect(self.emit_changed)
            l.addRow(size_label, fs)
            add_hline()
        self.changed_timer = t = QTimer(self)
        t.setSingleShot(True), t.setInterval(500), t.timeout.connect(self.emit_changed)

        def create_sz(label):
            ans = QSpinBox(self)
            ans.setSuffix(' px'), ans.setMinimum(100), ans.setMaximum(10000)
            l.addRow(label, ans)
            ans.valueChanged.connect(self.changed_timer.start)
            return ans

        self.cover_width = create_sz(_('Cover &width:'))
        self.cover_height = create_sz(_('Cover &height:'))
        fp.cla = la = QLabel(_(
            'Note that the preview to the side is of fixed aspect ratio, so changing the cover'
            ' width above will not have any effect. If you change the height, you should also change the width nevertheless'
            ' as it will be used in actual cover generation.'))
        la.setWordWrap(True)
        l.addRow(la)

        self.templates_page = tp = QWidget(st)
        st.addTab(tp, _('&Text'))
        tp.l = l = QVBoxLayout()
        tp.setLayout(l)
        tp.la = la = QLabel(_(
            'The text on the generated cover is taken from the metadata of the book.'
            ' This is controlled via templates. You can use the <b>, <i> and <br> tags'
            ' in the templates for bold, italic and line breaks, respectively. The'
            ' default templates use the title, series and authors. You can change them to use'
            ' whatever metadata you like.'))
        la.setWordWrap(True), la.setTextFormat(Qt.PlainText)
        l.addWidget(la)

        def create_template_widget(title, which, button):
            attr = which + '_template'
            heading = QLabel('<h2>' + title)
            setattr(tp, attr + '_heading', heading)
            l.addWidget(heading)
            la = QLabel()
            setattr(self, attr, la)
            l.addWidget(la), la.setTextFormat(Qt.PlainText), la.setStyleSheet('QLabel {font-family: monospace}')
            la.setWordWrap(True)
            b = QPushButton(button)
            b.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
            connect_lambda(b.clicked, self, lambda self: self.change_template(which))
            setattr(self, attr + '_button', b)
            l.addWidget(b)
            if which != 'footer':
                f = QFrame(tp)
                setattr(tp, attr + '_sep', f), f.setFrameShape(QFrame.HLine)
                l.addWidget(f)
            l.addSpacing(10)

        create_template_widget(_('The title template'), 'title', _('Change the &title template'))
        create_template_widget(_('The sub-title template'), 'subtitle', _('Change the &sub-title template'))
        create_template_widget(_('The footer template'), 'footer', _('Change the &footer template'))
        l.addStretch(2)

        self.apply_prefs(prefs)
        self.changed.connect(self.update_preview)
        self.styles_list.itemSelectionChanged.connect(self.update_preview)
        self.colors_list.itemSelectionChanged.connect(self.update_preview)
        self.update_preview()

    def __enter__(self):
        self.ignore_changed = True

    def __exit__(self, *args):
        self.ignore_changed = False

    def emit_changed(self):
        if not self.ignore_changed:
            self.changed.emit()

    def apply_prefs(self, prefs):
        with self:
            self._apply_prefs(prefs)

    def _apply_prefs(self, prefs):
        for x in ('title', 'subtitle', 'footer'):
            attr = '%s_font_family' % x
            getattr(self, attr).font_family = prefs[attr]
            attr = '%s_font_size' % x
            getattr(self, attr).setValue(prefs[attr])

        for x in ('title', 'subtitle', 'footer'):
            x += '_template'
            getattr(self, x).setText(prefs[x])

        for x in ('width', 'height'):
            x = 'cover_' + x
            getattr(self, x).setValue(prefs[x])

        color_themes = prefs['color_themes'].copy()
        color_themes.update(default_color_themes)
        disabled = set(prefs['disabled_color_themes'])
        self.colors_list.clear()
        self.colors_map = {}
        for name in sorted(color_themes, key=sort_key):
            self.colors_map[name] = li = QListWidgetItem(name, self.colors_list)
            li.setFlags(li.flags() | Qt.ItemIsUserCheckable)
            li.setCheckState(Qt.Unchecked if name in disabled else Qt.Checked)
            li.setData(Qt.UserRole, color_themes[name])
        lu = prefs.get('last_used_colors')
        if not self.for_global_prefs and lu in self.colors_map and self.colors_map[lu].checkState() == Qt.Checked:
            self.colors_map[lu].setSelected(True)
        else:
            for name, li in self.colors_map.iteritems():
                if li.checkState() == Qt.Checked:
                    li.setSelected(True)
                    break
            else:
                next(self.colors_map.itervalues()).setSelected(True)

        disabled = set(prefs['disabled_styles'])
        self.styles_list.clear()
        self.style_map.clear()
        for name in sorted(all_styles(), key=sort_key):
            self.style_map[name] = li = QListWidgetItem(name, self.styles_list)
            li.setFlags(li.flags() | Qt.ItemIsUserCheckable)
            li.setCheckState(Qt.Unchecked if name in disabled else Qt.Checked)
        lu = prefs.get('last_used_style')
        if not self.for_global_prefs and lu in self.style_map and self.style_map[lu].checkState() == Qt.Checked:
            self.style_map[lu].setSelected(True)
        else:
            for name, li in self.style_map.iteritems():
                if li.checkState() == Qt.Checked:
                    li.setSelected(True)
                    break
            else:
                next(self.style_map.itervalues()).setSelected(True)

    @property
    def current_colors(self):
        for name, li in self.colors_map.iteritems():
            if li.isSelected():
                return name

    @property
    def disabled_colors(self):
        for name, li in self.colors_map.iteritems():
            if li.checkState() == Qt.Unchecked:
                yield name

    @property
    def custom_colors(self):
        ans = {}
        for name, li in self.colors_map.iteritems():
            if name.startswith('#'):
                ans[name] = li.data(Qt.UserRole)
        return ans

    @property
    def current_style(self):
        for name, li in self.style_map.iteritems():
            if li.isSelected():
                return name

    @property
    def disabled_styles(self):
        for name, li in self.style_map.iteritems():
            if li.checkState() == Qt.Unchecked:
                yield name

    @property
    def current_prefs(self):
        prefs = {k:self.original_prefs[k] for k in self.original_prefs.defaults}
        for x in ('title', 'subtitle', 'footer'):
            attr = '%s_font_family' % x
            prefs[attr] = getattr(self, attr).font_family
            attr = '%s_font_size' % x
            prefs[attr] = getattr(self, attr).value()
        prefs['color_themes'] = self.custom_colors
        prefs['disabled_styles'] = list(self.disabled_styles)
        prefs['disabled_color_themes'] = list(self.disabled_colors)
        for x in ('title', 'subtitle', 'footer'):
            x += '_template'
            prefs[x] = getattr(self, x).text()
        for x in ('width', 'height'):
            x = 'cover_' + x
            prefs[x] = getattr(self, x).value()
        return prefs

    def insert_scheme(self, name, li):
        with self:
            self.colors_list.insertItem(0, li)
            cm = OrderedDict()
            cm[name] = li
            for k, v in self.colors_map.iteritems():
                cm[k] = v
            self.colors_map = cm
            li.setSelected(True)
            for i in range(1, self.colors_list.count()):
                self.colors_list.item(i).setSelected(False)

    def create_color_scheme(self):
        scheme = self.colors_map[self.current_colors].data(Qt.UserRole)
        d = CreateColorScheme('#' + _('My Color Scheme'), scheme, set(self.colors_map), parent=self)
        if d.exec_() == d.Accepted:
            name, scheme = d.data
            li = QListWidgetItem(name)
            li.setData(Qt.UserRole, scheme), li.setFlags(li.flags() | Qt.ItemIsUserCheckable), li.setCheckState(Qt.Checked)
            self.insert_scheme(name, li)
            self.emit_changed()
            self.original_prefs['color_themes'] = self.current_prefs['color_themes']

    def edit_color_scheme(self):
        cs = self.current_colors
        if cs is None or not cs.startswith('#'):
            return error_dialog(self, _('Cannot edit'), _(
                'Cannot edit a builtin color scheme. Create a new'
                ' color scheme instead.'), show=True)
        li = self.colors_map[cs]
        d = CreateColorScheme(cs, li.data(Qt.UserRole), set(self.colors_map), edit_scheme=True, parent=self)
        if d.exec_() == d.Accepted:
            name, scheme = d.data
            li.setText(name)
            li.setData(Qt.UserRole, scheme)
            if name != cs:
                self.colors_map.pop(cs, None)
                self.insert_scheme(name, li)
            self.emit_changed()
            self.original_prefs['color_themes'] = self.current_prefs['color_themes']

    def remove_color_scheme(self):
        cs = self.current_colors
        if cs is None or not cs.startswith('#'):
            return error_dialog(self, _('Cannot remove'), _(
                'Cannot remove a builtin color scheme.'), show=True)
        for i in range(self.colors_list.count()):
            item = self.colors_list.item(i)
            if item.isSelected():
                with self:
                    del self.colors_map[item.text()]
                    self.colors_list.takeItem(i)
                    i = i % self.colors_list.count()
                    self.colors_list.item(i).setSelected(True)
                self.emit_changed()
                self.original_prefs['color_themes'] = self.current_prefs['color_themes']
                return

    def change_template(self, which):
        from calibre.gui2.dialogs.template_dialog import TemplateDialog
        from calibre.gui2.ui import get_gui
        gui = get_gui()
        if gui is None:
            from calibre.ebooks.metadata.book.base import field_metadata
        else:
            field_metadata = gui.current_db.new_api.field_metadata
        attr = which + '_template'
        templ = getattr(self, attr).text()
        d = TemplateDialog(self, templ, mi=self.mi, fm=field_metadata)
        if d.exec_() == d.Accepted:
            templ = d.rule[1]
            getattr(self, attr).setText(templ)
            self.emit_changed()

    @property
    def prefs_for_rendering(self):
        prefs = self.current_prefs
        prefs = override_prefs(prefs, override_style=self.current_style, override_color_theme=self.current_colors)
        return prefs

    def update_preview(self):
        if self.ignore_changed:
            return
        dpr = getattr(self, 'devicePixelRatioF', self.devicePixelRatio)()
        w, h = int(dpr * self.preview_label.sizeHint().width()), int(dpr * self.preview_label.sizeHint().height())
        prefs = self.prefs_for_rendering
        hr = h / prefs['cover_height']
        for x in ('title', 'subtitle', 'footer'):
            attr = '%s_font_size' % x
            prefs[attr] = int(prefs[attr] * hr)
        prefs['cover_width'], prefs['cover_height'] = w, h
        img = generate_cover(self.mi, prefs=prefs, as_qimage=True)
        img.setDevicePixelRatio(dpr)
        self.preview_label.setPixmap(QPixmap.fromImage(img))

    def default_mi(self):
        from calibre.ebooks.metadata.book.base import Metadata
        mi = Metadata(_('A sample book'), [_('Author One'), _('Author Two')])
        mi.series = _('A series of samples')
        mi.series_index = 4
        mi.tags = [_('Tag One'), _('Tag Two')]
        mi.publisher = _('Some publisher')
        mi.rating = 4
        mi.identifiers = {'isbn':'123456789', 'url': 'https://calibre-ebook.com'}
        mi.languages = ['eng', 'fra']
        mi.pubdate = mi.timestamp = now()
        return mi

    def restore_defaults(self):
        defaults = self.original_prefs.defaults.copy()
        # Dont delete custom color themes when restoring defaults
        defaults['color_themes'] = self.custom_colors
        self.apply_prefs(defaults)
        self.update_preview()

    def save_state(self):
        self.original_prefs.set('last_used_colors', self.current_colors)
        self.original_prefs.set('last_used_style', self.current_style)

    def save_as_prefs(self):
        with self.original_prefs:
            for k, v in self.current_prefs.iteritems():
                self.original_prefs[k] = v