class ForgetfulStorage(object): implements(IStorage) def __init__(self, ttl=604800): """ By default, max age is a week. """ self.data = OrderedDict() self.ttl = ttl def __setitem__(self, key, value): if key in self.data: del self.data[key] self.data[key] = (time.time(), value) self.cull() def cull(self): for k, v in self.iteritemsOlderThan(self.ttl): self.data.popitem(last=False) def get(self, key, default=None): self.cull() if key in self.data: return False, self[key] return False, default def __getitem__(self, key): self.cull() return self.data[key][1] def __iter__(self): self.cull() return iter(self.data) def __repr__(self): self.cull() return repr(self.data) def iteritemsOlderThan(self, secondsOld): minBirthday = time.time() - secondsOld zipped = self._tripleIterable() matches = takewhile(lambda r: minBirthday >= r[1], zipped) return imap(operator.itemgetter(0, 2), matches) def _tripleIterable(self): ikeys = self.data.iterkeys() ibirthday = imap(operator.itemgetter(0), self.data.itervalues()) ivalues = imap(operator.itemgetter(1), self.data.itervalues()) return izip(ikeys, ibirthday, ivalues) def iteritems(self): self.cull() ikeys = self.data.iterkeys() ivalues = imap(operator.itemgetter(1), self.data.itervalues()) return izip(ikeys, ivalues)
def run_cloners(cls, old_event, new_event): selected = {x[7:] for x in request.values.getlist('cloners') if x.startswith('cloner_')} all_cloners = OrderedDict((name, cloner_cls(old_event)) for name, cloner_cls in get_event_cloners().iteritems()) if any(cloner.is_internal for name, cloner in all_cloners.iteritems() if name in selected): raise Exception('An internal cloner was selected') # enable internal cloners that are enabled by default or required by another cloner selected |= {c.name for c in all_cloners.itervalues() if c.is_internal and (c.is_default or c.required_by_deep & selected)} # enable unavailable cloners that may be pulled in as a dependency nonetheless extra = {c.name for c in all_cloners.itervalues() if not c.is_available and c.always_available_dep and c.required_by_deep & selected} selected |= extra active_cloners = OrderedDict((name, cloner) for name, cloner in all_cloners.iteritems() if name in selected) if not all((c.is_internal or c.is_visible) and c.is_available for c in active_cloners.itervalues() if c.name not in extra): raise Exception('An invisible/unavailable cloner was selected') for name, cloner in active_cloners.iteritems(): if not (selected >= cloner.requires_deep): raise Exception('Cloner {} requires {}'.format(name, ', '.join(cloner.requires_deep - selected))) shared_data = {} cloner_names = set(active_cloners) for name, cloner in active_cloners.iteritems(): shared_data[name] = cloner.run(new_event, cloner_names, cloner._prepare_shared_data(shared_data))
def by(self, name, index=None, field=None): """ facet by given function Examples: .by(index=lambda x: x) .by(field=lambda x: ('T_high' if x['T'] > 1000 else 'T_low')) """ if index: ret = OrderedDict( [(index(val),self[self.index.get_level_values(name) == val]) for val in self.index.get_level_values(name)] ) for _ in ret.itervalues(): _.properties = self.properties return MultiFrame(ret) else: self['cat'] = map(field, self[name]) cats = self.groupby('cat').groups.keys() ret = OrderedDict( [(cat,self[self['cat'] == cat]) for cat in cats] ) for _ in ret.itervalues(): _.properties = self.properties return MultiFrame(ret)
def process_static_data(cls, data): # create instances ret = OrderedDict() for t in sorted(data, key=lambda x: x["name"]): name = str(t["name"]) ret[name] = Type(name, t["effectiveAgainst"], t["weakAgainst"]) # additional manipulations size = len(ret) by_effectiveness = {} by_resistance = {} for t in ret.itervalues(): # type: Type t.attack_effective_against = [ret[name] for name in t.attack_effective_against] t.attack_weak_against = [ret[name] for name in t.attack_weak_against] # group types effective against, weak against specific types for l, d in (t.attack_effective_against, by_effectiveness), \ (t.attack_weak_against, by_resistance): for tt in l: if tt not in d: d[tt] = set() d[tt].add(t) # calc average factor for damage of this type relative to all types t.rate = (size + ((EFFECTIVENESS_FACTOR-1) * len(t.attack_effective_against)) - ((1-RESISTANCE_FACTOR) * len(t.attack_weak_against))) / size # set pokemon type resistance/weakness info for t in ret.itervalues(): # type: Type t.pokemon_resistant_to = by_resistance[t] t.pokemon_vulnerable_to = by_effectiveness[t] return ret
class ResourceLoader(object): def __init__(self): self.resources = OrderedDict() def require(self, name): if name in self.resources: return resource = resources.get(name) if not resource: raise LoaderError("There is no resource by the name %s." % name) for requirement in resource.requirements: self.require(requirement) self.resources[resource.name] = resource @property def styles(self): s = HtmlList() for resource in self.resources.itervalues(): for style in resource.styles: s.append(Style(**style)) return s @property def scripts(self): s = HtmlList() for resource in self.resources.itervalues(): for script in resource.scripts: s.append(Script(**script)) return s
def add_leveled_toc_items(self): added = OrderedDict() added2 = OrderedDict() counter = 1 def find_matches(expr, doc): try: ans = XPath(expr)(doc) len(ans) return ans except: self.log.warn("Invalid ToC expression, ignoring: %s" % expr) return [] for document in self.oeb.spine: previous_level1 = list(added.itervalues())[-1] if added else None previous_level2 = list(added2.itervalues())[-1] if added2 else None for elem in find_matches(self.opts.level1_toc, document.data): text, _href = self.elem_to_link(document, elem, counter) counter += 1 if text: node = self.oeb.toc.add(text, _href, play_order=self.oeb.toc.next_play_order()) added[elem] = node # node.add(_('Top'), _href) if self.opts.level2_toc is not None and added: for elem in find_matches(self.opts.level2_toc, document.data): level1 = None for item in document.data.iterdescendants(): if item in added: level1 = added[item] elif item == elem: if level1 is None: if previous_level1 is None: break level1 = previous_level1 text, _href = self.elem_to_link(document, elem, counter) counter += 1 if text: added2[elem] = level1.add(text, _href, play_order=self.oeb.toc.next_play_order()) break if self.opts.level3_toc is not None and added2: for elem in find_matches(self.opts.level3_toc, document.data): level2 = None for item in document.data.iterdescendants(): if item in added2: level2 = added2[item] elif item == elem: if level2 is None: if previous_level2 is None: break level2 = previous_level2 text, _href = self.elem_to_link(document, elem, counter) counter += 1 if text: level2.add(text, _href, play_order=self.oeb.toc.next_play_order()) break
class View(object): """ Presents a view of the set of `Option` arguments to a search command TODO: Description """ def __init__(self, command): self._items = OrderedDict([ (option.name, Option.Item(command, option)) for member_name, option in type(command).option_definitions]) return def __contains__(self, name): return name in self._items def __getitem__(self, name): return self._items[name] def __iter__(self): return self._items.__iter__() def __len__(self): return len(self._items) def __repr__(self): text = ''.join([ 'Option.View(', ','.join([repr(item) for item in self.itervalues()]), ')']) return text def __str__(self): text = ' '.join( [str(item) for item in self.itervalues() if item.is_set]) return text #region Methods def get_missing(self): missing = [ item.name for item in self._items.itervalues() if item.is_required and not item.is_set] return missing if len(missing) > 0 else None def iteritems(self): return self._items.iteritems() def iterkeys(self): return self.__iter__() def itervalues(self): return self._items.itervalues() def reset(self): for value in self.itervalues(): value.reset() return
class ForgetfulStorage(object): def __init__(self, ttl=604800): """ Implements a storage class for distributed log messages. Default max age is a week. """ self.data = OrderedDict() self.ttl = ttl def __setitem__(self, key, value): # if key in self.data: # self.data[key] = (time.time(),value) self.data[key] = (time.time(), value) self.cull() def __getitem__(self, key): self.cull() return self.data[key][1] def __iter__(self): self.cull() return iter(self.data) def __repr__(self): self.cull() return repr(self.data) def get(self, key, default=None): self.cull() if key in self.data: return self[key] return default def cull(self): """ Note that it may be useful to track what we evict. """ for k, v in self.iteritems_older_than(self.ttl): self.data.popitem(last=False) def iteritems_older_than(self, seconds_old): min_birthday = time.time() - seconds_old zipped = self._triple_iterable() matches = takewhile(lambda r: min_birthday >= r[1], zipped) return imap(operator.itemgetter(0, 2), matches) def _triple_iterable(self): ikeys = self.data.iterkeys() ibirthday = imap(operator.itemgetter(0), self.data.itervalues()) ivalues = imap(operator.itemgetter(1), self.data.itervalues()) return izip(ikeys, ibirthday, ivalues) def iteritems(self): self.cull() ikeys = self.data.iterkeys() ivalues = imap(operator.itemgetter(1), self.data.itervalues()) return izip(ikeys, ivalues)
def get(self): archive = memcache.get('archive') if archive is not None: self.response.out.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps({'articles':archive})) else: more = True archive = OrderedDict() curs = None while more: articles, curs, more = Article.query().order( -Article.when).fetch_page( 50, start_cursor=curs, projection=[Article.title, Article.when, Article.keywords]) for article in articles: article_key = article.key.urlsafe() if article_key in archive: archive[article_key]['keywords'].extend( article.keywords) else: archive[article_key] = { 'title': article.title, 'when': article.when.strftime("%d/%m/%Y %H:%M:%S"), 'keywords': article.keywords, 'key': article_key, 'ncomments': 0} more = True curs = None while more: page, curs, more = Comment.query().fetch_page( 100, start_cursor = curs) for comment in page: parent_key = comment.key.parent().urlsafe() if parent_key in archive: archive[parent_key]['ncomments'] += 1 memcache.add('archive', [a for a in archive.itervalues()], time=86400) self.response.out.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps( {'articles':[a for a in archive.itervalues()]} ))
def add_leveled_toc_items(self): added = OrderedDict() added2 = OrderedDict() counter = 1 for document in self.oeb.spine: previous_level1 = list(added.itervalues())[-1] if added else None previous_level2 = list(added2.itervalues())[-1] if added2 else None for elem in XPath(self.opts.level1_toc)(document.data): text, _href = self.elem_to_link(document, elem, counter) counter += 1 if text: node = self.oeb.toc.add(text, _href, play_order=self.oeb.toc.next_play_order()) added[elem] = node #node.add(_('Top'), _href) if self.opts.level2_toc is not None and added: for elem in XPath(self.opts.level2_toc)(document.data): level1 = None for item in document.data.iterdescendants(): if item in added: level1 = added[item] elif item == elem: if level1 is None: if previous_level1 is None: break level1 = previous_level1 text, _href = self.elem_to_link(document, elem, counter) counter += 1 if text: added2[elem] = level1.add(text, _href, play_order=self.oeb.toc.next_play_order()) break if self.opts.level3_toc is not None and added2: for elem in XPath(self.opts.level3_toc)(document.data): level2 = None for item in document.data.iterdescendants(): if item in added2: level2 = added2[item] elif item == elem: if level2 is None: if previous_level2 is None: break level2 = previous_level2 text, _href = \ self.elem_to_link(document, elem, counter) counter += 1 if text: level2.add(text, _href, play_order=self.oeb.toc.next_play_order()) break
class Atributos(OrderedDictBase): """Atributos da classe.""" def __init__(self, xmlclasse=None, data=None, class_associations=None): if xmlclasse is not None: self.__atributos = OrderedDict() xmlatributos = xmlclasse.iterdescendants(tag="Attribute") if xmlatributos is not None: for xmlatributo in xmlatributos: # Atribuição dos parâmetros de construção do objeto classe. xml_attributes = xmlatributo.attrib tagged_values = TaggedValues(xmlatributo) stereotypes = Stereotypes(xmlatributo) # Contrução do atributo e inclusão na lista de atributos de classe. if stereotypes.find('name', 'association_attribute') is not None: atributo = AssociationAttribute(xml_attributes, tagged_values, stereotypes=stereotypes) else: atributo = Attribute(xml_attributes, tagged_values, stereotypes=stereotypes) self.__atributos[atributo.name] = atributo else: logger.debug(u'Nenhum atributo encontrado.') # Cria os atributos associativos. if class_associations is not None: for association in class_associations: attribute = AssociationAttribute(association.xml_attributes, association.tagged_values) self.__atributos[attribute.name] = attribute elif data is not None: self.__atributos = data else: self.__atributos = OrderedDict() # Instancia a classe superior. super(Atributos, self).__init__(self.__atributos, Atributos) def __str__(self): return_msg = '' for attr in self.__atributos.itervalues(): return_msg += '\n %s' % str(attr) return return_msg @property def association_attributes(self): """Lista de classes principais.""" association_attributes = OrderedDict() for attribute in self.__atributos.itervalues(): if attribute.is_association_attribute: association_attributes[attribute.id] = attribute return Atributos(data=association_attributes)
class Region(object): """ Holds a region's layers along the blade. :param layers: Dictionary of Layer3D objects :type layers: dict """ def __init__(self): self.layers = OrderedDict() def add_layer(self, name): ''' Inserts a layer into layers dict. :param name: Name of the material :return: The layer added to the region ''' dubl = 0 for k in self.layers.iterkeys(): if name == k[:-2]: dubl += 1 lname = '%s%02d' % (name, dubl) layer = Layer() self.layers[lname] = layer return layer def init_stack(self): ''' Initiates thickness and angle matrices of the region plus the max thickness of each layer :return: thick_matrix: 2d np.array containing thicknesses of all layers (size: no_layers,s) :return: thick_max: 1d np.array max thickness per layer :return: angle_matrix: 2d np.array containing angles of all layers (size: no_layers,s) ''' thickmatdata = [] for v in self.layers.itervalues(): thickmatdata.append(v.thickness) self.thick_matrix = np.fliplr(np.rot90(np.array(thickmatdata), -1)) thickmaxdata = [] for l in range(self.thick_matrix.shape[1]): thickmaxdata.append(np.max(self.thick_matrix[:, l])) self.thick_max = np.array(thickmaxdata) anglematdata = [] for v in self.layers.itervalues(): anglematdata.append(v.angle) self.angle_matrix = np.fliplr(np.rot90(np.array(anglematdata), -1)) return self.thick_matrix, self.thick_max, self.angle_matrix
def _setupnames(self, names, vlst, sort=None, **dims): """Sets up the names dictionary for the plot""" if isinstance(names, dict): return OrderedDict(names.items()) if isinstance(names, (str, unicode)): names = [names] if isinstance(vlst, (str, unicode)): vlst = [vlst] iter_dims = OrderedDict() for key, val in sorted(dims.items()): # try if iterable try: iter(val) iter_dims[key] = dims.pop(key) except TypeError: pass if vlst is None: raise ValueError( "vlst must not be None if names is not a dictionary!") if sort is None: zipped_keys = ['var'] + iter_dims.keys() zipped_dims = product(vlst, *iter_dims.itervalues()) else: zipped_keys = list(sort) if not set(zipped_keys) == set(iter_dims.keys() + ['var']): raise ValueError( "Sorting parameter (%s) differs from iterable " "dimensions (%s)!" % ( ', '.join(zipped_keys), ', '.join(iter_dims.keys() + ['var']))) zipped_dims = product(*[ vlst if key == 'var' else iter_dims[key] for key in sort]) if names is None: names = OrderedDict([ ('line%i' % i, { dim: val for dim, val in zip(zipped_keys, dimstuple)}) for i, dimstuple in enumerate(zipped_dims)]) else: # assume a list of strings names = OrderedDict([ (str(name).format(i), { dim: val for dim, val in zip(zipped_keys, dimstuple)}) for i, (name, dimstuple) in enumerate(izip(cycle(names), zipped_dims))]) # update for non-iterable dimensions for settings in names.itervalues(): for dim, val in dims.items(): settings[dim] = val return names
def _make_raw_brackets_from_divisions(self): """Returns an OrderedDict mapping bracket names (normally numbers) to lists.""" brackets = OrderedDict() teams = list(self.teams) for team in teams: # Converting from bracket's name to a float (so it can pretend to be a Bracket) division = float(team.division.name) if division in brackets: brackets[division].append(team) else: brackets[division] = [team] print "------" # Assigning bye teams as needed for bracket in brackets.itervalues(): if len(bracket) % 2 != 0: from debate.models import Institution,Team bye_tournament = bracket[0].tournament bye_institution, created = Institution.objects.get_or_create( name="Byes" ) bye_reference = "Bye %s" % bracket[0].division bye_division = bracket[0].division bye_team = Team( institution = bye_institution, reference = bye_reference, short_reference = "Bye", tournament= bye_tournament, type = "B", use_institution_prefix = False, division = bye_division, cannot_break = True ) bye_team.aff_count = 0 bye_team.neg_count = 0 bye_team.save() bracket.append(bye_team) print "\t Created a bye team for divison %s" % bracket[0].division # Assigning subranks - fixed based on alphabetical for bracket in brackets.itervalues(): bracket.sort(key=lambda x: x.short_name, reverse=False) for i, team in enumerate(bracket): i += 1 team.subrank = i return brackets
def Run(self, runner): """Iterates over several runs and handles the output.""" traces = OrderedDict() for stdout in runner(): for line in stdout.strip().splitlines(): match = GENERIC_RESULTS_RE.match(line) if match: stddev = "" graph = match.group(1) trace = match.group(2) body = match.group(3) units = match.group(4) match_stddev = RESULT_STDDEV_RE.match(body) match_list = RESULT_LIST_RE.match(body) if match_stddev: result, stddev = map(str.strip, match_stddev.group(1).split(",")) results = [result] elif match_list: results = map(str.strip, match_list.group(1).split(",")) else: results = [body.strip()] trace_result = traces.setdefault(trace, Results([{ "graphs": self.graphs + [graph, trace], "units": (units or self.units).strip(), "results": [], "stddev": "", }], [])) trace_result.traces[0]["results"].extend(results) trace_result.traces[0]["stddev"] = stddev return reduce(lambda r, t: r + t, traces.itervalues(), Results())
def uniquify_points_and_return_input_index_to_unique_index_map( pts, threshold = 0 ): ''' Given a sequence of N points 'pts', and an optional 'threshold' indicating how many decimal places of accuracy (default: 0) returns two items: a sequence of all the unique elements in 'pts' and a list of length N where the i-th item in the list tells you where pts[i] can be found in the unique elements. ''' from collections import OrderedDict unique_pts = OrderedDict() pts_map = [] ## Add rounded points to a dictionary and set the key to ## ( the index into the ordered dictionary, the non-rounded point ) for i, ( pt, rounded_pt ) in enumerate( zip( pts, map( tuple, asarray( pts ).round( threshold ) ) ) ): index = unique_pts.setdefault( rounded_pt, ( len( unique_pts ), pt ) )[0] ## For fancier schemes: # index = unique_pts.setdefault( rounded_pt, ( len( unique_pts ), [] ) )[0] # unique_pts[ rounded_pt ][1].append( pt ) pts_map.append( index ) ## Return the original resolution points. ## The average of all points that round: # return [ tuple( average( pt, axis = 0 ) ) for i, pt in unique_pts.itervalues() ], pts_map ## The closest point to the rounded point: # return [ tuple( pt[ abs( asarray( pt ).round( threshold ) - asarray( pt ) ).sum(axis=1).argmin() ] ) for i, pt in unique_pts.itervalues() ], pts_map ## Simplest, the first rounded point: return [ tuple( pt ) for i, pt in unique_pts.itervalues() ], pts_map
def find_desktop_files(dirs=DESKTOP_DIRS): """ :param list dirs: :rtype: list """ all_files = chain.from_iterable( map(lambda f: os.path.join(f_path, f), find_files(f_path, '*.desktop')) for f_path in dirs) # dedup desktop file according to folow XDG data dir order # specifically the first file name (i.e. firefox.desktop) take precedence # and other files with the same name shoudl be ignored deduped_file_dict = OrderedDict() for file_path in all_files: file_name = os.path.basename(file_path) if file_name not in deduped_file_dict: deduped_file_dict[file_name] = file_path deduped_files = deduped_file_dict.itervalues() blacklisted_dirs_srt = Settings.get_instance().get_property('blacklisted-desktop-dirs') blacklisted_dirs = blacklisted_dirs_srt.split(':') if blacklisted_dirs_srt else [] for file in deduped_files: try: if any([force_unicode(file).startswith(dir) for dir in blacklisted_dirs]): continue except UnicodeDecodeError: continue yield file
class FieldSet(object): def __init__(self, delegate, schema_type): from pygtkhelpers.proxy import ProxyGroup, proxy_for self.delegate = delegate self.schema = schema_type() self.proxies = ProxyGroup() self.fields = OrderedDict() self.proxies.connect('changed', self._on_proxies_changed) for name, element in self.schema.items(): self._setup_widget(name, element) def _setup_widget(self, name, element): widget = getattr(self.delegate, name, None) #XXX (AA) this will always be the case, we are running too soon if widget is None: widget = widget_for(element) setattr(self.delegate, name, widget) field = self.fields[name] = Field(element, widget=widget) field_name = element.properties.get('label', name) field.set_label(field_name.capitalize()) self.proxies.add_proxy(name, field.proxy) def _on_proxies_changed(self, group, proxy, name, value): self.schema[name].set(value) def layout_as_table(self): # XXX: turn to utility function table = gtk.Table(len(self.fields), 2) table.set_row_spacings(6) table.set_col_spacings(6) table.set_border_width(6) for i, field_i in enumerate(self.fields.itervalues()): field_i.layout_as_table(table, i) return table
def smallest_subarray_covering_set(A, Q): """Find smallest range (i,j) s.t. all q in Q are in A[i..j].""" # handle 0-length covering if not Q: return 0, -1 # start with no best and an empty candidate covering range Q = set(Q) # want O(1) membership test min_size = None min_covering_range = None cand_locs = OrderedDict() # stream elements of A, maintaining the shortest covering range # that ends at the current element for j, elem in enumerate(A): # skip elements that can't contribute to a covering if elem not in Q: continue # extend the candidate range with the current elem, # removing any previous instance of the same elem cand_locs.pop(elem, None) cand_locs[elem] = j # will be added in final position # if the new candidate is legal and better, make it the current best if len(cand_locs) == len(Q): i = cand_locs.itervalues().next() # get front position, O(1) time if min_size is None or j - i < min_size: min_size = j - i min_covering_range = i, j return min_covering_range
def overwrite_asdc_authors(string_authors, asdc_auth): """ takes the authors string and update regarding what the admins decide param: take a long string of authors authors to underline return: string with updated information for asdc author """ repls = {'<b>': '', '</b>': '', '...': ''} string_clean = reduce(lambda a, kv: a.replace(*kv), repls.iteritems(), string_authors) author_list = string_clean.split(";") asdc_auth = OrderedDict((auth.split("_")[0], auth.split("_")[1]) for auth in asdc_auth) auth_dict = OrderedDict((auth.split(",")[0].lower().strip().replace(" ", ""), auth) for auth in author_list) append_last_element = False if next(reversed(auth_dict)).strip() == 'et al.': auth_dict.popitem() append_last_element = True for auth in asdc_auth.keys(): auth_dict[auth] = '<b> %s </b>' % (asdc_auth[auth].replace("%2C", ", ")).replace("+", " ") if append_last_element: auth_dict.update({' et al.': ' et al.'}) return ';'.join(auth_dict.itervalues())
class MemoryTypeAttributes(MemoryBase, Mapping): def __init__(self, owner): self._owner = owner # NOTE: OrderedDict used to make debug more convenient self._items = OrderedDict() # {} owner = roproperty("_owner") def on_complete(self, item): self._items[item.name] = item def new_sketch(self, restore=False): return MemoryTypeAttributeSketch(self) # unsafe def compose(self, ident=u"", file=None): if self.__dict__.get("_items"): file.write(u"%s<Attributes>\n" % ident) for attribute in self._items.itervalues(): attribute.compose(ident=ident + u"\t", file=file) file.write(u"%s</Attributes>\n" % ident) def __getitem__(self, name): return self._items[name] def __iter__(self): return iter(self._items) def __len__(self): return len(self._items) def __str__(self): return "attributes of %s" % self._owner
def box_data(keys, data=None): """ Box Data into appropiate pandas object Parameters ---------- keys : dict or iterable Either keys list or Result Dict data : iterable list of data """ if isinstance(keys, dict): rdict = keys else: rdict = OrderedDict(zip(keys, data)) test = next(rdict.itervalues()) if np.isscalar(test): return Series(rdict) if isinstance(test, Series): return DataFrame(rdict) if isinstance(test, DataFrame): return Panel(rdict) if isinstance(test, ColumnPanel): data = OrderedDict([(k, v.to_panel()) for k, v in rdict.iteritems()]) return Panel4D(data) if isinstance(test, Panel): return Panel4D(data) return rdict
class DocumentLengthTable: def __init__(self): self.table = OrderedDict() def __len__(self): return len(self.table) def write(self, filename='default.dlt'): with open(filename, 'w') as f: for docid in self.table: line = '%s %d\n' % (docid, self.table[docid]) f.write(line) def add(self, docid, length): self.table[docid] = length def get_length(self, docid): if docid in self.table: return self.table[docid] else: raise LookupError('%s not found in table' % str(docid)) def get_average_length(self): sum = 0 for length in self.table.itervalues(): sum += length return float(sum) / float(len(self.table))
class SVDPeripheralRegister: def __init__(self, svd_elem, parent): self.parent = parent self.name = str(svd_elem.name) self.description = str(svd_elem.description) self.offset = int(str(svd_elem.addressOffset),0) self.size = int(str(svd_elem.size),0) fields = svd_elem.fields.getchildren() self.fields = OrderedDict() for f in fields: self.fields[str(f.name)] = SVDPeripheralRegisterField(f, self) def refactor_parent(self, parent): self.parent = parent try: fields = self.fields.itervalues() except AttributeError: fields = self.fields.values() for f in fields: f.refactor_parent(self) def address(self): return self.parent.base_address + self.offset def __unicode__(self): return str(self.name)
def check_for_add_to_toolbars(self, plugin): from calibre.gui2.preferences.toolbar import ConfigWidget from calibre.customize import InterfaceActionBase if not isinstance(plugin, InterfaceActionBase): return all_locations = OrderedDict(ConfigWidget.LOCATIONS) plugin_action = plugin.load_actual_plugin(self.gui) installed_actions = OrderedDict([ (key, list(gprefs.get('action-layout-'+key, []))) for key in all_locations]) # If already installed in a GUI container, do nothing for action_names in installed_actions.itervalues(): if plugin_action.name in action_names: return allowed_locations = [(key, text) for key, text in all_locations.iteritems() if key not in plugin_action.dont_add_to] if not allowed_locations: return # This plugin doesn't want to live in the GUI from calibre.gui2.dialogs.choose_plugin_toolbars import ChoosePluginToolbarsDialog d = ChoosePluginToolbarsDialog(self, plugin_action, allowed_locations) if d.exec_() == d.Accepted: for key, text in d.selected_locations(): installed_actions = list(gprefs.get('action-layout-'+key, [])) installed_actions.append(plugin_action.name) gprefs['action-layout-'+key] = tuple(installed_actions)
def tx_modes_plot(consensus_data, ordered_genomes, tx_mode_plot_tgt): ordered_groups = ['transMap', 'transMap+TM', 'transMap+TMR', 'transMap+TM+TMR', 'TM', 'TMR', 'TM+TMR', 'CGP', 'PB', 'Other'] ordered_groups = OrderedDict([[frozenset(x.split('+')), x] for x in ordered_groups]) def split_fn(s): return ordered_groups.get(frozenset(s['Transcript Modes'].replace('aug', '').split(',')), 'Other') modes_df = json_biotype_counter_to_df(consensus_data, 'Transcript Modes') df = modes_df.pivot(index='genome', columns='Transcript Modes').transpose().reset_index() df['Modes'] = df.apply(split_fn, axis=1) df = df[['Modes'] + ordered_genomes] ordered_values = [x for x in ordered_groups.itervalues() if x in set(df['Modes'])] with tx_mode_plot_tgt.open('w') as outf, PdfPages(outf) as pdf: title_string = 'Transcript modes in protein coding consensus gene set' ylabel = 'Number of transcripts' if len(ordered_genomes) > 1: df['Ordered Modes'] = pd.Categorical(df['Modes'], ordered_values, ordered=True) df = df.sort_values('Ordered Modes') df = df[['Ordered Modes'] + ordered_genomes].set_index('Ordered Modes') df = df.fillna(0) generic_stacked_barplot(df, pdf, title_string, df.index, ylabel, ordered_genomes, 'Transcript mode(s)', bbox_to_anchor=(1.25, 0.7)) else: generic_barplot(pd.melt(df, id_vars='Modes'), pdf, 'Transcript mode(s)', ylabel, title_string, x='Modes', y='value', order=ordered_values)
class TreeItem(ItemModel): def __init__(self, data, parent=None): ItemModel.__init__(self, data, parent) self.children = OrderedDictionary() def remove_child(self, key_value): if key_value not in self.children: raise KeyError('No such child exists') del self.children[key_value] def row(self): """ This method is necessary because of the parent-child node structure of the model, where there is no simple way to find the overall relationship of all the items in the database, rather just one items' relationship with those surrounding it. :return: int """ if not self.parent: return 0 return self.parent.children.values().index(self) def __iter__(self): return self.children.itervalues()
class InterfaceSection (object): def __init__ (self, properties = None, traces = None, controls = None, title = "", name = None): self.name = name or title self.title = title self.properties = properties or [] self.controls = controls or [] self.traces = OrderedDict() if traces is not None: for t in traces: try: self.traces[t["name"]] = t except KeyError: self.traces[t["title"]] = t def __getitem__ (self, key): if key in ("name", "title", "properties", "controls", "traces"): return getattr(self, key) else: raise KeyError def output (self): return { "name": self.name, "title": self.title, "traces": [_trace(t) for t in self.traces.itervalues()], "properties": [_prop(p) for p in self.properties if p.type not in ("Image",)], "images": [_img(p) for p in self.properties if p.type == "Image"], "controls": [_control(c) for c in self.controls] }
def getDeviceShortNames(self, devicepaths, pciids): shortnames = OrderedDict() namecount = {} try: # Initialise PciIdsParser, throws customExceptions.PciIdsFileNotFound # if fail pciIdsParser = parseutil.PciIdsParser(pciids) except customExceptions.PciIdsFileNotFound: message.addError("pci.ids file could not be located in tool " "directory: %s. " % paths.CURRENTDIR + "Device " "names could not be obtained. Please ensure that " "the file is in the directory.", False) else: for devicepath in devicepaths: # Add entry to dictionary shortnames shortnames[devicepath] = self.getClassName( devicepath, pciIdsParser) namelist = [] for value in shortnames.itervalues(): namelist.append(value) listnumberer = util.ListNumberer(namelist) for devicepath in shortnames.iterkeys(): shortnames[devicepath] = listnumberer.getName( shortnames[devicepath]) return shortnames
class DictVal(Collapsible): def __init__(self, expanded=True): super(Dict, self).__init__(expanded) self.items = OrderedDict() def render_items(self): r = [] for key, item in self.items.iteritems(): r += [t(key), t(":"), indent(), nl()] r += [ElementTag(item)] r += [dedent(), nl()] return r def __getitem__(self, i): return self.items[i] def __setitem__(self, i, v): self.items[i] = v v.parent = self def fix_parents(self): super(Dict, self).fix_parents() self._fix_parents(self.items.values()) def flatten(self): return [self] + [v.flatten() for v in self.items.itervalues() if isinstance(v, Node)] #skip Widgets, as in settings def add(self, (key, val)): assert(not self.items.has_key(key)) self.items[key] = val assert(isinstance(key, str)) assert(isinstance(val, element.Element)) val.parent = self
class Environment(object): """Environment within which all agents operate.""" valid_actions = [None, 'forward', 'left', 'right'] valid_inputs = { 'light': TrafficLight.valid_states, 'oncoming': valid_actions, 'left': valid_actions, 'right': valid_actions } valid_headings = [(1, 0), (0, -1), (-1, 0), (0, 1)] # E, N, W, S hard_time_limit = -100 # Set a hard time limit even if deadline is not enforced. def __init__(self, verbose=False, num_dummies=100, grid_size=(8, 6)): self.num_dummies = num_dummies # Number of dummy driver agents in the environment self.verbose = verbose # If debug output should be given # Initialize simulation variables self.done = False self.t = 0 self.agent_states = OrderedDict() self.step_data = {} self.success = None # Road network self.grid_size = grid_size # (columns, rows) self.bounds = (1, 2, self.grid_size[0], self.grid_size[1] + 1) self.block_size = 100 self.hang = 0.6 self.intersections = OrderedDict() self.roads = [] for x in xrange(self.bounds[0], self.bounds[2] + 1): for y in xrange(self.bounds[1], self.bounds[3] + 1): # A traffic light at each intersection self.intersections[(x, y)] = TrafficLight() for a in self.intersections: for b in self.intersections: if a == b: continue if (abs(a[0] - b[0]) + abs(a[1] - b[1])) == 1: # L1 distance = 1 self.roads.append((a, b)) # Add environment boundaries for x in xrange(self.bounds[0], self.bounds[2] + 1): self.roads.append( ((x, self.bounds[1] - self.hang), (x, self.bounds[1]))) self.roads.append( ((x, self.bounds[3] + self.hang), (x, self.bounds[3]))) for y in xrange(self.bounds[1], self.bounds[3] + 1): self.roads.append( ((self.bounds[0] - self.hang, y), (self.bounds[0], y))) self.roads.append( ((self.bounds[2] + self.hang, y), (self.bounds[2], y))) # Create dummy agents for i in xrange(self.num_dummies): self.create_agent(DummyAgent) # Primary agent and associated parameters self.primary_agent = None # to be set explicitly self.enforce_deadline = False # Trial data (updated at the end of each trial) self.trial_data = { 'testing': False, # if the trial is for testing a learned policy 'initial_distance': 0, # L1 distance from start to destination 'initial_deadline': 0, # given deadline (time steps) to start with 'net_reward': 0.0, # total reward earned in current trial 'final_deadline': None, # deadline value (time remaining) at the end 'actions': { 0: 0, 1: 0, 2: 0, 3: 0, 4: 0 }, # violations and accidents 'success': 0 # whether the agent reached the destination in time } def create_agent(self, agent_class, *args, **kwargs): """ When called, create_agent creates an agent in the environment. """ agent = agent_class(self, *args, **kwargs) self.agent_states[agent] = { 'location': random.choice(self.intersections.keys()), 'heading': (0, 1) } return agent def set_primary_agent(self, agent, enforce_deadline=False): """ When called, set_primary_agent sets 'agent' as the primary agent. The primary agent is the smartcab that is followed in the environment. """ self.primary_agent = agent agent.primary_agent = True self.enforce_deadline = enforce_deadline def reset(self, testing=False): """ This function is called at the beginning of a new trial. """ self.done = False self.t = 0 # Reset status text self.step_data = {} # Reset traffic lights for traffic_light in self.intersections.itervalues(): traffic_light.reset() # Pick a start and a destination start = random.choice(self.intersections.keys()) destination = random.choice(self.intersections.keys()) # Ensure starting location and destination are not too close while self.compute_dist(start, destination) < 4: start = random.choice(self.intersections.keys()) destination = random.choice(self.intersections.keys()) start_heading = random.choice(self.valid_headings) distance = self.compute_dist(start, destination) deadline = distance * 5 # 5 time steps per intersection away if (self.verbose == True): # Debugging print "Environment.reset(): Trial set up with start = {}, destination = {}, deadline = {}".format( start, destination, deadline) # Create a map of all possible initial positions positions = dict() for location in self.intersections: positions[location] = list() for heading in self.valid_headings: positions[location].append(heading) # Initialize agent(s) for agent in self.agent_states.iterkeys(): if agent is self.primary_agent: self.agent_states[agent] = { 'location': start, 'heading': start_heading, 'destination': destination, 'deadline': deadline } # For dummy agents, make them choose one of the available # intersections and headings still in 'positions' else: intersection = random.choice(positions.keys()) heading = random.choice(positions[intersection]) self.agent_states[agent] = { 'location': intersection, 'heading': heading, 'destination': None, 'deadline': None } # Now delete the taken location and heading from 'positions' positions[intersection] = list( set(positions[intersection]) - {heading}) if positions[intersection] == list( ): # No headings available for intersection del positions[ intersection] # Delete the intersection altogether agent.reset(destination=(destination if agent is self.primary_agent else None), testing=testing) if agent is self.primary_agent: # Reset metrics for this trial (step data will be set during the step) self.trial_data['testing'] = testing self.trial_data['initial_deadline'] = deadline self.trial_data['final_deadline'] = deadline self.trial_data['net_reward'] = 0.0 self.trial_data['actions'] = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0} self.trial_data['parameters'] = { 'e': agent.epsilon, 'a': agent.alpha } self.trial_data['success'] = 0 def step(self): """ This function is called when a time step is taken turing a trial. """ # Pretty print to terminal print "" print "/-------------------" print "| Step {} Results".format(self.t) print "\-------------------" print "" if (self.verbose == True): # Debugging print "Environment.step(): t = {}".format(self.t) # Update agents, primary first if self.primary_agent is not None: self.primary_agent.update() for agent in self.agent_states.iterkeys(): if agent is not self.primary_agent: agent.update() # Update traffic lights for intersection, traffic_light in self.intersections.iteritems(): traffic_light.update(self.t) if self.primary_agent is not None: # Agent has taken an action: reduce the deadline by 1 agent_deadline = self.agent_states[ self.primary_agent]['deadline'] - 1 self.agent_states[self.primary_agent]['deadline'] = agent_deadline if agent_deadline <= self.hard_time_limit: self.done = True self.success = False if self.verbose: # Debugging print "Environment.step(): Primary agent hit hard time limit ({})! Trial aborted.".format( self.hard_time_limit) elif self.enforce_deadline and agent_deadline <= 0: self.done = True self.success = False if self.verbose: # Debugging print "Environment.step(): Primary agent ran out of time! Trial aborted." self.t += 1 def sense(self, agent): """ This function is called when information is requested about the sensor inputs from an 'agent' in the environment. """ assert agent in self.agent_states, "Unknown agent!" state = self.agent_states[agent] location = state['location'] heading = state['heading'] if (self.intersections[location].state and heading[1] != 0) or ( (not self.intersections[location].state) and heading[0] != 0): light = 'green' else: light = 'red' # Populate oncoming, left, right oncoming = None left = None right = None for other_agent, other_state in self.agent_states.iteritems(): if agent == other_agent or location != other_state['location'] or \ (heading[0] == other_state['heading'][0] and heading[1] == other_state['heading'][1]): continue # For dummy agents, ignore the primary agent # This is because the primary agent is not required to follow the waypoint if other_agent == self.primary_agent: continue other_heading = other_agent.get_next_waypoint() if (heading[0] * other_state['heading'][0] + heading[1] * other_state['heading'][1]) == -1: if oncoming != 'left': # we don't want to override oncoming == 'left' oncoming = other_heading elif (heading[1] == other_state['heading'][0] and -heading[0] == other_state['heading'][1]): if right != 'forward' and right != 'left': # we don't want to override right == 'forward or 'left' right = other_heading else: if left != 'forward': # we don't want to override left == 'forward' left = other_heading return { 'light': light, 'oncoming': oncoming, 'left': left, 'right': right } def get_deadline(self, agent): """ Returns the deadline remaining for an agent. """ return self.agent_states[agent][ 'deadline'] if agent is self.primary_agent else None def act(self, agent, action): """ Consider an action and perform the action if it is legal. Receive a reward for the agent based on traffic laws. """ assert agent in self.agent_states, "Unknown agent!" if action not in self.valid_actions: print("Illegal action: ") print(action) assert action in self.valid_actions, "Invalid action!" state = self.agent_states[agent] location = state['location'] heading = state['heading'] if (self.intersections[location].state and heading[1] != 0) or ( (not self.intersections[location].state) and heading[0] != 0): light = 'green' else: light = 'red' inputs = self.sense(agent) # Assess whether the agent can move based on the action chosen. # Either the action is okay to perform, or falls under 4 types of violations: # 0: Action okay # 1: Minor traffic violation # 2: Major traffic violation # 3: Minor traffic violation causing an accident # 4: Major traffic violation causing an accident violation = 0 # Reward scheme # First initialize reward uniformly random from [-1, 1] reward = 2 * random.random() - 1 # Create a penalty factor as a function of remaining deadline # Scales reward multiplicatively from [0, 1] fnc = self.t * 1.0 / ( self.t + state['deadline']) if agent.primary_agent else 0.0 gradient = 10 # No penalty given to an agent that has no enforced deadline penalty = 0 # If the deadline is enforced, give a penalty based on time remaining if self.enforce_deadline: penalty = (math.pow(gradient, fnc) - 1) / (gradient - 1) # Agent wants to drive forward: if action == 'forward': if light != 'green': # Running red light violation = 2 # Major violation if inputs['left'] == 'forward' or inputs[ 'right'] == 'forward': # Cross traffic violation = 4 # Accident # Agent wants to drive left: elif action == 'left': if light != 'green': # Running a red light violation = 2 # Major violation if inputs['left'] == 'forward' or inputs[ 'right'] == 'forward': # Cross traffic violation = 4 # Accident elif inputs[ 'oncoming'] == 'right': # Oncoming car turning right violation = 4 # Accident else: # Green light if inputs['oncoming'] == 'right' or inputs[ 'oncoming'] == 'forward': # Incoming traffic violation = 3 # Accident else: # Valid move! heading = (heading[1], -heading[0]) # Agent wants to drive right: elif action == 'right': if light != 'green' and inputs[ 'left'] == 'forward': # Cross traffic violation = 3 # Accident else: # Valid move! heading = (-heading[1], heading[0]) # Agent wants to perform no action: elif action == None: if light == 'green': violation = 1 # Minor violation # Did the agent attempt a valid move? if violation == 0: if action == agent.get_next_waypoint( ): # Was it the correct action? reward += 2 - penalty # (2, 1) elif action == None and light != 'green' and agent.get_next_waypoint( ) == 'right': # valid action but incorrect (idling at red light, when we should have gone right on red) reward += 1 - penalty # (1, 0) elif action == None and light != 'green': # Was the agent stuck at a red light? reward += 2 - penalty # (2, 1) else: # Valid but incorrect reward += 1 - penalty # (1, 0) # Move the agent if action is not None: location = ( (location[0] + heading[0] - self.bounds[0]) % (self.bounds[2] - self.bounds[0] + 1) + self.bounds[0], (location[1] + heading[1] - self.bounds[1]) % (self.bounds[3] - self.bounds[1] + 1) + self.bounds[1] ) # wrap-around state['location'] = location state['heading'] = heading # Agent attempted invalid move else: if violation == 1: # Minor violation reward += -5 elif violation == 2: # Major violation reward += -10 elif violation == 3: # Minor accident reward += -20 elif violation == 4: # Major accident reward += -40 # Did agent reach the goal after a valid move? if agent is self.primary_agent: if state['location'] == state['destination']: # Did agent get to destination before deadline? if state['deadline'] >= 0: self.trial_data['success'] = 1 # Stop the trial self.done = True self.success = True if (self.verbose == True): # Debugging print "Environment.act(): Primary agent has reached destination!" if (self.verbose == True): # Debugging print "Environment.act() [POST]: location: {}, heading: {}, action: {}, reward: {}".format( location, heading, action, reward) # Update metrics self.step_data['t'] = self.t self.step_data['violation'] = violation self.step_data['state'] = agent.get_state() self.step_data['deadline'] = state['deadline'] self.step_data['waypoint'] = agent.get_next_waypoint() self.step_data['inputs'] = inputs self.step_data['light'] = light self.step_data['action'] = action self.step_data['reward'] = reward self.trial_data['final_deadline'] = state['deadline'] - 1 self.trial_data['net_reward'] += reward self.trial_data['actions'][violation] += 1 if (self.verbose == True): # Debugging print "Environment.act(): Step data: {}".format(self.step_data) return reward def compute_dist(self, a, b): """ Compute the Manhattan (L1) distance of a spherical world. """ dx1 = abs(b[0] - a[0]) dx2 = abs(self.grid_size[0] - dx1) dx = dx1 if dx1 < dx2 else dx2 dy1 = abs(b[1] - a[1]) dy2 = abs(self.grid_size[1] - dy1) dy = dy1 if dy1 < dy2 else dy2 return dx + dy
class Environment(object): """Environment within which all agents operate.""" valid_actions = [None, 'forward', 'left', 'right'] valid_inputs = { 'light': TrafficLight.valid_states, 'oncoming': valid_actions, 'left': valid_actions, 'right': valid_actions } valid_headings = [(1, 0), (0, -1), (-1, 0), (0, 1)] # ENWS hard_time_limit = -100 # even if enforce_deadline is False, end trial when deadline reaches this value (to avoid deadlocks) def __init__(self): self.done = False self.results = [] self.t = 0 self.agent_states = OrderedDict() self.status_text = "" self.count = 1 # Road network self.grid_size = (8, 6) # (cols, rows) self.bounds = (1, 1, self.grid_size[0], self.grid_size[1]) self.block_size = 100 self.intersections = OrderedDict() self.roads = [] for x in xrange(self.bounds[0], self.bounds[2] + 1): for y in xrange(self.bounds[1], self.bounds[3] + 1): self.intersections[(x, y)] = TrafficLight( ) # a traffic light at each intersection for a in self.intersections: for b in self.intersections: if a == b: continue if (abs(a[0] - b[0]) + abs(a[1] - b[1])) == 1: # L1 distance = 1 self.roads.append((a, b)) # Dummy agents self.num_dummies = 3 # no. of dummy agents for i in xrange(self.num_dummies): self.create_agent(DummyAgent) # Primary agent self.primary_agent = None # to be set explicitly self.enforce_deadline = False def create_agent(self, agent_class, *args, **kwargs): agent = agent_class(self, *args, **kwargs) self.agent_states[agent] = { 'location': random.choice(self.intersections.keys()), 'heading': (0, 1) } return agent def set_primary_agent(self, agent, enforce_deadline=False): self.primary_agent = agent self.enforce_deadline = enforce_deadline def reset(self): self.done = False self.t = 0 # Reset traffic lights for traffic_light in self.intersections.itervalues(): traffic_light.reset() # Pick a start and a destination start = random.choice(self.intersections.keys()) destination = random.choice(self.intersections.keys()) # Ensure starting location and destination are not too close while self.compute_dist(start, destination) < 4: start = random.choice(self.intersections.keys()) destination = random.choice(self.intersections.keys()) start_heading = random.choice(self.valid_headings) deadline = self.compute_dist(start, destination) * 5 # print "Environment.reset(): Trial set up with start = {}, destination = {}, deadline = {}".format(start, destination, deadline) # Initialize agent(s) for agent in self.agent_states.iterkeys(): self.agent_states[agent] = { 'location': start if agent is self.primary_agent else random.choice( self.intersections.keys()), 'heading': start_heading if agent is self.primary_agent else random.choice(self.valid_headings), 'destination': destination if agent is self.primary_agent else None, 'deadline': deadline if agent is self.primary_agent else None } agent.reset(destination=( destination if agent is self.primary_agent else None)) def step(self): #print "Environment.step(): t = {}".format(self.t) # [debug] # Update traffic lights for intersection, traffic_light in self.intersections.iteritems(): traffic_light.update(self.t) # Update agents for agent in self.agent_states.iterkeys(): agent.update(self.t) self.t += 1 if self.primary_agent is not None and self.done is not True: agent_deadline = self.agent_states[self.primary_agent]['deadline'] if agent_deadline <= self.hard_time_limit: self.done = True self.results.append((0, self.primary_agent.score, self.primary_agent.penalties)) print "Environment.step(): Primary agent hit hard time limit ({})! Trial aborted.".format( self.hard_time_limit) elif self.enforce_deadline and agent_deadline <= 0: self.done = True # print "0" self.results.append((0, self.primary_agent.score, self.primary_agent.penalties)) print "Environment.step(): Primary agent ran out of time! Trial aborted." self.agent_states[ self.primary_agent]['deadline'] = agent_deadline - 1 def sense(self, agent): assert agent in self.agent_states, "Unknown agent!" state = self.agent_states[agent] location = state['location'] heading = state['heading'] light = 'green' if (self.intersections[location].state and heading[1] != 0) or ( (not self.intersections[location].state) and heading[0] != 0) else 'red' # Populate oncoming, left, right oncoming = None left = None right = None for other_agent, other_state in self.agent_states.iteritems(): if agent == other_agent or location != other_state['location'] or ( heading[0] == other_state['heading'][0] and heading[1] == other_state['heading'][1]): continue other_heading = other_agent.get_next_waypoint() if (heading[0] * other_state['heading'][0] + heading[1] * other_state['heading'][1]) == -1: if oncoming != 'left': # we don't want to override oncoming == 'left' oncoming = other_heading elif (heading[1] == other_state['heading'][0] and -heading[0] == other_state['heading'][1]): if right != 'forward' and right != 'left': # we don't want to override right == 'forward or 'left' right = other_heading else: if left != 'forward': # we don't want to override left == 'forward' left = other_heading return { 'light': light, 'oncoming': oncoming, 'left': left, 'right': right } # TODO: make this a namedtuple def get_deadline(self, agent): return self.agent_states[agent][ 'deadline'] if agent is self.primary_agent else None def act(self, agent, action): assert agent in self.agent_states, "Unknown agent!" assert action in self.valid_actions, "Invalid action!" state = self.agent_states[agent] location = state['location'] heading = state['heading'] light = 'green' if (self.intersections[location].state and heading[1] != 0) or ( (not self.intersections[location].state) and heading[0] != 0) else 'red' sense = self.sense(agent) # Move agent if within bounds and obeys traffic rules reward = 0 # reward/penalty move_okay = True if action == 'forward': if light != 'green': move_okay = False elif action == 'left': if light == 'green' and (sense['oncoming'] == None or sense['oncoming'] == 'left'): heading = (heading[1], -heading[0]) else: move_okay = False elif action == 'right': if light == 'green' or sense['left'] != 'straight': heading = (-heading[1], heading[0]) else: move_okay = False if move_okay: # Valid move (could be null) if action is not None: # Valid non-null move location = ( (location[0] + heading[0] - self.bounds[0]) % (self.bounds[2] - self.bounds[0] + 1) + self.bounds[0], (location[1] + heading[1] - self.bounds[1]) % (self.bounds[3] - self.bounds[1] + 1) + self.bounds[1] ) # wrap-around #if self.bounds[0] <= location[0] <= self.bounds[2] and self.bounds[1] <= location[1] <= self.bounds[3]: # bounded state['location'] = location state['heading'] = heading reward = 2.0 if action == agent.get_next_waypoint( ) else -0.5 # valid, but is it correct? (as per waypoint) else: # Valid null move reward = 0.0 else: # Invalid move reward = -1.0 if agent is self.primary_agent: if state['location'] == state['destination']: if state['deadline'] >= 0: reward += 10 # bonus self.done = True self.results.append((1, self.primary_agent.score, self.primary_agent.penalties)) print "Environment.act(): Primary agent has reached destination!" # [debug] self.status_text = "state: {}\naction: {}\nreward: {}".format( agent.get_state(), action, reward) # print "Environment.act() [POST]: location: {}, heading: {}, action: {}, reward: {}".format(location, heading, action, reward) # [debug] return reward def compute_dist(self, a, b): """L1 distance between two points.""" return abs(b[0] - a[0]) + abs(b[1] - a[1])
class ResponseHeaders(object): """Dictionary-like object holding the headers for the response""" def __init__(self): self.data = OrderedDict() def set(self, key, value): """Set a header to a specific value, overwriting any previous header with the same name :param key: Name of the header to set :param value: Value to set the header to """ self.data[key.lower()] = (key, [value]) def append(self, key, value): """Add a new header with a given name, not overwriting any existing headers with the same name :param key: Name of the header to add :param value: Value to set for the header """ if key.lower() in self.data: self.data[key.lower()][1].append(value) else: self.set(key, value) def get(self, key, default=missing): """Get the set values for a particular header.""" try: return self[key] except KeyError: if default is missing: return [] return default def __getitem__(self, key): """Get a list of values for a particular header """ return self.data[key.lower()][1] def __delitem__(self, key): del self.data[key.lower()] def __contains__(self, key): return key.lower() in self.data def __setitem__(self, key, value): self.set(key, value) def __iter__(self): for key, values in self.data.itervalues(): for value in values: yield key, value def items(self): return list(self) def update(self, items_iter): for name, value in items_iter: self.set(name, value) def __repr__(self): return repr(self.data)
class RecipeContainer(Observable, Configurable, Validatable): """Base class for organizing pieces of a FitRecipe. RecipeContainers are hierarchical organizations of Parameters and other RecipeContainers. This class provides attribute-access to these contained objects. Parameters and other RecipeContainers can be found within the hierarchy with the _locateManagedObject method. A RecipeContainer can manage dictionaries for that store various objects. These dictionaries can be added to the RecipeContainer using the _manage method. RecipeContainer methods that add, remove or retrieve objects will work with any managed dictionary. This makes it easy to add new types of objects to be contained by a RecipeContainer. By default, the RecipeContainer is configured to manage an OrderedDict of Parameter objects. RecipeContainer is an Observable, and observes its managed objects and Parameters. This allows hierarchical calculation elements, such as ProfileGenerator, to detect changes in Parameters and Restraints on which it may depend. Attributes name -- A name for this RecipeContainer. Names should be unique within a RecipeContainer and should be valid attribute names. _parameters -- A managed OrderedDict of contained Parameters. __managed -- A list of managed dictionaries. This is used for attribute access, addition and removal. _configobjs -- A set of configurable objects that must know of configuration changes within this object. Properties names -- Variable names (read only). See getNames. values -- Variable values (read only). See getValues. """ names = property(lambda self: self.getNames()) values = property(lambda self: self.getValues()) def __init__(self, name): Observable.__init__(self) Configurable.__init__(self) validateName(name) self.name = name self._parameters = OrderedDict() self.__managed = [] self._manage(self._parameters) return def _manage(self, d): """Manage a dictionary of objects. This adds the dictionary to the __managed list. Dictionaries in __managed are used for attribute access, addition, and removal. """ self.__managed.append(d) return def _iterManaged(self): """Get iterator over managed objects.""" return chain(*(d.values() for d in self.__managed)) def iterPars(self, pattern="", recurse=True): """Iterate over the Parameters contained in this object. Parameters ---------- pattern : str Iterate over parameters with names matching this regular expression (all parameters by default). recurse : bool Recurse into managed objects when True (default). """ regexp = re.compile(pattern) for par in list(self._parameters.values()): if regexp.search(par.name): yield par if not recurse: return # Iterate over objects within the managed dictionaries. managed = self.__managed[:] managed.remove(self._parameters) for m in managed: for obj in m.values(): if hasattr(obj, "iterPars"): for par in obj.iterPars(pattern=pattern): yield par return def __iter__(self): """Iterate over top-level parameters.""" return self._parameters.itervalues() def __len__(self): """Get number of top-level parameters.""" return len(self._parameters) def __getitem__(self, idx): """Get top-level parameters by index.""" return self._parameters.values()[idx] def __getattr__(self, name): """Gives access to the contained objects as attributes.""" arg = self.get(name) if arg is None: raise AttributeError(name) return arg # Ensure there is no __dir__ override in the base class. assert (getattr(Observable, '__dir__', None) is getattr( Configurable, '__dir__', None) is getattr(Validatable, '__dir__', None) is getattr(object, '__dir__', None)) def __dir__(self): "Return sorted list of attributes for this object." rv = set(dir(type(self))) rv.update(self.__dict__) # self.get fetches looks up for items in all managed dictionaries. # Add keys from each dictionary in self.__managed. rv.update(*self.__managed) rv = sorted(rv) return rv # Needed by __setattr__ _parameters = OrderedDict() __managed = [] def __setattr__(self, name, value): """Parameter access and object checking.""" if name in self._parameters: par = self._parameters[name] if isinstance(value, Parameter): par.value = value.value else: par.value = value return m = self.get(name) if m is not None: raise AttributeError("Cannot set '%s'" % name) super(RecipeContainer, self).__setattr__(name, value) return def __delattr__(self, name): """Delete parameters with del. This does not allow deletion of non-parameters, as this may require configuration changes that are not yet handled in a general way. """ if name in self._parameters: self._removeParameter(self._parameters[name]) return m = self.get(name) if m is not None: raise AttributeError("Cannot delete '%s'" % name) super(RecipeContainer, self).__delattr__(name) return def get(self, name, default=None): """Get a managed object.""" for d in self.__managed: arg = d.get(name) if arg is not None: return arg return default def getNames(self): """Get the names of managed parameters.""" return [p.name for p in self._parameters.values()] def getValues(self): """Get the values of managed parameters.""" return [p.value for p in self._parameters.values()] def _addObject(self, obj, d, check=True): """Add an object to a managed dictionary. obj -- The object to be stored. d -- The managed dictionary to store the object in. check -- If True (default), a ValueError is raised an object of the given name already exists. Raises ValueError if the object has no name. Raises ValueError if the object has the same name as some other managed object. """ # Check name if not obj.name: message = "%s has no name" % obj.__class__.__name__ raise ValueError(message) # Check for extant object in d with same name oldobj = d.get(obj.name) if check and oldobj is not None: message = "%s with name '%s' already exists"%\ (obj.__class__.__name__, obj.name) raise ValueError(message) # Check for object with same name in other dictionary. if oldobj is None and self.get(obj.name) is not None: message = "Non-%s with name '%s' already exists"%\ (obj.__class__.__name__, obj.name) raise ValueError(message) # Detach the old object, if there is one if oldobj is not None: oldobj.removeObserver(self._flush) # Add the object d[obj.name] = obj # Observe the object obj.addObserver(self._flush) # Store this as a configurable object self._storeConfigurable(obj) return def _removeObject(self, obj, d): """Remove an object from a managed dictionary. Raises ValueError if obj is not part of the dictionary. """ if obj not in d.values(): m = "'%s' is not part of the %s" % (obj, self.__class__.__name__) raise ValueError(m) del d[obj.name] obj.removeObserver(self._flush) return def _locateManagedObject(self, obj): """Find the location a managed object within the hierarchy. obj -- The object to find. Returns a list of objects. The first member of the list is this object, and each subsequent member is a sub-object of the previous one. The last entry in the list is obj. If obj cannot be found, the list is empty. """ loc = [self] # This handles the case that an object is asked to locate itself. if obj is self: return loc for m in self._iterManaged(): # Check locally for the object if m is obj: loc.append(obj) return loc # Check within managed objects if hasattr(m, "_locateManagedObject"): subloc = m._locateManagedObject(obj) if subloc: return loc + subloc return [] def _flush(self, other): """Invalidate cached state. This will force any observer to invalidate its state. By default this does nothing. """ self.notify(other) return def _validate(self): """Validate my state. This validates that contained Parameters and managed objects are valid. Raises AttributeError if validation fails. """ iterable = chain(self.__iter__(), self._iterManaged()) self._validateOthers(iterable) return
class OCRRecord(object): """ A composite object of containing recognition results for a single scanned page. A page is divided into lines which may contain segments and graphemes. For practical purposes this means that the appropriate line or segment (the later overriding the first) has to be brought into scope first before adding any characters to it. Each element may be associated with a responsibility statement, identifying the origin of each alteration if the final serialization supports it. """ # automatically generated properties on the class fields = [ 'title', 'author', 'editor', 'funder', 'principal', 'sponsor', 'meeting', 'edition', 'publisher', 'distributor', 'authority', 'idno', 'pub_place', 'licence', 'series_title', 'note', 'source_desc', 'img', 'dimensions' ] xml_ns = '{http://www.w3.org/XML/1998/namespace}' tei_ns = '{http://www.tei-c.org/ns/1.0}' abbyy_ns = '{http://www.abbyy.com/FineReader_xml/FineReader10-schema-v1.xml}' # automatically generated properties in the fileDesc element and xpath to their location _tei_fields = [ ('titleStmt', [( 'title', '/' + tei_ns + 'title', ), ('author', '/' + tei_ns + 'author', 'ref'), ('editor', '/' + tei_ns + 'editor', 'ref'), ('funder', '/' + tei_ns + 'funder', 'ref'), ('principal', '/' + tei_ns + 'principal', 'ref'), ('sponsor', '/' + tei_ns + 'sponsor', 'ref'), ('meeting', '/' + tei_ns + 'meeting')]), ('editionStmt', [('edition', '/' + tei_ns + 'edition')]), ('publicationStmt', [ ('publisher', '/' + tei_ns + 'publisher', 'target'), ('distributor', '/' + tei_ns + 'distributor', 'target'), ('authority', '/' + tei_ns + 'authority', 'target'), ('idno', '/' + tei_ns + 'idno', 'type'), ('pub_place', '/' + tei_ns + 'pubPlace'), ('licence', '/{0}availability/{0}licence'.format(tei_ns), 'target') ]), ('seriesStmt', [('series_title', '/' + tei_ns + 'p')]), ('notesStmt', [('note', '/' + tei_ns + 'note')]), ('sourceDesc', [('source_desc', '/' + tei_ns + 'p')]) ] def __init__(self): self.meta = {} self.img = None self.respstmt = OrderedDict() self.resp_scope = None self.line_scope = None self.segment_scope = None self.lines = OrderedDict() # generic setter/getter for metadata def _generic_getter(self, field): if field in self.meta: return self.meta[field] else: return None def _generic_setter(self, value, field): self.meta[field] = value # responsibility statement functionality def add_respstmt(self, resp, name, **kwargs): """ Adds a responsibility statement and returns its identifier. The new responsibility statement is automatically scoped. Args: resp (unicode): Nature of the responsible process. name (unicode): Text describing the processing software. kwargs (dict): Additional data used be the final serialization. Returns: A string containing the respstmt ID. """ kwargs['resp'] = resp kwargs['name'] = name id = u'resp_' + unicode(len(self.respstmt) + 1) self.respstmt[id] = kwargs self.resp_scope = id return id def scope_respstmt(self, id): """ Scopes a responsibility statement. Args: id (unicode): String of targeted resposibility statement. Raises: NidabaRecordException if the responsibility statement couldn't be found. """ if id not in self.respstmt: raise NidabaRecordException('No such responsibility statement') self.resp_scope = id def reset_respstmt_scope(self): """ Clears the current responsibility scope. """ self.resp_scope = None # writer functions for topographical data def add_line(self, dim, **kwargs): """ Marks the start of a new topographical line and scopes it. Args: dim (tuple): A tuple (x0, y0, x1, y1) denoting the bounding box. kwargs (dict): Additional data used by the final serialization. Returns: A string containing the line's identifier. """ id = u'line_' + unicode(len(self.lines) + 1) kwargs['bbox'] = dim kwargs['content'] = OrderedDict() if self.resp_scope: kwargs['resp'] = self.resp_scope self.lines[id] = kwargs self.line_scope = id return id def add_segment(self, dim, language=None, confidence=None, **kwargs): """ Marks the beginning of a new topographical segment in the current scope. Most often this correspond to a word recognized by an engine. Args: dim (tuple): A tuple containing the bounding box (x0, y0, x1, y1) lang (unicode): Optional identifier of the segment language. confidence (int): Optional confidence value between 0 and 100. kwargs (dict): Additional data used by the final serialization. Returns: A string containing the segment's indeitifier. Raises: NidabaRecordException if no line is scoped. """ if not self.line_scope: raise NidabaRecordException('No line scoped.') id = u'seg_' + unicode(len(self.segments) + 1) kwargs['type'] = 'segment' kwargs['bbox'] = dim if language: kwargs['language'] = language if confidence: if confidence < 0 or confidence > 100: raise NidabaRecordException( 'Segmentconfidence {} outside valid ' 'range'.format(confidence)) kwargs['confidence'] = confidence if self.resp_scope: kwargs['resp'] = self.resp_scope kwargs['content'] = OrderedDict() self.lines[self.line_scope]['content'][id] = kwargs self.segment_scope = id return id # actual recognition result writer def add_graphemes(self, it): """ Adds a number of graphemes to the current scope (either line or segment). A line and/or segment has to be created beforehand. Args: it (iterable): An iterable returning a dictionary which at least contains a key 'grapheme' with the recognition result. A bounding box has to be placed under the key 'bbox'; a confidence value in the range 0-100 (int) is expected under 'confidence'. Additional data (style etc.) will be retained for serializer use. """ if self.line_scope is None: raise NidabaRecordException('No element scoped.') if self.segment_scope is not None: target = self.lines[self.line_scope]['content'][ self.segment_scope]['content'] else: target = self.lines[self.line_scope]['content'] gr_cnt = len(self.graphemes) ids = [] for glyph in it: gr_cnt += 1 id = u'grapheme_' + unicode(gr_cnt) ids.append(id) glyph['type'] = 'grapheme' if 'confidence' in glyph and (glyph['confidence'] < 0 or glyph['confidence'] > 100): raise NidabaRecordException( 'Glyph confidence {} outside valid ' 'range'.format(glyph['confidence'])) if 'grapheme' not in glyph: raise NidabaRecordException( 'Mandatory field missing when adding graphemes.') if self.resp_scope: glyph['resp'] = self.resp_scope target[id] = glyph return ids def add_choices(self, id, it): """ Adds alternative interpretations to an element. Args: id (unicode): ID of the element. it (iterable): An iterable returning a dictionary containing an alternative reading ('alternative') and an optional confidence value ('confidence') in the range between 0 and 100. Raises: NidabaRecordException if no element with the ID could be found. """ if id in self.lines: target = self.lines[id] for line in self.lines.itervalues(): if id in line['content']: target = line['content'][id] break for seg in line['content'].itervalues(): if 'content' in seg and id in seg['content']: target = seg['content'][id] break alt = {'content': list(it)} if self.resp_scope: alt['resp'] = self.resp_scope target['alternatives'] = alt # scoping of topographical elements def scope_line(self, id): """ Scopes a line. Args: id (unicode): ID of the line to scope. Raises: NidabaRecordException if no line with the ID could be found. """ if id not in self.lines: raise NidabaRecordException('Invalid line ID.') self.line_scope = id def scope_segment(self, id): """ Scopes a segment (and by association its line). Args: id (unicode): ID of the segment to scope. Raises """ for line_id, line in self.lines.iteritems(): if id in line['content']: self.line_scope = line_id self.segment_scope = id return raise NidabaRecordException('Invalid segment ID.') def reset_line_scope(self): """ Resets line scope. """ self.line_scope = None self.segment_scope = None def reset_segment_scope(self): """ Resets segment scope. """ self.segment_scope = None # clearing topographical data def clear_lines(self): """ Deletes all lines and their content from the record. """ self.reset_line_scope() self.lines = OrderedDict() def clear_segments(self): """ Deletes all segments and their content from the record. """ self.reset_segment_scope() self.segment_scope = None for line in self.lines.itervalues(): line['content'] = OrderedDict() def clear_graphemes(self): """ Deletes all graphemes from the record. """ for line in self.lines.itervalues(): for seg in line['content'].itervalues(): if seg['type'] == 'grapheme': line['content'] = OrderedDict() break else: seg['content'] = OrderedDict() # properties offering short cuts (line are already top-level records) @property def segments(self): """ Returns an OrderedDict of segments with each segment being a dictionary. """ seg = OrderedDict() for line in self.lines.itervalues(): for seg_id, el in line['content'].iteritems(): if el['type'] == 'segment': seg[seg_id] = el return seg @property def graphemes(self): """ Returns a list of graphemes with each grapheme being a dictionary. """ g = OrderedDict() for line_id, line in self.lines.iteritems(): for seg_id, el in line['content'].iteritems(): if el['type'] == 'segment': for g_id, gr in el['content'].iteritems(): g[g_id] = gr elif el['type'] == 'grapheme': g[seg_id] = el return g # de-/serializers def load_tei(self, fp): """ Reads in a TEI facsimile and populates the record. Args: fp (File): Source file descriptor. """ doc = etree.parse(fp) self.respstmt = OrderedDict() self.resp_scope = None for stmt, fields in self._tei_fields: stmt_el = doc.find('{0}teiHeader/{0}fileDesc/{0}{1}'.format( self.tei_ns, stmt)) if stmt_el is None: continue for field in fields: f_el = stmt_el.find('./' + field[1]) if f_el is not None: if len(field) == 3 and f_el.get(field[2]): self.meta[field[0]] = [f_el.text, f_el.get(field[2])] else: self.meta[field[0]] = f_el.text for resp in doc.iter(self.tei_ns + 'respStmt'): id = resp.get(self.xml_ns + 'id') r = resp.find('.//{}resp'.format(self.tei_ns)).text n = resp.find('.//{}name'.format(self.tei_ns)).text self.respstmt[id] = {'resp': r, 'name': n} surface = doc.find('{0}sourceDoc/{0}surface'.format(self.tei_ns)) if surface.get('lrx') is not None and surface.get('lry') is not None: self.dimensions = (int(surface.get('lrx')), int(surface.get('lry'))) graphic = surface.find('{}graphic'.format(self.tei_ns)) if graphic is not None: self.img = graphic.get('url') root_zone = doc.find('{0}sourceDoc/{0}surface/{0}zone'.format( self.tei_ns)) corr_flag = False alts = [] sic = None last_el = None def _get_dict_from_key(id): if id in self.lines: return self.lines[id] for line in self.lines.itervalues(): if id in line['content']: return line['content'][id] for seg in line['content'].itervalues(): if 'content' in seg and id in seg['content']: return seg['content'][id] for el in islice(root_zone.iter(), 1, None): if el.tag != self.tei_ns + 'corr' and corr_flag: corr_flag = False # flush alternatives self.add_choices(sic, alts) alts = [] elif el.tag == self.tei_ns + 'sic': sic = None elif el.tag == self.tei_ns + 'corr': corr_flag = True alts.append({'alternative': ''.join(el.text)}) last_el = alts[-1] elif el.tag == self.tei_ns + 'line': if el.get('resp') is not None: self.scope_respstmt(el.get('resp')[1:]) id = self.add_line((int(el.get('ulx')), int(el.get('uly')), int(el.get('lrx')), int(el.get('lry')))) last_el = _get_dict_from_key(id) sic = id if not sic else None elif el.tag == self.tei_ns + 'zone' and el.get( 'type') == 'segment': if el.get('resp') is not None: self.scope_respstmt(el.get('resp')[1:]) id = self.add_segment((int(el.get('ulx')), int(el.get('uly')), int(el.get('lrx')), int(el.get('lry')))) last_el = _get_dict_from_key(id) sic = id if not sic else None elif el.tag == self.tei_ns + 'zone' and el.get( 'type') == 'grapheme': gr = { 'bbox': (int(el.get('ulx')), int(el.get('uly')), int(el.get('lrx')), int(el.get('lry'))), 'grapheme': el.findtext('./{0}seg/{0}g'.format(self.tei_ns)) } id = self.add_graphemes([gr])[0] last_el = _get_dict_from_key(id) sic = id if not sic else None elif el.tag == self.tei_ns + 'certainty': last_el['confidence'] = float(el.get('degree')) * 100 elif el.tag in [ self.tei_ns + 'seg', self.tei_ns + 'g', self.tei_ns + 'choice' ]: pass else: raise NidabaRecordException( 'Unknown tag {} encountered'.format(el.tag)) def write_tei(self, fp): """ Serializes the record to a TEI facsimile. Args: fp (File): Target file descriptor. """ doc = Element('TEI', nsmap={None: 'http://www.tei-c.org/ns/1.0'}, version='5.0') header = SubElement(doc, self.tei_ns + 'teiHeader') fileDesc = SubElement(header, self.tei_ns + 'fileDesc') sourceDoc = SubElement(doc, self.tei_ns + 'sourceDoc') kwargs = {} if self.dimensions: kwargs = { 'ulx': '0', 'uly': '0', 'lrx': str(self.dimensions[0]), 'lry': str(self.dimensions[1]) } surface = SubElement(sourceDoc, self.tei_ns + 'surface', **kwargs) if self.img: SubElement(surface, self.tei_ns + 'graphic', url=self.img) surface_zone = SubElement(surface, self.tei_ns + 'zone') for stmt, fields in self._tei_fields: # create *Stmt in correct order parent = Element(self.tei_ns + stmt) for field in fields: if field[0] in self.meta: el = parent for node in field[1].split('/{')[1:]: el = SubElement(el, '{' + node) value = self.meta[field[0]] if isinstance(value, list): el.set(field[2], value[1]) value = value[0] el.text = value # insert *Stmt only when needed if list(parent): fileDesc.append(parent) titleStmt = doc.find('{0}teiHeader/{0}fileDesc/{0}titleStmt'.format( self.tei_ns)) if titleStmt is None: titleStmt = Element(self.tei_ns + 'titleStmt') fileDesc.insert(0, titleStmt) for id, resp in self.respstmt.iteritems(): r = SubElement(titleStmt, self.tei_ns + 'respStmt') r.set(self.xml_ns + 'id', id) SubElement(r, self.tei_ns + 'resp').text = resp['resp'] SubElement(r, self.tei_ns + 'name').text = resp['name'] def _set_confidence(el, up, dic): cert = None if 'confidence' in dic: cert = SubElement(el, self.tei_ns + 'certainty', degree=u'{0:.2f}'.format(dic['confidence'] / 100.0), locus='value') if el.get(self.xml_ns + 'id'): cert.set('target', '#' + el.get(self.xml_ns + 'id')) if 'resp' in up: el.set('resp', '#' + up['resp']) if cert is not None: cert.set('resp', '#' + up['resp']) def _wrap_choices(alternatives, sic, parent): choice = SubElement(parent, self.tei_ns + 'choice') sic_el = SubElement(choice, self.tei_ns + 'sic') sic_el.append(sic) for alt in alternatives['content']: corr = SubElement(choice, self.tei_ns + 'corr') corr.text = alt['alternative'] _set_confidence(corr, alternatives, alt) def _add_grapheme(grapheme_id, grapheme, parent): g_el = Element(self.tei_ns + 'zone', type='grapheme') g_el.set(self.xml_ns + 'id', grapheme_id) if 'bbox' in grapheme: g_el.set('ulx', str(grapheme['bbox'][0])) g_el.set('uly', str(grapheme['bbox'][1])) g_el.set('lrx', str(grapheme['bbox'][2])) g_el.set('lry', str(grapheme['bbox'][3])) if 'alternatives' in grapheme: _wrap_choices(grapheme['alternatives'], g_el, parent) else: parent.append(g_el) glyph = SubElement(SubElement(g_el, self.tei_ns + 'seg'), self.tei_ns + 'g') glyph.text = grapheme['grapheme'] _set_confidence(g_el, grapheme, grapheme) for line_id, line in self.lines.iteritems(): line_el = Element(self.tei_ns + 'line', ulx=str(line['bbox'][0]), uly=str(line['bbox'][1]), lrx=str(line['bbox'][2]), lry=str(line['bbox'][3])) line_el.set(self.xml_ns + 'id', line_id) _set_confidence(line_el, line, line) if 'alternatives' in line: _wrap_choices(line['alternatives'], line_el, surface_zone) else: surface_zone.append(line_el) for seg_id, seg in line['content'].iteritems(): if seg['type'] == 'segment': seg_el = Element(self.tei_ns + 'zone', ulx=str(seg['bbox'][0]), uly=str(seg['bbox'][1]), lrx=str(seg['bbox'][2]), lry=str(seg['bbox'][3]), type=seg['type']) seg_el.set(self.xml_ns + 'id', seg_id) _set_confidence(seg_el, seg, seg) for grapheme_id, grapheme in seg['content'].iteritems(): _add_grapheme(grapheme_id, grapheme, seg_el) if 'alternatives' in seg: _wrap_choices(seg['alternatives'], seg_el, line_el) else: line_el.append(seg_el) elif seg['type'] == 'grapheme': _add_grapheme(seg_id, seg, line_el) else: raise NidabaRecordException( 'Unknown nodes beneath line records') fp.write( etree.tostring(doc, xml_declaration=True, encoding='utf-8', pretty_print=True)) fp.flush() def write_abbyyxml(self, fp): """ Writes the TEI document in a format reminiscent of Abbyy FineReader's XML output. Its basic format is: <document> <page> <text> <line l="0" r="111" t="6" b="89"> <charParams l="0" r="78" t="6" b="89" charConfidence="76" wordStart="true">D</charParams> <charParams l="86" r="111" t="24" b="89" charConfidence="76" wordStart="false">e</charParams> </line> .... </text> </page> </document> Please note that alternative readings as produced for example by spell checking are dropped from the output. Responsibility statements, metadata, and source image information is likewise lost. Args: fp (file): File descriptor to write to. """ page = Element( 'document', xmlns= 'http://www.abbyy.com/FineReader_xml/FineReader10-schema-v1.xml', version='1.0', producer='nidaba') p = SubElement(page, 'page') p.set('width', str(self.dimensions[0])) p.set('height', str(self.dimensions[1])) p.set('resolution', '0') p.set('originalCoords', '1') b = SubElement(p, 'block', blockType='Text') text = SubElement(b, 'text') par = SubElement(text, 'par') for line in self.lines.itervalues(): lel = SubElement(par, 'line') # XXX: meaning of baseline is nowere documented lel.set('baseline', '0') lel.set('l', str(line['bbox'][0])) lel.set('t', str(line['bbox'][1])) lel.set('r', str(line['bbox'][2])) lel.set('b', str(line['bbox'][3])) for seg in line['content'].itervalues(): if seg['type'] == 'segment': formatting = SubElement(lel, 'formatting') if 'language' in seg: formatting.set('lang', seg['language']) word_start = True for g in seg['content'].itervalues(): if 'bbox' not in g: raise NidabaRecordException( 'No bounding box for grapheme') el = SubElement(formatting, 'charParams') if word_start: el.set('wordStart', 'true') word_start = False else: el.set('wordStart', 'false') el.text = g['grapheme'] el.set('l', str(g['bbox'][0])) el.set('t', str(g['bbox'][1])) el.set('r', str(g['bbox'][2])) el.set('b', str(g['bbox'][3])) if 'confidence' in g: el.set('charConfidence', str(g['confidence'])) elif seg['type'] == 'grapheme': formatting = SubElement(lel, 'formatting') if 'language' in seg: formatting.set('lang', seg['language']) el = SubElement(formatting, 'charParams') el.text = g['grapheme'] el.set('l', str(g['bbox'][0])) el.set('t', str(g['bbox'][1])) el.set('r', str(g['bbox'][2])) el.set('b', str(g['bbox'][3])) if 'confidence' in g: el.set('charConfidence', str(g['confidence'])) else: raise NidabaRecordException( 'Unknown nodes beneath line records') fp.write(etree.tostring(page, xml_declaration=True, encoding='utf-8')) fp.flush() def write_alto(self, fp): """ Serializes the record as an ALTO XML document. See [0] for further information and schemata. Output will conform to version 3.1. Please note that the output will not be split into a series of "paragraphs" as segmentation algorithms don't produce them and they are dependent on typographic convention. Scores for alternatives are dropped as the standard does not provide for a way to encode them. Character confidences are rounded to the next lower confidence value (.98 -> 0, 0.05 -> 9). Alternatives are only serialized for segments. Line and grapheme alternatives are discarded. Args: fp (file): File descriptor to write to. [0] http://www.loc.gov/standards/alto/ """ alto = etree.Element('alto', xmlns="http://www.loc.gov/standards/alto/ns-v3#") description = SubElement(alto, 'Description') SubElement(description, 'MeasurementUnit').text = 'pixel' # use the image url as source image file name source_img = SubElement(description, 'sourceImageInformation') if self.img is not None: SubElement(source_img, 'fileName').text = self.img # convert responsibility statements to ocrProcessingSteps. As TEI # offers no way to distinguish between pre-, OCR, and postprocessing # for responsibility statements everything before a respStmt containing # 'recognition' is converted into preprocessing and everything after it # to postprocessing. ocr_proc = SubElement(description, 'OCRProcessing') ocr_proc.set('ID', 'OCR_0') mode = 'preProcessingStep' for id, respstmt in self.respstmt.iteritems(): if 'recognition' in respstmt[ 'resp'] and mode == 'preProcessingStep': mode = 'ocrProcessingStep' proc = SubElement(ocr_proc, mode) SubElement(proc, 'processingStepDescription').text = respstmt['resp'] ps = SubElement(proc, 'processingSoftware') SubElement(ps, 'softwareName').text = respstmt['name'] if mode == 'ocrProcessingStep': mode = 'postProcessingStep' layout = SubElement(alto, 'Layout') page = SubElement(layout, 'Page') if self.dimensions is not None: page.set('WIDTH', str(self.dimensions[0])) page.set('HEIGHT', str(self.dimensions[1])) page.set('PHYSICAL_IMG_NR', '0') page.set('ID', 'page_0') # why do all OCR formats insist on creating 'paragraph' containers? As # paragraphs are highly variable and dependent on typographic # conventions all text on the page is wrapped into a single paragraph. print_space = SubElement(page, 'PrintSpace') print_space.set('HPOS', str(0.0)) print_space.set('VPOS', str(0.0)) if self.dimensions is not None: print_space.set('WIDTH', str(self.dimensions[0])) print_space.set('HEIGHT', str(self.dimensions[1])) text_block = SubElement(print_space, 'TextBlock') text_block.set('HPOS', str(0.0)) text_block.set('VPOS', str(0.0)) text_block.set('ID', 'textblock_0') if self.dimensions is not None: text_block.set('WIDTH', str(self.dimensions[0])) text_block.set('HEIGHT', str(self.dimensions[1])) for line_id, line in self.lines.iteritems(): text_line = SubElement(text_block, 'TextLine') text_line.set('HPOS', str(line['bbox'][0])) text_line.set('VPOS', str(line['bbox'][1])) text_line.set('WIDTH', str(line['bbox'][2] - line['bbox'][0])) text_line.set('HEIGHT', str(line['bbox'][3] - line['bbox'][1])) text_line.set('ID', line_id) # There are 3 cases of content beneath a line: a list of graphemes # (which are outright dumped into a SINGLE String element), some # segments (each is converted to a String or SP node depending on # content), or some corr node which are converted to String nodes # containing ALTERNATIVE nodes. for seg_id, seg in line['content'].iteritems(): if seg['type'] == 'grapheme': text = u'' certs = [] for g in line['content'].itervalues(): text += g['grapheme'] # confidences for graphemes are integers between 0 and # 9 with 0 (wtf?) representing highest confidence if 'confidence' in g: certs.append( str(int(10 - 10 * (g['confidence'] / 100.0)))) text_el = SubElement(text_line, 'String') text_el.set('CONTENT', text) text_el.set('CC', ' '.join(certs)) break elif seg['type'] == 'segment': text = ''.join(x['grapheme'] for x in seg['content'].itervalues()) if text.isspace(): text_el = SubElement(text_line, 'SP') else: text_el = SubElement(text_line, 'String') text_el.set('CONTENT', text) # extract word confidences if 'confidence' in seg: text_el.set('WC', str(seg['confidence'] / 100.0)) certs = [] # extract character confidences for g in seg['content'].itervalues(): if 'confidence' in g: certs.append( str( int(10 - 10 * (g['confidence'] / 100.0)))) if certs: text_el.set('CC', ' '.join(certs)) text_el.set('HPOS', str(seg['bbox'][0])) text_el.set('VPOS', str(seg['bbox'][1])) text_el.set('WIDTH', str(seg['bbox'][2] - seg['bbox'][0])) text_el.set('HEIGHT', str(seg['bbox'][1] - seg['bbox'][3])) text_el.set('ID', seg_id) if 'alternatives' in seg: for corr in seg['alternatives']['content']: SubElement( text_el, 'ALTERNATIVE').text = corr['alternative'] # add an empty String element if no children exist if not list(text_line): text_el = SubElement(text_line, 'String') text_el.set('CONTENT', '') fp.write( etree.tostring(alto, pretty_print=True, xml_declaration=True, encoding='utf-8')) fp.flush() def load_hocr(self, fp): """ Reads an hOCR file and populates the record. Args: fp (file): File descriptor to read from. """ doc = etree.HTML(fp.read()) el = doc.find(".//meta[@name='ocr-system']") if el is not None: self.add_respstmt(el.get('content'), 'ocr-system') page = doc.find('body/div[@class="ocr_page"]') o = _parse_hocr(page.get('title')) if 'bbox' in o: self.dimensions = o['bbox'][2:] if 'image' in o: self.img = o['image'][0] corr_flag = False sic = None alts = [] cuts = [] for el in page.iter('span', 'ins', 'del'): el_class = el.get('class') if el_class != 'alt' and corr_flag: corr_flag = False # flush alternatives self.add_choices(sic, alts) alts = [] if el.tag == 'ins': sic = None elif el.tag == 'del': corr_flag = True o = _parse_hocr(el.get('title')) alts.append({ 'confidence': 100 - o['x_cost'][0], 'alternative': ''.join(el.itertext()) }) elif el_class == 'ocr_line': o = _parse_hocr(el.get('title')) id = self.add_line(o['bbox']) if el.xpath('.//span[starts-with(@class, "ocrx")]') is None: sym = ''.join(el.itertext()) self.add_graphemes([{'grapheme': x} for x in sym]) sic = id if not sic else None elif 'ocrx' in el_class: o = _parse_hocr(el.get('title')) id = self.add_segment(o['bbox'], o['x_conf'] if 'x_conf' in o else None) sic = id if not sic else None sym = ''.join(el.itertext()) self.add_graphemes([{'grapheme': x} for x in sym]) def write_hocr(self, fp): """ Serializes the OCR record in hOCR format. Metadata except image source and dimensions are lost, as are responsibility statements. Alternatives EXCEPT grapheme alternatives are inserted using the INS-DEL syntax described in section 10 of the hOCR standard [0]. Grapheme coordinates and confidences are added as cuts/x_confs to the ocr_line element. [0] https://docs.google.com/document/d/1QQnIQtvdAC_8n92-LhwPcjtAUFwBlzE8EWnKAxlgVf0/preview Args: fp (file): File descriptor to write to. """ page = etree.Element('html', xmlns="http://www.w3.org/1999/xhtml") head = SubElement(page, 'head') if 'title' in self.meta: SubElement(head, 'title').text = self.meta['title'] if self.respstmt: SubElement(head, 'meta', name="ocr-system", content=self.respstmt.values()[-1]['name']) capa = "ocr_page" if self.lines: capa += ", ocr_line" if self.segments: capa += ", ocrx_word" SubElement(head, 'meta', name='ocr-capabilities', content=capa) body = SubElement(page, 'body') p_hocr = _micro_hocr() if 'dimensions' in self.meta: p_hocr.add('bbox', 0, 0, *self.meta['dimensions']) if self.img is not None: p_hocr.add('image', self.img) ocr_page = SubElement(body, 'div', title=str(p_hocr)) ocr_page.set('class', 'ocr_page') def _wrap_alternatives(alternatives, ins, parent): span = SubElement(parent, 'span') span.set('class', 'alternatives') ins_el = SubElement(span, 'ins') ins_el.set('class', 'alt') ins_el.append(ins) for alt in alternatives['content']: corr = SubElement(span, 'del') corr.set('class', 'alt') corr.text = alt['alternative'] if 'confidence' in alt: corr.set('title', 'x_cost {}'.format(100 - alt['confidence'])) for line_id, line in self.lines.iteritems(): ocr_line = Element('span', id=line_id) ocr_line.set('class', 'ocr_line') ocr_line.text = u'' l_hocr = _micro_hocr() l_hocr.add('bbox', line['bbox']) gr_boxes = [] gr_confidences = [] if 'alternatives' in line: _wrap_alternatives(line['alternatives'], ocr_line, ocr_page) else: ocr_page.append(ocr_line) SubElement(ocr_page, 'br') for seg_id, seg in line['content'].iteritems(): if seg['type'] == 'grapheme': if 'bbox' in seg: gr_boxes.append(seg['bbox']) if 'confidence' in seg: gr_confidences.append(seg['confidence']) ocr_line.text += seg['grapheme'] elif seg['type'] == 'segment': ocrx_word = Element('span', id=seg_id) ocrx_word.set('class', 'ocrx_word') s_hocr = _micro_hocr() if 'bbox' in seg: s_hocr.add('bbox', seg['bbox']) if 'confidence' in seg: s_hocr.add('x_wconf', seg['confidence']) ocrx_word.set('title', str(s_hocr)) ocrx_word.text = u'' if 'alternatives' in seg: _wrap_alternatives(seg['alternatives'], ocrx_word, ocr_line) else: ocr_line.append(ocrx_word) for g in seg['content'].itervalues(): if 'bbox' in g: gr_boxes.append(g['bbox']) if 'confidence' in g: gr_confidences.append(g['confidence']) ocrx_word.text += g['grapheme'] else: raise NidabaRecordException( 'Unknown nodes beneath line records') if gr_boxes: l_hocr.add('cuts', *list(_delta(line['bbox'], gr_boxes))) if gr_confidences: l_hocr.add('x_confs', *gr_confidences) ocr_line.set('title', str(l_hocr)) fp.write( etree.tostring(page, pretty_print=True, xml_declaration=True, encoding='utf-8')) fp.flush() def write_text(self, fp, header=True): """ Writes the OCR record as plain text. Args: fp (file): File descriptor to write to. header (bool): Serialize metadata before the recognized text between '+++'. """ if self.meta and header is True: fp.write('+++\n') for key, val in self.meta.iteritems(): fp.write(u'{} = {}\n'.format(key, val).encode('utf-8')) fp.write('+++\n') for line in self.lines.itervalues(): fp.write('\n') for seg in line['content'].itervalues(): if seg['type'] == 'grapheme': fp.write(g['grapheme'].encode('utf-8')) else: for g in seg['content'].itervalues(): fp.write(g['grapheme'].encode('utf-8')) fp.flush()
class fftView(QtGui.QTabWidget): def __init__(self, *args, **kwargs): super(fftView, self).__init__(*args, **kwargs) self.img_dict = OrderedDict() def add_images(self, image_list, do_fft=True, loadingfactors=None): self.clear() if not image_list: return if type(image_list) == dict: for path, img in image_list.iteritems(): try: img = np.array(img) len(img) flag = True if do_fft: img = self.do_fft(img) for item in self.img_dict.itervalues(): if np.array_equal(img, item): flag = False if flag: self.img_dict[path.split('/')[-1]] = img except TypeError: continue else: for img in image_list: try: img = np.array(img) len(img) flag = True if do_fft: img = self.do_fft(img) for item in self.img_dict.itervalues(): if np.array_equal(img, item): flag = False if flag: self.img_dict[len(self.img_dict)] = img except TypeError: continue # sort img dictionary by key img_dict = OrderedDict() for key in sorted(self.img_dict.keys()): img_dict[key] = self.img_dict[key] del self.img_dict self.img_dict = img_dict data = imagetimeline(self.img_dict.itervalues()) sizemax = max(map(np.shape, data))[0] view = TimelineView(sizemax) view.setImage(data) scale = calcscale(view) # Sets up the scale view.imageItem.resetTransform() view.imageItem.scale(scale, scale) view.autoRange() view.getHistogramWidget().setHidden(False) view.ui.roiBtn.setHidden(True) view.ui.menuBtn.setHidden(True) view.sigImageChanged.connect(self.imageNameChanged) if loadingfactors is None: self.addTab(view, u"Tile " + str(1)) else: self.addTab(view, str(loadingfactors)) self.tabBar().hide() view.sigImageChanged.emit() def do_fft(self, img): """ Returns absolute value of 2-D FFT of image. Assumes 2-D image """ return np.log(np.fft.fftshift(np.abs(np.fft.fft2(img)**2)) + 1e-10) def open_from_rmcView(self, image_list): images = {} for lst in image_list: path = "/" for item in lst: path = os.path.join(path, item) img = Image.open(path).convert('L') img = np.array(img) images[path] = img self.add_images(images) def imageNameChanged(self): view = self.currentWidget() for key, val in self.img_dict.iteritems(): if np.array_equal(val, view.image[view.currentIndex]): self.currentWidget().label.setText(key)
class OperatorsFactory(object): __metaclass__ = Singleton def __init__(self): self._entries = OrderedDict() @property def name(self): return self._name def register(self, operator_class): if not issubclass(operator_class, BaseOperator): raise InvalidClassException( "The given class is not a BaseOperator " "subclass.") name = operator_class.name() ver = operator_class.version() entry = self._entries.get(name) if entry: entry[int(ver)] = operator_class else: self._entries[name] = {int(ver): operator_class} def has_operator(self, name, version=None): if version is None: return name in self._entries return name in self._entries and int(version) in self._entries[name] def get_all_operator_names(self, app=None): if app is None: return self._entries.keys() res = set() for versions in self._entries.itervalues(): for c in versions.itervalues(): if c.is_app_supported(app): res.add(c.name()) break return list(res) def get_channel_operator_names(self, app=None): res = set() for versions in self._entries.itervalues(): for c in versions.itervalues(): if c.is_channel_operator() and (app is None or c.is_app_supported(app)): res.add(c.name()) break return list(res) def get_item_operator_names(self, app=None): res = set() for versions in self._entries.itervalues(): for c in versions.itervalues(): import os if not c.is_channel_operator() and (app is None or c.is_app_supported(app)): res.add(c.name()) break return list(res) def get_operator(self, name, version): if name in self._entries and int(version) in self._entries[name]: return self._entries[name][int(version)] def create(self, name, version, facade, node, channel=None): if not isinstance(facade, BaseFacade): raise InvalidFacadeException('You need to provide a valid facade ' 'to create an operator') if name in self._entries and int(version) in self._entries[name]: return self._entries[name][int(version)](facade, node, channel=channel) def get_latest_version(self, name): if not name in self._entries: return None versions = self._entries[name].keys() versions.sort() return versions[-1] def remove_operator(self, name): del self._entries[name] def clear_operators(self): self._entries.clear() def reload_operators(self): for name, versions in self._entries.iteritems(): for ver, constructor in versions.iteritems(): module_name = constructor.__module__ class_name = constructor.__name__ del sys.modules[module_name] self._entries[name][ver] = __import__(module_name, globals(), locals(), [class_name])
def create_allele_dataframes(imgt_dat, fasta_gen, fasta_nuc, **kwargs): if kwargs.get("verbosity", 0): print now(), 'Loading IMGT allele dat file...' alleles = OrderedDict() with open(imgt_dat, 'rU') as handle: for i, record in enumerate(SeqIO.parse(handle, "imgt")): alleles[record.id] = record if kwargs.get("verbosity", 0): print now(), 'Initializing allele DataFrame...' ''' id HLA000001 type A*01:01:01:01 locus A class I --- I, II, TAP, MIC, other flags 0 --- problems with allele stored here, see below len_gen 3503 len_nuc 1098 full_gen 1 full_nuc 1 flagging: sum of the below codes +1 if HLA type ends in a letter (N, Q, etc.) +2 if CDS annotation != exon annotation +4 if features don't add up to gen sequence +8 if exons don't add up to nuc sequence ''' allele_info = 'id type 4digit locus flags len_dat len_gen len_nuc full_gen full_nuc' table = pd.DataFrame(index=alleles.keys(), columns=allele_info.split()) sequences = [] if kwargs.get("verbosity", 0): print now(), 'Filling DataFrame with allele data...' all_features = [ ] # contains tuples: (HLA id, feature type, feature number, feature start, feature end) for allele in alleles.itervalues(): allele_type = allele.description.replace('HLA-', '').split(',')[0] table.ix[allele.id]['id'] = allele.id table.ix[allele.id]['type'] = allele_type table.ix[allele.id]['4digit'] = ':'.join(allele_type.split(':')[:2]) table.ix[allele.id]['locus'] = allele_type.split('*')[0] table.ix[allele.id]['flags'] = 0 if allele_type[-1].isdigit() else 1 table.ix[allele.id]['len_dat'] = len(str(allele.seq)) table.ix[allele.id][ 'len_gen'] = 0 # TODO: IT STILL DOESNT SOLVE PERFORMANCEWARNING! table.ix[allele.id][ 'len_nuc'] = 0 # we initialize these nulls so that we don't get a table.ix[allele.id][ 'full_gen'] = 0 # PerformanceWarning + pickling when storing HDF table.ix[allele.id][ 'full_nuc'] = 0 # because of NaNs (they don't map to ctypes) sequences.append((allele.id, 'dat', str(allele.seq))) # number of features in 2013-04-30 hla.dat: # 9296 source (total # of alleles) # 9291 CDS, 5 gene (HLA-P pseudogenes) # 24493 exons, 3697 introns, 1027 UTRs # CDS matches exons in 99+% of cases, some have a single base-pair extension at the end for some # weird single-base exons or such (all on unimportant pseudogene loci) # so we extract exons, introns and UTRs features = [ f for f in allele.features if f.type in ('exon', 'intron', 'UTR') ] for feature in features: if feature.type in ('exon', 'intron'): feature_num = int(feature.qualifiers['number'][0]) else: # UTR # UTR either starts at the beginning or ends at the end assert feature.location.start == 0 or feature.location.end == len( allele.seq) feature_num = 1 if feature.location.start == 0 else 2 # 1 if 5' UTR, 2 if 3' UTR all_features.append((allele.id, feature.type, feature_num, int(feature.location.start), int(feature.location.end), len(feature))) # small sanity check, can be commented out cds = [f for f in allele.features if f.type == 'CDS'] if cds: if sum(map(len, [f for f in features if f.type == 'exon'])) != len( cds[0]): print "\tCDS length doesn't match sum of exons for", allele.id, allele_type table.ix[allele.id]['flags'] += 2 else: print "\tNo CDS found for", allele.id, allele_type table.ix[allele.id]['flags'] += 2 if kwargs.get("verbosity", 0): print now(), 'Loading gen and nuc files...' with open(fasta_gen, 'r') as fasta_gen: for record in SeqIO.parse(fasta_gen, 'fasta'): allele_id = record.id.replace('HLA:', '') table.ix[allele_id]['len_gen'] = len(record.seq) sequences.append((allele_id, 'gen', str(record.seq))) with open(fasta_nuc, 'r') as fasta_nuc: for record in SeqIO.parse(fasta_nuc, 'fasta'): allele_id = record.id.replace('HLA:', '') table.ix[allele_id]['len_nuc'] = len(record.seq) sequences.append((allele_id, 'nuc', str(record.seq))) # convert list of tuples into DataFrame for features and sequences all_features = pd.DataFrame( all_features, columns=['id', 'feature', 'number', 'start', 'end', 'length']) sequences = pd.DataFrame(sequences, columns=['id', 'source', 'sequence']) joined = pd.merge(table, all_features, how='inner', on='id') exons_for_locus = {} for i_locus, i_group in joined.groupby('locus'): exons_for_locus[i_locus] = i_group[i_group['feature'] == 'exon']['number'].max() # for i_id, i_group in joined.groupby('id'): # max_exons = exons_for_locus[table.ix[i_id]['locus']] # if len(i_group) >= 2*max_exons-1: # print i_id, 'is fully annotated on all exons and introns and UTRs' if kwargs.get("verbosity", 0): print now(), 'Checking dat features vs gen/nuc sequences...' for allele, features in joined.groupby('id'): row = features.irow( 0 ) # first row of the features subtable. Contains all allele information because of the join sum_features_length = features['length'].sum() sum_exons_length = features.ix[features['feature'] == 'exon']['length'].sum() if row['len_gen'] > 0 and row['len_gen'] != sum_features_length: print "\tFeature lengths don't add up to gen sequence length", allele, row[ 'len_gen'], sum_features_length, row['type'] table.ix[allele]['flags'] += 4 if row['len_nuc'] > 0 and row['len_nuc'] != sum_exons_length: print "\tExon lengths don't add up to nuc sequence length", allele, row[ 'len_nuc'], sum_exons_length, row['type'] table.ix[allele]['flags'] += 8 if kwargs.get("verbosity", 0): print now(), 'Sanity check finished...' return table, all_features, sequences
_b.mu * 1000.**3, # converting units: km^3/s^2 --> m^3/s^2 # radius # body radius (SI units, i.e. meters) _b.R * 1000., # safe_radius # body distance safe for a spacecraft fly-by ((2. * _b.R) if _b.body == 'jupiter' else (_b.R + 50.)) * 1000., # "The spacecraft range to Jupiter cannot go below 2*R_J at any time" # "The flyby altitudes at the satellites (i.e. the range to the satellite centre at closest approach on the flyby minus the satellite radius) cannot be below 50 km" # name # body name _b.body, )) for _b in body_tuple.itervalues() ]) (jupiter, io, europa, ganymede, callisto) = body_obj.values() # http://en.wikipedia.org/wiki/Orbital_elements body_label = { # 'Object UID' : , # a specified point in time # 'Epoch (MJD)' : , # measure of the radius of an orbit taken from the points of that same orbit's two most distant points # http://en.wikipedia.org/wiki/Semimajor_axis 'a (km)': r'$a$: semi major axis (km)',
class CodeClass(object): def __init__(self, generator, name, module=None, base='object', description=None): self.generator = generator self.name = name self.module = module self.base = base self.description = description self.properties = OrderedDict() self.methods = OrderedDict() @property def fullname(self): return '%s.%s' % (self.module.fullname, self.name) def make_names_unique(self): names = [] def unique_name(name): if name not in names: names.append(name) return name else: return unique_name(name + '_') for m in self.methods.itervalues(): m.name = unique_name(m.name) def __str__(self): self.make_names_unique() with Writer() as w: w.write('@has_validated_properties') w.write('@has_interfaces') base = self.base if isinstance( self.base, str) else (self.base.name if self.base.module == self.module else self.base.fullname) w.write('class %s(%s):' % (self.name, base)) w.i() if self.description: w.write_docstring(self.description) w.write('def __init__(self, context):') w.i() w.write('self.context = context') w.o() for n, p in self.properties.iteritems(): w.write() if p.default is not None: w.write('@property_default(%s)' % safe_repr(p.default)) if p.type: w.write('@property_type(%s)' % self.generator.get_classname(p.type)) w.write('@validated_property') w.write('def %s():' % n) w.i() if p.description or p.type is not None: w.write('"""') if p.description: w.write(p.description.strip()) if p.type is not None: if p.description: w.write() w.write(':rtype: :class:`%s`' % self.generator.get_classname(p.type)) w.write('"""') else: w.write('pass') w.o() for n, m in self.methods.iteritems(): w.write() w.write(m) return str(w)
class TechnicalBase(PlotInterface): """ 指标基类。 :ivar name: 指标对象名称 :ivar series: 单值指标的序列变量或多值指标字典 :ivar is_multiple: 是否是多值指标 """ def __init__(self, name='', widget=None): super(TechnicalBase, self).__init__(name, widget) self.name = name self.series = None self._args = None def _rolling_algo(self, data, n, i): """ 逐步运行函数。""" raise NotImplementedError def _vector_algo(self, data, n): """向量化运行, 结果必须赋值给self.values。 Args: data (np.ndarray): 数据 n (int): 时间窗口大小 """ raise NotImplementedError def compute(self): """ 构建时间序列变量,执行指标的向量算法。 """ if not hasattr(self, '_args'): raise Exception("每个指标都必须有_args属性,代表指标计算的参数!") self.data = self._args[0] # 数据转化成ta-lib能处理的格式 self._args[0] = transform2ndarray(self._args[0]) apply(self._vector_algo, tuple(self._args)) if not hasattr(self, 'values'): raise Exception("每个指标都必须有value属性,代表指标计算结果!") if isinstance(self.values, dict): self.series = OrderedDict() for key, value in self.values.iteritems(): self.series[key] = series.NumberSeries(value, self.name, self, float('nan')) for key, value in self.series.iteritems(): setattr(self, key, value) self.is_multiple = True else: self.series = [ series.NumberSeries(self.values, self.name, self, float('nan')) ] self.is_multiple = False self._init_bound() def compute_element(self, cache_index, rolling_index): """ 计算一个回溯值, 被Series延迟调用。 Args: cache_index (int): 缓存索引 rolling_index (int): 回溯索引 """ #rolling_index = min(len(self.data)-1, self.curbar) #values = None #if self._cache[cache_index][0] == self.curbar: #values = self._cache[cache_index][1] # 缓存命中 #else: #self._rolling_data = transform2ndarray(self.data) # 输入 ## 指标一次返回多个值 #args = (self._rolling_data, ) + self._args + (rolling_index,) #values = apply(self._rolling_algo, args) #self._cache[cache_index] = (self.curbar, values) #for i, v in enumerate(values): #if self.is_multiple: #self.series.values()[i].update(v) #else: #self.series[i].update(v) pass @property def curbar(self): if self.is_multiple: return self.series.itervalues().next().curbar return self.series[0].curbar def __size__(self): """""" if self.is_multiple: return len(self.series.itervalues().next()) return len(self.series[0]) #def debug_data(self): #""" 主要用于调试""" #return [s.data for s in self.series] def _added_to_tracker(self, tracker): if tracker: tracker.add_indicator(self) #def __tuple__(self): #""" 返回元组。某些指标,比如布林带有多个返回值。 #这里以元组的形式返回多个序列变量。 #""" #if isinstance(self.series, list): #return tuple(self.series) #else: #return (self.series,) #def __iter__(self): #return self #def next(self): #"""docstring for next""" #iter(self.series) def __call__(self, index): return self[index] def __getitem__(self, index): # 解析多元值, 返回series # python 3.x 有这种机制? # print self.name, index # print self.series[0].data if self.is_multiple: return self.series[index] # 返回单变量的值。 if index >= 0: return self.series[0][index] else: raise SeriesIndexError def __float__(self): return self.series[0][0] def __str__(self): return str(self.series[0][0]) # def __eq__(self, r): return float(self) == float(r) def __lt__(self, other): return float(self) < float(other) def __le__(self, other): return float(self) <= float(other) def __ne__(self, other): return float(self) != float(other) def __gt__(self, other): return float(self) > float(other) def __ge__(self, other): return float(self) >= float(other) # def __add__(self, r): return self.series[0][0] + float(r) def __sub__(self, r): return self.series[0][0] - float(r) def __mul__(self, r): return self.series[0][0] * float(r) def __div__(self, r): return self.series[0][0] / float(r) def __mod__(self, r): return self.series[0][0] % float(r) def __pow__(self, r): return self.series[0][0]**float(r) # def __radd__(self, r): return self.series[0][0] + float(r) def __rsub__(self, r): return self.series[0][0] - float(r) def __rmul__(self, r): return self.series[0][0] * float(r) def __rdiv__(self, r): return self.series[0][0] / float(r) def __rmod__(self, r): return self.series[0][0] % float(r) def __rpow__(self, r): return self.series[0][0]**float(r)
class JarWriter(object): ''' Class with methods to write Jar files. Can write more-or-less standard jar archives as well as jar archives optimized for Gecko. See the documentation for the close() member function for a description of both layouts. ''' def __init__(self, file=None, fileobj=None, compress=True, optimize=True, compress_level=9): ''' Initialize a Jar archive in the given file. Use the given file-like object if one is given instead of opening the given file name. The compress option determines the default behavior for storing data in the jar archive. The optimize options determines whether the jar archive should be optimized for Gecko or not. ``compress_level`` defines the zlib compression level. It must be a value between 0 and 9 and defaults to 9, the highest and slowest level of compression. ''' if fileobj: self._data = fileobj else: self._data = open(file, 'wb') if compress is True: compress = JAR_DEFLATED self._compress = compress self._compress_level = compress_level self._contents = OrderedDict() self._last_preloaded = None self._optimize = optimize def __enter__(self): ''' Context manager __enter__ method for JarWriter. ''' return self def __exit__(self, type, value, tb): ''' Context manager __exit__ method for JarWriter. ''' self.finish() def finish(self): ''' Flush and close the Jar archive. Standard jar archives are laid out like the following: - Local file header 1 - File data 1 - Local file header 2 - File data 2 - (...) - Central directory entry pointing at Local file header 1 - Central directory entry pointing at Local file header 2 - (...) - End of central directory, pointing at first central directory entry. Jar archives optimized for Gecko are laid out like the following: - 32-bits unsigned integer giving the amount of data to preload. - Central directory entry pointing at Local file header 1 - Central directory entry pointing at Local file header 2 - (...) - End of central directory, pointing at first central directory entry. - Local file header 1 - File data 1 - Local file header 2 - File data 2 - (...) - End of central directory, pointing at first central directory entry. The duplication of the End of central directory is to accomodate some Zip reading tools that want an end of central directory structure to follow the central directory entries. ''' offset = 0 headers = {} preload_size = 0 # Prepare central directory entries for entry, content in self._contents.itervalues(): header = JarLocalFileHeader() for name in entry.STRUCT: if name in header: header[name] = entry[name] entry['offset'] = offset offset += len(content) + header.size if entry['filename'] == self._last_preloaded: preload_size = offset headers[entry] = header # Prepare end of central directory end = JarCdirEnd() end['disk_entries'] = len(self._contents) end['cdir_entries'] = end['disk_entries'] end['cdir_size'] = reduce(lambda x, y: x + y[0].size, self._contents.values(), 0) # On optimized archives, store the preloaded size and the central # directory entries, followed by the first end of central directory. if self._optimize: end['cdir_offset'] = 4 offset = end['cdir_size'] + end['cdir_offset'] + end.size if preload_size: preload_size += offset self._data.write(struct.pack('<I', preload_size)) for entry, _ in self._contents.itervalues(): entry['offset'] += offset self._data.write(entry.serialize()) self._data.write(end.serialize()) # Store local file entries followed by compressed data for entry, content in self._contents.itervalues(): self._data.write(headers[entry].serialize()) self._data.write(content) # On non optimized archives, store the central directory entries. if not self._optimize: end['cdir_offset'] = offset for entry, _ in self._contents.itervalues(): self._data.write(entry.serialize()) # Store the end of central directory. self._data.write(end.serialize()) self._data.close() def add(self, name, data, compress=None, mode=None, skip_duplicates=False): ''' Add a new member to the jar archive, with the given name and the given data. The compress option indicates how the given data should be compressed (one of JAR_STORED, JAR_DEFLATE or JAR_BROTLI), or compressed according to the default defined when creating the JarWriter (None). True and False are allowed values for backwards compatibility, mapping, respectively, to JAR_DEFLATE and JAR_STORED. When the data should be compressed, it is only really compressed if the compressed size is smaller than the uncompressed size. The mode option gives the unix permissions that should be stored for the jar entry. If a duplicated member is found skip_duplicates will prevent raising an exception if set to True. The given data may be a buffer, a file-like instance, a Deflater or a JarFileReader instance. The latter two allow to avoid uncompressing data to recompress it. ''' name = mozpath.normsep(name) if name in self._contents and not skip_duplicates: raise JarWriterError("File %s already in JarWriter" % name) if compress is None: compress = self._compress if compress is True: compress = JAR_DEFLATED if compress is False: compress = JAR_STORED if (isinstance(data, (JarFileReader, Deflater)) and \ data.compress == compress): deflater = data else: deflater = Deflater(compress, compress_level=self._compress_level) if isinstance(data, basestring): deflater.write(data) elif hasattr(data, 'read'): if hasattr(data, 'seek'): data.seek(0) deflater.write(data.read()) else: raise JarWriterError("Don't know how to handle %s" % type(data)) # Fill a central directory entry for this new member. entry = JarCdirEntry() entry['creator_version'] = 20 if mode is not None: # Set creator host system (upper byte of creator_version) # to 3 (Unix) so mode is honored when there is one. entry['creator_version'] |= 3 << 8 entry['external_attr'] = (mode & 0xFFFF) << 16 if deflater.compressed: entry['min_version'] = 20 # Version 2.0 supports deflated streams entry['general_flag'] = 2 # Max compression entry['compression'] = deflater.compress else: entry['min_version'] = 10 # Version 1.0 for stored streams entry['general_flag'] = 0 entry['compression'] = JAR_STORED # January 1st, 2010. See bug 592369. entry['lastmod_date'] = ((2010 - 1980) << 9) | (1 << 5) | 1 entry['lastmod_time'] = 0 entry['crc32'] = deflater.crc32 entry['compressed_size'] = deflater.compressed_size entry['uncompressed_size'] = deflater.uncompressed_size entry['filename'] = name self._contents[name] = entry, deflater.compressed_data def preload(self, files): ''' Set which members of the jar archive should be preloaded when opening the archive in Gecko. This reorders the members according to the order of given list. ''' new_contents = OrderedDict() for f in files: if not f in self._contents: continue new_contents[f] = self._contents[f] self._last_preloaded = f for f in self._contents: if not f in new_contents: new_contents[f] = self._contents[f] self._contents = new_contents
class alg: def __repr__(self): return "LIRS" def __init__(self, c, **kwargs): # c is cache size self.c = c # Max self.lirs = 0 self.hirs = 0 self.maxhirs = max(1, 0.01*c) self.maxlirs = c - self.maxhirs self.S = OrderedDict() # Recently accessed pages (LIR pages and non-resident HIR pages), newest element is last self.Q = OrderedDict() # Resident HIR pages self.hitcount = 0 self.count = 0 def setup(self, reqlist): # I'm an online algorithm :-) pass def get(self, key, warm=0): #if self.count % 100 == 0: # sys.stderr.write("S: %d\n" % len(self.S)) #self.countResident() if not warm: self.count += 1 if (key in self.S and self.S[key].resident) or key in self.Q: if key in self.S: e = self.S[key] del self.S[key] if e.flag == HIR: # A HIR block with lower reuse distance than the bottom LIR element if e.resident: del self.Q[key] self.hirs -= 1 self.swap() # Change one LIR page to a HIR page self.evict() # Keep the number of resident HIR pages <= self.maxhirs self.lirs += 1 e.resident = True e.flag = LIR self.S[key] = e self.prune() # If this was a LIR element at the bottom elif key in self.Q: # Resident HIR page which is not in S, reuse distance is large so we don't make it a LIR page e = self.Q[key] #assert e.resident and e.flag == HIR del self.Q[key] self.Q[key] = e self.S[key] = e #print "HIT for %s" % key if not warm: self.hitcount += 1 return 1 return 0 #print "MISS for %s" % key #old get is not right # def get(self, key): # #if self.count % 100 == 0: # # sys.stderr.write("S: %d\n" % len(self.S)) # self.countResident() # self.count += 1 # if (key in self.S and self.S[key].resident) or key in self.Q: # #print "HIT for %s" % key # self.hitcount += 1 # return 1 # return 0 # #print "MISS for %s" % key def put(self, key, val=1): if key in self.S: e = self.S[key] del self.S[key] if e.flag == HIR: # A HIR block with lower reuse distance than the bottom LIR element if e.resident: del self.Q[key] self.hirs -= 1 #assert self.oldestS().flag == LIR #assert self.hirs <= self.maxhirs self.swap() # Change one LIR page to a HIR page self.evict() # Keep the number of resident HIR pages <= self.maxhirs self.lirs += 1 e.resident = True e.flag = LIR self.S[key] = e self.prune() # If this was a LIR element at the bottom elif key in self.Q: # Resident HIR page which is not in S, reuse distance is large so we don't make it a LIR page e = self.Q[key] #assert e.resident and e.flag == HIR del self.Q[key] self.Q[key] = e self.S[key] = e else: # not in cache # When a miss occurs and a free block is needed for replacement, we choose an HIR block that is resident in the cache e = entry(key) if self.lirs < self.maxlirs: e.flag = LIR # Not using all the cache, make it a LIR page self.lirs += 1 else: # NOTE: Not changing the number of LIR blocks! self.Q[key] = e self.hirs += 1 self.evict() # Evicting if we have max number of HIR blocks self.S[key] = e #assert self.oldestS().flag == LIR #self.countResident() def countResident(self): sm = 0 for key, e in self.S.items(): sm += 1 if e.resident else 0 for key, e in self.Q.items(): sm += 1 if key not in self.S else 0 if sm != self.lirs + self.hirs: print "sm=%d, self.lirs=%d, self.hirs=%d" % (sm, self.lirs, self.hirs) assert sm == self.lirs + self.hirs def evict(self): if self.hirs > self.maxhirs: # Setting the size of Q k, e_hir = self.Q.popitem(last=False) #assert e_hir.flag == HIR and e_hir.resident e_hir.resident = False # It is maybe in S self.hirs -= 1 if self.hirs > self.maxhirs: print "hirs: %d maxhirs: %d" % (self.hirs, self.maxhirs) #assert self.hirs <= self.maxhirs # TODO: If there are many nonresident entries in S, should we delete this from S? def swap(self): key, e = self.S.popitem(last=False) #assert e.flag == LIR and e.resident e.flag = HIR self.hirs += 1 self.lirs -= 1 self.Q[e.key] = e # Switched this LIR block to HIR and put it in Q # Important! We don't put it back in S def prune(self): while self.S: oldest = self.oldestS() if oldest.flag == LIR: break k, dele = self.S.popitem(last=False) #assert dele.flag == HIR #if dele.resident: #assert k in self.Q #else: #assert k not in self.Q def oldestS(self): return self.S.itervalues().next() def oldestQ(self): return self.Q.itervalues().next()
class Environment(object): """ Environment within which all agents operate. """ hard_time_limit = -100 # Set a hard time limit even if deadline is not enforced. #. define these here? # num_dummies = 100 # number of other agents on the roads # grid_size = (8, 6) # columns, rows valid_actions = [None, 'forward', 'left', 'right'] valid_inputs = { 'light': TrafficLight.valid_states, 'oncoming': valid_actions, 'left': valid_actions, 'right': valid_actions } valid_headings = [(1, 0), (0, -1), (-1, 0), (0, 1)] # E, N, W, S #. use symbols #. put verbose last def __init__(self, verbose=False, num_dummies=100, grid_size = (8, 6)): #. magic #s """ Create an environment verbose - set to True to display additional output from the simulation num_dummies - discrete number of dummy agents in the environment grid_size - discrete number of intersections (columns, rows) """ self.num_dummies = num_dummies # Number of dummy driver agents in the environment self.verbose = verbose # If debug output should be given # Initialize simulation variables self.done = False self.t = 0 #? integer time step self.agent_states = OrderedDict() self.step_data = {} self.success = None #? did the agent reach the destination in time? # Road network self.grid_size = grid_size # (columns, rows) self.bounds = (1, 2, self.grid_size[0], self.grid_size[1] + 1) # ? bad - storing info in unlabeled tuple self.block_size = 100 #? self.hang = 0.6 #? self.intersections = OrderedDict() #? self.roads = [] #? #? Add traffic lights to each intersection for x in xrange(self.bounds[0], self.bounds[2] + 1): for y in xrange(self.bounds[1], self.bounds[3] + 1): self.intersections[(x, y)] = TrafficLight() # A traffic light at each intersection #? Add roads between adjacent traffic lights for a in self.intersections: for b in self.intersections: if a == b: continue if (abs(a[0] - b[0]) + abs(a[1] - b[1])) == 1: # L1 distance = 1 self.roads.append((a, b)) # Add environment boundaries #? what is this? roads along edges? #. prefer range over xrange for upwards compatibility for x in range(self.bounds[0], self.bounds[2] + 1): self.roads.append(((x, self.bounds[1] - self.hang), (x, self.bounds[1]))) self.roads.append(((x, self.bounds[3] + self.hang), (x, self.bounds[3]))) for y in range(self.bounds[1], self.bounds[3] + 1): self.roads.append(((self.bounds[0] - self.hang, y), (self.bounds[0], y))) self.roads.append(((self.bounds[2] + self.hang, y), (self.bounds[2], y))) # Create dummy agents for i in range(self.num_dummies): self.create_agent(DummyAgent) # Primary agent and associated parameters self.primary_agent = None # to be set explicitly self.enforce_deadline = False # Trial data (updated at the end of each trial) self.trial_data = { 'testing': False, # if the trial is for testing a learned policy 'initial_distance': 0, # L1 distance from start to destination 'initial_deadline': 0, # given deadline (time steps) to start with 'net_reward': 0.0, # total reward earned in current trial 'final_deadline': None, # deadline value (time remaining) at the end 'actions': {0: 0, 1: 0, 2: 0, 3: 0, 4: 0}, # violations and accidents #. unlabelled dict! 'coverage': 0, # percentage of state-actions seen 'success': 0, # whether the agent reached the destination in time } def create_agent(self, agent_class, *args, **kwargs): """ Create an agent in the environment at a random location. """ #. parameters # learning - set to True to force the driving agent to use Q-learning # epsilon - continuous value for the exploration factor, default is 1 # alpha - continuous value for the learning rate, default is 0.5 agent = agent_class(self, *args, **kwargs) # ? what is heading? why a tuple? # self.agent_states[agent] = {'location': random.choice(self.intersections.keys()), 'heading': (0, 1)} intersection = random.choice(self.intersections.keys()) self.agent_states[agent] = {'location': intersection, 'heading': (0, 1)} return agent def set_primary_agent(self, agent, enforce_deadline=False): """ Set the given agent as the primary agent. The primary agent is the smartcab that is followed in the environment. """ self.primary_agent = agent agent.primary_agent = True self.enforce_deadline = enforce_deadline def reset(self, testing=False): """ Begin a new trial. """ self.done = False self.t = 0 # Reset status text self.step_data = {} #? this is status text? # Reset traffic lights for traffic_light in self.intersections.itervalues(): traffic_light.reset() # Pick a random start and destination start = random.choice(self.intersections.keys()) destination = random.choice(self.intersections.keys()) # Ensure starting location and destination are not too close while self.compute_dist(start, destination) < 4: #. magic number start = random.choice(self.intersections.keys()) destination = random.choice(self.intersections.keys()) # # Pick a random start and destination that aren't too close # min_distance = 4 # distance = 0 # while distance < min_distance: # start = random.choice(self.intersections.keys()) # destination = random.choice(self.intersections.keys()) # distance = self.compute_dist(start, destination) start_heading = random.choice(self.valid_headings) #. pick a direction distance = self.compute_dist(start, destination) #. would already have from above deadline = distance * 5 # 5 time steps per intersection away #. magic number if self.verbose: # Debugging print "Environment.reset(): Trial set up with start = {}, destination = {}, deadline = {}".format(start, destination, deadline) # Create a map of all possible initial positions #? what is this? positions = dict() for location in self.intersections: positions[location] = list() for heading in self.valid_headings: positions[location].append(heading) # Initialize agent(s) #? why is the set of agents called agent_states and not just agents? #. call it agent_key also for agent in self.agent_states.iterkeys(): #. agent = self.agents[agent_key] # or # for agent in self.agents.values(): if agent is self.primary_agent: # if agent.primary_agent: #. better? #. wait, the environment is storing the agents states? why? self.agent_states[agent] = { 'location': start, 'heading': start_heading, 'destination': destination, 'deadline': deadline } # For dummy agents, make them choose one of the available # intersections and headings still in 'positions' else: intersection = random.choice(positions.keys()) heading = random.choice(positions[intersection]) self.agent_states[agent] = { 'location': intersection, 'heading': heading, 'destination': None, 'deadline': None } # Now delete the taken location and heading from 'positions' positions[intersection] = list(set(positions[intersection]) - set([heading])) if positions[intersection] == list(): # No headings available for intersection del positions[intersection] # Delete the intersection altogether #? whoa, so agent is an object, and it's used as the key in self.agent_states - # weird... agent.reset(destination=(destination if agent is self.primary_agent else None), testing=testing) if agent is self.primary_agent: # Reset metrics for this trial (step data will be set during the step) self.trial_data['testing'] = testing self.trial_data['initial_deadline'] = deadline self.trial_data['final_deadline'] = deadline self.trial_data['net_reward'] = 0.0 self.trial_data['actions'] = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0} self.trial_data['parameters'] = {'e': agent.epsilon, 'a': agent.alpha} self.trial_data['success'] = 0 self.trial_data['coverage'] = 0 def step(self): """ This function is called when a time step is taken turing a trial. """ # Pretty print to terminal # print "" # print "/-------------------" # print "| Step {} Results".format(self.t) # print "\-------------------" # print "" if self.verbose: # Debugging print "Environment.step(): t = {}".format(self.t) # Update agents, primary first if self.primary_agent is not None: self.primary_agent.update() for agent in self.agent_states.iterkeys(): if agent is not self.primary_agent: agent.update() # Update traffic lights for intersection, traffic_light in self.intersections.iteritems(): traffic_light.update(self.t) #? update primary agent's deadline #. rename to time_remaining if self.primary_agent is not None: #. if self.primary_agent: # Agent has taken an action: reduce the deadline by 1 agent_deadline = self.agent_states[self.primary_agent]['deadline'] - 1 self.agent_states[self.primary_agent]['deadline'] = agent_deadline #. Agent has taken an action: reduce the time remaining by 1 #. self.agent_states[self.primary_agent]['time_remaining'] -= 1 if agent_deadline <= self.hard_time_limit: self.done = True self.success = False if self.verbose: # Debugging print "Environment.step(): Primary agent hit hard time limit ({})! Trial aborted.".format(self.hard_time_limit) elif self.enforce_deadline and agent_deadline <= 0: self.done = True self.success = False if self.verbose: # Debugging print "Environment.step(): Primary agent ran out of time! Trial aborted." self.t += 1 # environment stores the global time state def sense(self, agent): #. rename - getSensorInformation? """ Get sensor input information for an 'agent' in the environment. """ assert (agent in self.agent_states), "Unknown agent!" #? Get location, heading, light status for given agent state = self.agent_states[agent] #. should just get state directly from agent, not environment location = state['location'] heading = state['heading'] #. confusing... - use red/green instead of heading 0/1 light = 'green' if (self.intersections[location].state and heading[1] != 0) or \ ((not self.intersections[location].state) and heading[0] != 0) else 'red' # Populate oncoming, left, right #? which are...? oncoming = None left = None right = None for other_agent, other_state in self.agent_states.iteritems(): #. confusing if agent == other_agent or location != other_state['location'] or \ (heading[0] == other_state['heading'][0] and heading[1] == other_state['heading'][1]): continue # For dummy agents, ignore the primary agent # This is because the primary agent is not required to follow the waypoint if other_agent == self.primary_agent: #. should use 'is' ? continue other_heading = other_agent.get_next_waypoint() # ? #. confusing if (heading[0] * other_state['heading'][0] + heading[1] * other_state['heading'][1]) == -1: if oncoming != 'left': # we don't want to override oncoming == 'left' oncoming = other_heading elif (heading[1] == other_state['heading'][0] and -heading[0] == other_state['heading'][1]): if right != 'forward' and right != 'left': # we don't want to override right == 'forward or 'left' right = other_heading else: if left != 'forward': # we don't want to override left == 'forward' left = other_heading #. sensors = {light: light, ...} # return sensors return {'light': light, 'oncoming': oncoming, 'left': left, 'right': right} #. bad name - should be time remaining def get_deadline(self, agent): """ Returns the deadline remaining for an agent. """ return self.agent_states[agent]['deadline'] if agent is self.primary_agent else None def act(self, agent, action): """ Consider an action and perform the action if it is legal. Returns a reward for the agent based on traffic laws. """ #. long fn - break down a bit #. doesn't include reward for reaching destination? #. this belongs in Agent, yeah? # assert agent in self.agent_states, "Unknown agent!" # assert action in self.valid_actions, "Invalid action!" assert (agent in self.agent_states), "Unknown agent!" assert (action in self.valid_actions), "Invalid action!" state = self.agent_states[agent] location = state['location'] heading = state['heading'] #. confusing, duplicate code light = 'green' if (self.intersections[location].state and heading[1] != 0) or \ ((not self.intersections[location].state) and heading[0] != 0) else 'red' inputs = self.sense(agent) # Assess whether the agent can move based on the action chosen. # Either the action is okay to perform, or falls under 4 types of violations: #. use strings or consts for violation types # 0: Action okay # 1: Minor traffic violation # 2: Major traffic violation # 3: Minor traffic violation causing an accident # 4: Major traffic violation causing an accident violation = 0 # Reward scheme # First initialize reward uniformly random from [-1, 1] reward = 2 * random.random() - 1 # Create a penalty factor as a function of remaining deadline # Scales reward multiplicatively from [0, 1] #? fnc means what? fnc = self.t * 1.0 / (self.t + state['deadline']) if agent.primary_agent else 0.0 gradient = 10 #. magic number - gradient of what? # No penalty given to an agent that has no enforced deadline #. put this in an else clause below penalty = 0 # If the deadline is enforced, give a penalty based on time remaining #? explain penalty fn if self.enforce_deadline: penalty = (math.pow(gradient, fnc) - 1) / (gradient - 1) # Agent wants to drive forward: if action == 'forward': if light != 'green': # Running red light violation = 2 # Major violation #. magic# (and several more) if inputs['left'] == 'forward' or inputs['right'] == 'forward': # Cross traffic violation = 4 # Accident # Agent wants to drive left: elif action == 'left': if light != 'green': # Running a red light violation = 2 # Major violation if inputs['left'] == 'forward' or inputs['right'] == 'forward': # Cross traffic violation = 4 # Accident elif inputs['oncoming'] == 'right': # Oncoming car turning right violation = 4 # Accident else: # Green light if inputs['oncoming'] == 'right' or inputs['oncoming'] == 'forward': # Incoming traffic violation = 3 # Accident else: # Valid move! heading = (heading[1], -heading[0]) # Agent wants to drive right: elif action == 'right': if light != 'green' and inputs['left'] == 'forward': # Cross traffic violation = 3 # Accident else: # Valid move! heading = (-heading[1], heading[0]) # Agent wants to perform no action: elif action == None: if light == 'green' and inputs['oncoming'] != 'left': # No oncoming traffic violation = 1 # Minor violation # Did the agent attempt a valid move? if violation == 0: #. magic numbers if action == agent.get_next_waypoint(): # Was it the correct action? reward += 2 - penalty # (2, 1) elif action == None and light != 'green': # Was the agent stuck at a red light? reward += 2 - penalty # (2, 1) else: # Valid but incorrect reward += 1 - penalty # (1, 0) # Move the agent if action is not None: #. confusing - wrap around torus world? location = ((location[0] + heading[0] - self.bounds[0]) % (self.bounds[2] - self.bounds[0] + 1) + self.bounds[0], (location[1] + heading[1] - self.bounds[1]) % (self.bounds[3] - self.bounds[1] + 1) + self.bounds[1]) # wrap-around state['location'] = location state['heading'] = heading # Agent attempted invalid move else: #. magic #s - keys and rewards - define as a dictionary at start if violation == 1: # Minor violation reward += -5 elif violation == 2: # Major violation reward += -10 elif violation == 3: # Minor accident reward += -20 elif violation == 4: # Major accident reward += -40 # Did agent reach the goal after a valid move? if agent is self.primary_agent: if state['location'] == state['destination']: # Did agent get to destination before deadline? if state['deadline'] >= 0: self.trial_data['success'] = 1 # Stop the trial self.done = True self.success = True if self.verbose: # Debugging print "Environment.act(): Primary agent has reached destination!" if self.verbose: # Debugging print "Environment.act() [POST]: location: {}, heading: {}, action: {}, reward: {}".format(location, heading, action, reward) # Update metrics self.step_data['t'] = self.t self.step_data['violation'] = violation self.step_data['state'] = agent.get_state() self.step_data['deadline'] = state['deadline'] self.step_data['waypoint'] = agent.get_next_waypoint() self.step_data['inputs'] = inputs self.step_data['light'] = light self.step_data['action'] = action self.step_data['reward'] = reward self.trial_data['final_deadline'] = state['deadline'] - 1 self.trial_data['net_reward'] += reward self.trial_data['actions'][violation] += 1 self.trial_data['coverage'] = agent.coverage if self.verbose: # Debugging print "Environment.act(): Step data: {}".format(self.step_data) return reward def compute_dist(self, a, b): """ Compute the Manhattan (L1) distance of a toroidal world. """ dx1 = abs(b[0] - a[0]) dx2 = abs(self.grid_size[0] - dx1) dx = dx1 if dx1 < dx2 else dx2 dy1 = abs(b[1] - a[1]) dy2 = abs(self.grid_size[1] - dy1) dy = dy1 if dy1 < dy2 else dy2 return dx + dy
class CoverSettingsWidget(QWidget): changed = pyqtSignal() def __init__(self, mi=None, prefs=None, parent=None, for_global_prefs=False): QWidget.__init__(self, parent) self.ignore_changed = False self.for_global_prefs = for_global_prefs self.l = l = QHBoxLayout(self) l.setContentsMargins(0, 0, 0, 0) self.setLayout(l) self.settings_tabs = st = QTabWidget(self) l.addWidget(st) self.preview_label = la = Preview(self) l.addWidget(la) if prefs is None: prefs = cprefs self.original_prefs = prefs self.mi = mi or self.default_mi() self.colors_page = cp = QWidget(st) st.addTab(cp, _('&Colors')) cp.l = l = QGridLayout() cp.setLayout(l) if for_global_prefs: msg = _('When generating covers, a color scheme for the cover is chosen at random from the' ' color schemes below. You can prevent an individual scheme from being selected by' ' unchecking it. The preview on the right shows the currently selected color scheme.') else: msg = _('Choose a color scheme to be used for this generated cover.') + '<p>' + _( 'In normal cover generation, the color scheme is chosen at random from the list of color schemes below. You' ' can prevent an individual color scheme from being chosen by unchecking it here.') cp.la = la = QLabel('<p>' + msg) la.setWordWrap(True) l.addWidget(la, 0, 0, 1, -1) self.colors_list = cl = QListWidget(cp) l.addWidget(cl, 1, 0, 1, -1) self.colors_map = OrderedDict() self.ncs = ncs = QPushButton(QIcon(I('plus.png')), _('&New color scheme'), cp) ncs.clicked.connect(self.create_color_scheme) l.addWidget(ncs) self.ecs = ecs = QPushButton(QIcon(I('format-fill-color.png')), _('&Edit color scheme'), cp) ecs.clicked.connect(self.edit_color_scheme) l.addWidget(ecs, l.rowCount()-1, 1) self.rcs = rcs = QPushButton(QIcon(I('minus.png')), _('&Remove color scheme'), cp) rcs.clicked.connect(self.remove_color_scheme) l.addWidget(rcs, l.rowCount()-1, 2) self.styles_page = sp = QWidget(st) st.addTab(sp, _('&Styles')) sp.l = l = QVBoxLayout() sp.setLayout(l) if for_global_prefs: msg = _('When generating covers, a style for the cover is chosen at random from the' ' styles below. You can prevent an individual style from being selected by' ' unchecking it. The preview on the right shows the currently selected style.') else: msg = _('Choose a style to be used for this generated cover.') + '<p>' + _( 'In normal cover generation, the style is chosen at random from the list of styles below. You' ' can prevent an individual style from being chosen by unchecking it here.') sp.la = la = QLabel('<p>' + msg) la.setWordWrap(True) l.addWidget(la) self.styles_list = sl = QListWidget(sp) l.addWidget(sl) self.style_map = OrderedDict() self.font_page = fp = QWidget(st) st.addTab(fp, _('&Fonts and sizes')) fp.l = l = QFormLayout() fp.setLayout(l) fp.f = [] def add_hline(): f = QFrame() fp.f.append(f) f.setFrameShape(f.HLine) l.addRow(f) for x, label, size_label in ( ('title', _('&Title font family:'), _('&Title font size:')), ('subtitle', _('&Subtitle font family'), _('&Subtitle font size:')), ('footer', _('&Footer font family'), _('&Footer font size')), ): attr = '%s_font_family' % x ff = FontFamilyChooser(fp) setattr(self, attr, ff) l.addRow(label, ff) ff.family_changed.connect(self.emit_changed) attr = '%s_font_size' % x fs = QSpinBox(fp) setattr(self, attr, fs) fs.setMinimum(8), fs.setMaximum(200), fs.setSuffix(' px') fs.setValue(prefs[attr]) fs.valueChanged.connect(self.emit_changed) l.addRow(size_label, fs) add_hline() self.changed_timer = t = QTimer(self) t.setSingleShot(True), t.setInterval(500), t.timeout.connect(self.emit_changed) def create_sz(label): ans = QSpinBox(self) ans.setSuffix(' px'), ans.setMinimum(100), ans.setMaximum(10000) l.addRow(label, ans) ans.valueChanged.connect(self.changed_timer.start) return ans self.cover_width = create_sz(_('Cover &width:')) self.cover_height = create_sz(_('Cover &height:')) fp.cla = la = QLabel(_( 'Note that the preview to the side is of fixed aspect ratio, so changing the cover' ' width above will not have any effect. If you change the height, you should also change the width nevertheless' ' as it will be used in actual cover generation.')) la.setWordWrap(True) l.addRow(la) self.templates_page = tp = QWidget(st) st.addTab(tp, _('&Text')) tp.l = l = QVBoxLayout() tp.setLayout(l) tp.la = la = QLabel(_( 'The text on the generated cover is taken from the metadata of the book.' ' This is controlled via templates. You can use the <b>, <i> and <br> tags' ' in the templates for bold, italic and line breaks, respectively. The' ' default templates use the title, series and authors. You can change them to use' ' whatever metadata you like.')) la.setWordWrap(True), la.setTextFormat(Qt.PlainText) l.addWidget(la) def create_template_widget(title, which, button): attr = which + '_template' heading = QLabel('<h2>' + title) setattr(tp, attr + '_heading', heading) l.addWidget(heading) la = QLabel() setattr(self, attr, la) l.addWidget(la), la.setTextFormat(Qt.PlainText), la.setStyleSheet('QLabel {font-family: monospace}') la.setWordWrap(True) b = QPushButton(button) b.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) connect_lambda(b.clicked, self, lambda self: self.change_template(which)) setattr(self, attr + '_button', b) l.addWidget(b) if which != 'footer': f = QFrame(tp) setattr(tp, attr + '_sep', f), f.setFrameShape(QFrame.HLine) l.addWidget(f) l.addSpacing(10) create_template_widget(_('The title template'), 'title', _('Change the &title template')) create_template_widget(_('The sub-title template'), 'subtitle', _('Change the &sub-title template')) create_template_widget(_('The footer template'), 'footer', _('Change the &footer template')) l.addStretch(2) self.apply_prefs(prefs) self.changed.connect(self.update_preview) self.styles_list.itemSelectionChanged.connect(self.update_preview) self.colors_list.itemSelectionChanged.connect(self.update_preview) self.update_preview() def __enter__(self): self.ignore_changed = True def __exit__(self, *args): self.ignore_changed = False def emit_changed(self): if not self.ignore_changed: self.changed.emit() def apply_prefs(self, prefs): with self: self._apply_prefs(prefs) def _apply_prefs(self, prefs): for x in ('title', 'subtitle', 'footer'): attr = '%s_font_family' % x getattr(self, attr).font_family = prefs[attr] attr = '%s_font_size' % x getattr(self, attr).setValue(prefs[attr]) for x in ('title', 'subtitle', 'footer'): x += '_template' getattr(self, x).setText(prefs[x]) for x in ('width', 'height'): x = 'cover_' + x getattr(self, x).setValue(prefs[x]) color_themes = prefs['color_themes'].copy() color_themes.update(default_color_themes) disabled = set(prefs['disabled_color_themes']) self.colors_list.clear() self.colors_map = {} for name in sorted(color_themes, key=sort_key): self.colors_map[name] = li = QListWidgetItem(name, self.colors_list) li.setFlags(li.flags() | Qt.ItemIsUserCheckable) li.setCheckState(Qt.Unchecked if name in disabled else Qt.Checked) li.setData(Qt.UserRole, color_themes[name]) lu = prefs.get('last_used_colors') if not self.for_global_prefs and lu in self.colors_map and self.colors_map[lu].checkState() == Qt.Checked: self.colors_map[lu].setSelected(True) else: for name, li in self.colors_map.iteritems(): if li.checkState() == Qt.Checked: li.setSelected(True) break else: next(self.colors_map.itervalues()).setSelected(True) disabled = set(prefs['disabled_styles']) self.styles_list.clear() self.style_map.clear() for name in sorted(all_styles(), key=sort_key): self.style_map[name] = li = QListWidgetItem(name, self.styles_list) li.setFlags(li.flags() | Qt.ItemIsUserCheckable) li.setCheckState(Qt.Unchecked if name in disabled else Qt.Checked) lu = prefs.get('last_used_style') if not self.for_global_prefs and lu in self.style_map and self.style_map[lu].checkState() == Qt.Checked: self.style_map[lu].setSelected(True) else: for name, li in self.style_map.iteritems(): if li.checkState() == Qt.Checked: li.setSelected(True) break else: next(self.style_map.itervalues()).setSelected(True) @property def current_colors(self): for name, li in self.colors_map.iteritems(): if li.isSelected(): return name @property def disabled_colors(self): for name, li in self.colors_map.iteritems(): if li.checkState() == Qt.Unchecked: yield name @property def custom_colors(self): ans = {} for name, li in self.colors_map.iteritems(): if name.startswith('#'): ans[name] = li.data(Qt.UserRole) return ans @property def current_style(self): for name, li in self.style_map.iteritems(): if li.isSelected(): return name @property def disabled_styles(self): for name, li in self.style_map.iteritems(): if li.checkState() == Qt.Unchecked: yield name @property def current_prefs(self): prefs = {k:self.original_prefs[k] for k in self.original_prefs.defaults} for x in ('title', 'subtitle', 'footer'): attr = '%s_font_family' % x prefs[attr] = getattr(self, attr).font_family attr = '%s_font_size' % x prefs[attr] = getattr(self, attr).value() prefs['color_themes'] = self.custom_colors prefs['disabled_styles'] = list(self.disabled_styles) prefs['disabled_color_themes'] = list(self.disabled_colors) for x in ('title', 'subtitle', 'footer'): x += '_template' prefs[x] = getattr(self, x).text() for x in ('width', 'height'): x = 'cover_' + x prefs[x] = getattr(self, x).value() return prefs def insert_scheme(self, name, li): with self: self.colors_list.insertItem(0, li) cm = OrderedDict() cm[name] = li for k, v in self.colors_map.iteritems(): cm[k] = v self.colors_map = cm li.setSelected(True) for i in range(1, self.colors_list.count()): self.colors_list.item(i).setSelected(False) def create_color_scheme(self): scheme = self.colors_map[self.current_colors].data(Qt.UserRole) d = CreateColorScheme('#' + _('My Color Scheme'), scheme, set(self.colors_map), parent=self) if d.exec_() == d.Accepted: name, scheme = d.data li = QListWidgetItem(name) li.setData(Qt.UserRole, scheme), li.setFlags(li.flags() | Qt.ItemIsUserCheckable), li.setCheckState(Qt.Checked) self.insert_scheme(name, li) self.emit_changed() self.original_prefs['color_themes'] = self.current_prefs['color_themes'] def edit_color_scheme(self): cs = self.current_colors if cs is None or not cs.startswith('#'): return error_dialog(self, _('Cannot edit'), _( 'Cannot edit a builtin color scheme. Create a new' ' color scheme instead.'), show=True) li = self.colors_map[cs] d = CreateColorScheme(cs, li.data(Qt.UserRole), set(self.colors_map), edit_scheme=True, parent=self) if d.exec_() == d.Accepted: name, scheme = d.data li.setText(name) li.setData(Qt.UserRole, scheme) if name != cs: self.colors_map.pop(cs, None) self.insert_scheme(name, li) self.emit_changed() self.original_prefs['color_themes'] = self.current_prefs['color_themes'] def remove_color_scheme(self): cs = self.current_colors if cs is None or not cs.startswith('#'): return error_dialog(self, _('Cannot remove'), _( 'Cannot remove a builtin color scheme.'), show=True) for i in range(self.colors_list.count()): item = self.colors_list.item(i) if item.isSelected(): with self: del self.colors_map[item.text()] self.colors_list.takeItem(i) i = i % self.colors_list.count() self.colors_list.item(i).setSelected(True) self.emit_changed() self.original_prefs['color_themes'] = self.current_prefs['color_themes'] return def change_template(self, which): from calibre.gui2.dialogs.template_dialog import TemplateDialog from calibre.gui2.ui import get_gui gui = get_gui() if gui is None: from calibre.ebooks.metadata.book.base import field_metadata else: field_metadata = gui.current_db.new_api.field_metadata attr = which + '_template' templ = getattr(self, attr).text() d = TemplateDialog(self, templ, mi=self.mi, fm=field_metadata) if d.exec_() == d.Accepted: templ = d.rule[1] getattr(self, attr).setText(templ) self.emit_changed() @property def prefs_for_rendering(self): prefs = self.current_prefs prefs = override_prefs(prefs, override_style=self.current_style, override_color_theme=self.current_colors) return prefs def update_preview(self): if self.ignore_changed: return dpr = getattr(self, 'devicePixelRatioF', self.devicePixelRatio)() w, h = int(dpr * self.preview_label.sizeHint().width()), int(dpr * self.preview_label.sizeHint().height()) prefs = self.prefs_for_rendering hr = h / prefs['cover_height'] for x in ('title', 'subtitle', 'footer'): attr = '%s_font_size' % x prefs[attr] = int(prefs[attr] * hr) prefs['cover_width'], prefs['cover_height'] = w, h img = generate_cover(self.mi, prefs=prefs, as_qimage=True) img.setDevicePixelRatio(dpr) self.preview_label.setPixmap(QPixmap.fromImage(img)) def default_mi(self): from calibre.ebooks.metadata.book.base import Metadata mi = Metadata(_('A sample book'), [_('Author One'), _('Author Two')]) mi.series = _('A series of samples') mi.series_index = 4 mi.tags = [_('Tag One'), _('Tag Two')] mi.publisher = _('Some publisher') mi.rating = 4 mi.identifiers = {'isbn':'123456789', 'url': 'https://calibre-ebook.com'} mi.languages = ['eng', 'fra'] mi.pubdate = mi.timestamp = now() return mi def restore_defaults(self): defaults = self.original_prefs.defaults.copy() # Dont delete custom color themes when restoring defaults defaults['color_themes'] = self.custom_colors self.apply_prefs(defaults) self.update_preview() def save_state(self): self.original_prefs.set('last_used_colors', self.current_colors) self.original_prefs.set('last_used_style', self.current_style) def save_as_prefs(self): with self.original_prefs: for k, v in self.current_prefs.iteritems(): self.original_prefs[k] = v