class DownloaderSimulator(BaseDownloaderSimulator): def __init__(self, rate): self._requests_per_slot = rate self.slots = OrderedDict() super(DownloaderSimulator, self).__init__() def update(self, requests): for request in requests: hostname = urlparse(request.url).hostname or "" self.slots.setdefault(hostname, deque()).append(request) def download(self): output = [] _trash_can = [] for key, requests in six.iteritems(self.slots): for i in range(min(len(requests), self._requests_per_slot)): output.append(requests.popleft()) if not requests: _trash_can.append(key) for key in _trash_can: del self.slots[key] return output def downloader_info(self): info = {"key_type": "domain", "overused_keys": []} for key, requests in six.iteritems(self.slots): if len(requests) > self._requests_per_slot: info["overused_keys"].append(key) return info def idle(self): return len(self.slots) == 0
def get(content, keep_order=False): """Get dataentry from *content*. *keep_order* indicates whether to return an ordered dictionnay.""" if keep_order: from collections import OrderedDict dataentry = OrderedDict() else: dataentry = {} found = False for line in content.split('\n'): if line.strip().startswith('---- dataentry'): found = True continue elif line == '----': break elif not found: continue line_split = line.split(':') key = line_split[0].strip() value = re.sub('#.*$', '', ':'.join(line_split[1:])).strip() dataentry.setdefault(key, value) if not found: raise DokuWikiError('no dataentry found') return dataentry
def list_releases(moniker=None, limit=0): """ List currently available releases. Arguments: moniker -- kernel release moniker (e.g., mainline, stable, longterm, etc.). Defaults to everything limit -- maximum number of releases to list per moniker. Defaults to 0, which is no limit """ releases = get_releases() if moniker is None: releases_by_moniker = OrderedDict() for release in releases['releases']: releases_by_moniker.setdefault(release['moniker'], []).append(release) first = True for moniker, r in releases_by_moniker.items(): if not first: print() first = False print('%s:' % moniker) for release in r[:limit] if limit > 0 else r: print(release['version']) else: r = get_releases_by_moniker(releases, moniker) for release in r[:limit] if limit > 0 else r: print(release['version'])
def _parse(self, block): """ Translate Tiger XML structure to the fields of export format, so that tree writing code for export format can be used. """ nodes = OrderedDict() root = block.find('graph').get('root') for term in block.find('graph').find('terminals'): fields = nodes.setdefault(term.get('id'), 6 * [None]) fields[WORD] = term.get('word') fields[LEMMA] = term.get('lemma') fields[TAG] = term.get('pos') fields[MORPH] = term.get('morph') nodes[term.get('id')] = fields for nt in block.find('graph').find('nonterminals'): if nt.get('id') == root: ntid = '0' else: fields = nodes.setdefault(nt.get('id'), 6 * [None]) ntid = nt.get('id').split('_')[-1] fields[WORD] = '#' + ntid fields[TAG] = nt.get('cat') fields[LEMMA] = fields[MORPH] = '--' for edge in nt: idref = edge.get('idref') nodes.setdefault(idref, 6 * [None]) if nodes[idref][FUNC] is None: nodes[idref][FUNC] = edge.get('label') nodes[idref][PARENT] = ntid else: # secondary edge nodes[idref].extend((edge.get('label'), ntid)) tree = exportparse(list(nodes.values()), self.morphology, self.lemmas) sent = self._word(block, orig=True) return tree, sent
def get_stacktrace_processing_task(infos, processors): """Returns a list of all tasks for the processors. This can skip over processors that seem to not handle any frames. """ by_processor = {} to_lookup = {} # by_stacktrace_info requires stable sorting as it is used in # StacktraceProcessingTask.iter_processable_stacktraces. This is important # to guarantee reproducible symbolicator requests. by_stacktrace_info = OrderedDict() for info in infos: processable_frames = get_processable_frames(info, processors) for processable_frame in processable_frames: processable_frame.processor.preprocess_frame(processable_frame) by_processor.setdefault(processable_frame.processor, []) \ .append(processable_frame) by_stacktrace_info.setdefault(processable_frame.stacktrace_info, []) \ .append(processable_frame) if processable_frame.cache_key is not None: to_lookup[processable_frame.cache_key] = processable_frame frame_cache = lookup_frame_cache(to_lookup) for cache_key, processable_frame in six.iteritems(to_lookup): processable_frame.cache_value = frame_cache.get(cache_key) return StacktraceProcessingTask( processable_stacktraces=by_stacktrace_info, processors=by_processor )
def sum_factorise(variable, tail_ordering, monomial_sum): if tail_ordering: key_ordering = OrderedDict() sub_monosums = defaultdict(MonomialSum) for sum_indices, atomics, rest in monomial_sum: # Pull out those sum indices that are not contained in the # tail ordering, together with those atomics which do not # share free indices with the tail ordering. # # Based on this, split the monomial sum, then recursively # optimise each sub monomial sum with the first tail index # removed. tail_indices = tuple(i for i in sum_indices if i in tail_ordering) tail_atomics = tuple(a for a in atomics if set(tail_indices) & set(a.free_indices)) head_indices = tuple(i for i in sum_indices if i not in tail_ordering) head_atomics = tuple(a for a in atomics if a not in tail_atomics) key = (head_indices, head_atomics) key_ordering.setdefault(key) sub_monosums[key].add(tail_indices, tail_atomics, rest) sub_monosums = [(k, sub_monosums[k]) for k in key_ordering] monomial_sum = MonomialSum() for (sum_indices, atomics), monosum in sub_monosums: new_rest = sum_factorise(variable, tail_ordering[1:], monosum) monomial_sum.add(sum_indices, atomics, new_rest) # Use COFFEE algorithm to optimise the monomial sum return optimise_monomial_sum(monomial_sum, variable.index_ordering())
def get_all_kplex(graph, k): triangles = find_triangles(graph) kplexes=OrderedDict() while len(triangles) > 0: t = triangles.pop() [kplex, peripheral] = get_kplex(graph, t, k) kplex.sort() peripheral.sort() kplex = tuple(kplex) kplexes.setdefault(kplex,set()) kplexes[kplex]=kplexes[kplex].union(set(peripheral)) #here we remove triangles containing a vertex in the kplex to prevent duplicates #not sure what we should do about vertex belonging to multiple kplexes? #new_triangles = [] #for t in triangles: # is_valid = True # for v in kplex: # if v in t: # is_valid = False # break # if is_valid: # new_triangles.append(t) # #triangles = new_triangles return kplexes
def _groupResults(self, results, formatter): """ Group tests together based on their results. @param results: An iterable of tuples of two or more elements. The first element of each tuple is a test case. The remaining elements describe the outcome of that test case. @param formatter: A callable which turns a test case result into a string. The elements after the first of the tuples in C{results} will be passed as positional arguments to C{formatter}. @return: A C{list} of two-tuples. The first element of each tuple is a unique string describing one result from at least one of the test cases in C{results}. The second element is a list of the test cases which had that result. """ groups = OrderedDict() for content in results: case = content[0] outcome = content[1:] key = formatter(*outcome) groups.setdefault(key, []).append(case) return groups.items()
def print_queues(): """ Print each queue with waiting or delayed jobs, by priority """ queues = OrderedDict() for q in Queue.collection().sort(by='name', alpha=True).instances(): waiting = q.waiting.llen() delayed = q.delayed.zcard() if waiting + delayed == 0: continue name, priority = q.hmget('name', 'priority') queues.setdefault(name, []).append({ 'priority': int(priority), 'waiting': waiting, 'delayed': delayed, }) for name in queues: sub_queues = sorted(queues[name], key=itemgetter('priority'), reverse=True) total_waiting = sum([q['waiting'] for q in sub_queues]) total_delayed = sum([q['delayed'] for q in sub_queues]) if len(sub_queues) == 1: priority_part = sub_queues[0]['priority'] else: priority_part = '----' print('%30s %4s %4d %4d' % (name, priority_part, total_waiting, total_delayed)) if len(sub_queues) > 1: for i, q in enumerate(sub_queues): print('%30s %4d %4d %4d' % (' ', q['priority'], q['waiting'], q['delayed']))
def query(area, keywords, preRender=True): transKeywords = transKeyWords(keywords) colNameDict = getColNameDict() global redis rowDict = {} tableOrderedDict = OrderedDict() for key in redis.sscan_iter('rowSet',match=area.encode("utf8")+"[0-9][0-9][0-9][0-9]%%&&%%*%s*" % transKeywords[0], count=10000): #020304 六位,02表示专业,03表示该专业的第三个工作簿, 04表示该工作簿的第四个sheet flag = True tabIndex = key.split("%&&%")[0] rowContent = key.split("%&&%")[1] for keyword in transKeywords[1:]: #To Do 要分割出来再判断,加了*再后面加%%&&%%,增了差不多一倍的时间。。查询时过滤这个关键词吧。 if rowContent.find(keyword)==-1: flag = False break if flag: if preRender: for i in range(len(keywords)): #不使用前端的正则替换,很容易出错。后端把字符替换完即可。注意使用未转换redis特殊字符的关键字列表 rowContent = rowContent.replace(keywords[i],"".join(['<span class="keyword',str(i),'">',keywords[i],'</span>'])) rowDict.setdefault(tabIndex, []).append(rowContent.split("@$$@")) # ToDo 不能用:区分,因为单元各种会有这种符号 # rowOrderedDict = OrderedDict(sorted(rowDict.items(),cmp=lambda a,b:int(a[0][1:])-int(b[0][1:]) if a[0][0]==b[0][0] else int(a[0][0])-int(b[0][0]), key=lambda d:int(d[0]))) #使用OrderedDict解决显示输出表格排序的问题 rowOrderedDict = OrderedDict(sorted(rowDict.items(),cmp=lambda a,b:int(a[0][2:])-int(b[0][2:]) if a[0][0:2]==b[0][0:2] else int(a[0][0:2])-int(b[0][0:2]))) #使用OrderedDict解决显示输出表格排序的问题 for k, v in rowOrderedDict.items(): # row = OrderedDict(zip(colNameDict[k][0], v)) # 列名一样的话,使用zip会自动删除一列!注意,使用dict会打乱顺序,使用OrderDict解决 colNames = [colName.decode('utf8')[:60].encode('utf8') for colName in colNameDict[k][0]] # 限定列名为公式而导致过长的情况,只限定75字节 tableOrderedDict.setdefault(colNameDict[k][1]+" ---> "+colNameDict[k][2]+"(更新时间:"+colNameDict[k][3]+")", {"colName":colNames,"rowData":v,"tabIndex":k}) # 如何排序,按键值。导入相同的文件,相同表名,字典会被重置 return tableOrderedDict
def parse_recipe(self, response, title, picture): hxs = HtmlXPathSelector(response) section = None sections = OrderedDict() for node in hxs.select(xp_ingredients): text = html_to_text(node.extract()).strip() if not text: continue if node.select('strong'): section = text continue sections.setdefault(section, []).append(text) ingredients = sections.pop(None, None) or sections.pop(sections.keys()[-1]) extra_ingredients = [x for y in sections.values() for x in y] yield CocktailItem( title=title, picture=picture, url=response.url, ingredients=ingredients, extra_ingredients=extra_ingredients )
def normalizeRAmemberlist(inputstream,filterexpdate=None): #---------------------------------------------------------------------- ''' Take RunningAHEAD membership list (Export individual membership records), and produce member list. For a given expiration date, the earliest renewal date is used This allows "first renewal for year" computations :param inputstream: open file with csv exported from RunningAHEAD (e.g., from request.files['file'].stream) :param filterexpdate: yyyy-mm-dd for expiration date to filter on, else None :rtype: csv file data, string format (e.g., data for make_response(data)) ''' memberships = csv.DictReader(inputstream) members = OrderedDict({}) # for each membership, unique member can be found from combination of names, dob, gender (should be unique) # make list of memberships for this member for mship in memberships: first = mship['GivenName'] middle = mship['MiddleName'] last = mship['FamilyName'] gender = mship['Gender'] dob = mship['DOB'] # make list of memberships for this member members.setdefault((last,first,middle,dob,gender),[]).append(mship) # get ready for output outdatalist = wlist() OUT = csv.DictWriter(outdatalist,MINHDR,extrasaction='ignore') OUT.writeheader() # for each member, find earliest renew date for each expiration date for key in members: renewbyexp = {} thesememberships = members[key] for mship in thesememberships: thisexpdate = mship['ExpirationDate'] thisrenewdate = mship['RenewalDate'] # special case for 11/11/2013, FSRC bulk import if thisrenewdate =='2013-11-11': thisrenewdate = mship['JoinDate'] # optional filter if not filterexpdate or thisexpdate == filterexpdate: if thisexpdate not in renewbyexp or thisrenewdate < renewbyexp[thisexpdate]: renewbyexp[thisexpdate] = thisrenewdate # for each expiration date (already filtered appropriately), # grab a record, update renewal and expiration dates, then save for thisexpdate in renewbyexp: outrec = copy(members[key][-1]) # doesn't matter which one, pick the last outrec['RenewalDate'] = renewbyexp[thisexpdate] outrec['ExpirationDate'] = thisexpdate OUT.writerow(outrec) # note this adds \r\n # one big string for return data outputdata = ''.join(outdatalist) return outputdata
def main(): args = arguments() motif_sites = load_sitecounts(args.input_file, args.cutoff, args.proxyBED) if args.output_dir: outdir = args.output_dir else: outdir = '' double_sitecounts = OrderedDict() for motif in os.listdir(args.dirname): motevo_output_file = os.path.join(args.dirname, motif) double_sitecounts.setdefault(motif, 0) double_sitecounts[motif], dist = \ count_nonoverlapping_motifs(motif_sites, motevo_output_file, args.cutoff) fname = os.path.join(outdir, '%s.dist' % motif) with open(fname, 'w') as outf: for i, d in enumerate(dist): outf.write('\t'.join([ str(i), str(d) + '\n', ])) with open( os.path.join(outdir, os.path.basename(args.input_file)), 'w') as outf: outf.write('\t'.join([motif for motif in double_sitecounts.keys()]) + '\n') for region in motif_sites.keys(): outf.write(region + '\t') outf.write('\t'.join([str(double_sitecounts[motif][region]) for motif in double_sitecounts.keys()])) outf.write('\n') return 0
def get_milestones_choices(self, milestones): data = OrderedDict() for milestone in milestones: data.setdefault(milestone.state, []).append( (milestone.id, milestone.title) ) return [('', 'No milestone')] + list(data.items())
def archives(request) : try: post_date = Article.objects.dates('pub_date', 'month') post_date = post_date.reverse() post_date_article = [] for i in range(len(post_date)): post_date_article.append([]) for i in range(len(post_date)): cur_year = post_date[i].year cur_month = post_date[i].month temp_arc = Article.objects.filter(pub_date__year=cur_year).filter(pub_date__month = cur_month) post_date_article[i] = temp_arc dicts = OrderedDict() for i in range(len(post_date)): dicts.setdefault(post_date[i], post_date_article[i]) except Article.DoesNotExist: raise Http404 return render(request, 'archives.html', { 'select_module': 'archive', 'dicts' : dicts, 'error' : False, 'tag_cloud' : tagCloud()})
def get_data_values(time_series_results): # type: (QuerySet) -> object result_ids = [result_id[0] for result_id in time_series_results.values_list('pk')] data_values_queryset = TimeSeriesResultValue.objects.filter(result_id__in=result_ids).order_by('value_datetime').values('value_datetime', 'value_datetime_utc_offset', 'result_id', 'data_value') data_values_map = OrderedDict() for value in data_values_queryset: data_values_map.setdefault(value['value_datetime'], {}).update({ 'utc_offset': value['value_datetime_utc_offset'], value['result_id']: value['data_value'] }) data = [] for timestamp, values in data_values_map.iteritems(): local_timestamp = timestamp + timedelta(hours=values['utc_offset']) row = [ local_timestamp.strftime(CSVDataApi.date_format), # Local DateTime '{0}:00'.format(values['utc_offset']), # UTC Offset timestamp.strftime(CSVDataApi.date_format) # UTC DateTime ] for result_id in result_ids: try: row.append(values[result_id]) except KeyError: row.append('') data.append(row) return data
def compress(cls, operators): sets = OrderedDict() incs = OrderedDict() rval = [] for op in operators: if isinstance(op, cls): if op.sets: assert len(op.sets) == 1 and len(op.incs) == 0 sets.setdefault(op.sets[0], []).append(op) else: assert len(op.incs) == 1 and len(op.sets) == 0 incs.setdefault(op.incs[0], []).append(op) else: rval.append(op) # combine incs into sets if on same view for view, set_ops in iteritems(sets): set_op, = set_ops inc_ops = incs.get(view, []) for inc_op in inc_ops[:]: set_op.As.extend(inc_op.As) set_op.Xs.extend(inc_op.Xs) inc_ops.remove(inc_op) rval.append(set_op) # combine remaining incs if on same view for view, inc_ops in iteritems(incs): if len(inc_ops) > 0: inc_op0 = inc_ops[0] for inc_op in inc_ops[1:]: inc_op0.As.extend(inc_op.As) inc_op0.Xs.extend(inc_op.Xs) rval.append(inc_op0) return rval
def _parseStats(self,str_statsData,counter=False): listHeaderData = [] listInterfaces = [] dictInterfaceData = OrderedDict() listStatsData = str_statsData.split("#")[1].lstrip().rstrip().split("\n") for interfaceIndex in range(len(listStatsData)): str_interfaceData = "" listValues = listStatsData[interfaceIndex].split(",") if interfaceIndex == 0: for eachValue in range(len(listValues)-1): listHeaderData.append(listValues[eachValue]) #print("\n HeaderData \n" + str(listHeaderData)) else: str_dictKey = str(listValues[0])+'_'+str(listValues[1]) if str_dictKey in listInterfaces: continue else: listInterfaces.append(str_dictKey) for eachValue in range(len(listValues)-1): if listHeaderData[eachValue] in dict_reqdMet.keys(): dictInterfaceData.setdefault(str_dictKey+ "_" + str(dict_reqdMet[listHeaderData[eachValue]]),listValues[eachValue]) if (counter == True and (listHeaderData[eachValue] == 'bin' or listHeaderData[eachValue] == 'bout')): finalValue = ((int(listValues[eachValue]) - int(self.dictCounterValues[str_dictKey+ "_" + str(dict_reqdMet[listHeaderData[eachValue]])]))/5) dictInterfaceData[str_dictKey+ "_" + str(dict_reqdMet[listHeaderData[eachValue]])] = str(finalValue) else: pass if counter==False: self.dictCounterValues = dictInterfaceData else: dictInterfaceData['plugin_version'] = PLUGIN_VERSION dictInterfaceData['heartbeat_required']=HEARTBEAT print(str(json.dumps(dictInterfaceData)))
def fatlink_statistics_view(request, year=datetime.date.today().year, month=datetime.date.today().month): year = int(year) month = int(month) start_of_month = datetime.datetime(year, month, 1) start_of_next_month = first_day_of_next_month(year, month) start_of_previous_month = first_day_of_previous_month(year, month) fatStats = OrderedDict() if settings.IS_CORP: fatStats[settings.CORP_NAME] = CorpStat(settings.CORP_ID) else: alliance_corps = EveCorporationInfo.objects.filter(alliance__alliance_id=settings.ALLIANCE_ID) for corp in alliance_corps: fatStats[corp.corporation_name] = CorpStat(corp.corporation_id, corp=corp) fatlinks_in_span = Fatlink.objects.filter(fatdatetime__gte = start_of_month).filter(fatdatetime__lt = start_of_next_month) for fatlink in fatlinks_in_span: fats_in_fatlink = Fat.objects.filter(fatlink=fatlink) for fat in fats_in_fatlink: fatStats.setdefault(fat.character.corporation_name, CorpStat(fat.character.corporation_id, blue=True) ).n_fats += 1 fatStatsList = [fatStat for corp_name, fatStat in fatStats.items()] fatStatsList.sort(key=lambda stat: stat.corp.corporation_name) fatStatsList.sort(key=lambda stat: (stat.n_fats, stat.n_fats/stat.corp.member_count), reverse=True) if datetime.datetime.now() > start_of_next_month: context = {'fatStats':fatStatsList, 'month':start_of_month.strftime("%B"), 'year':year, 'previous_month': start_of_previous_month,'next_month': start_of_next_month} else: context = {'fatStats':fatStatsList, 'month':start_of_month.strftime("%B"), 'year':year, 'previous_month': start_of_previous_month} return render_to_response('registered/fatlinkstatisticsview.html', context, context_instance=RequestContext(request))
def iet_make(stree): """Create an IET from a ScheduleTree.""" nsections = 0 queues = OrderedDict() for i in stree.visit(): if i == stree: # We hit this handle at the very end of the visit return List(body=queues.pop(i)) elif i.is_Exprs: exprs = [Increment(e) if e.is_Increment else Expression(e) for e in i.exprs] body = ExpressionBundle(i.shape, i.ops, i.traffic, body=exprs) elif i.is_Conditional: body = Conditional(i.guard, queues.pop(i)) elif i.is_Iteration: # Order to ensure deterministic code generation uindices = sorted(i.sub_iterators, key=lambda d: d.name) # Generate Iteration body = Iteration(queues.pop(i), i.dim, i.dim._limits, offsets=i.limits, direction=i.direction, uindices=uindices) elif i.is_Section: body = Section('section%d' % nsections, body=queues.pop(i)) nsections += 1 elif i.is_Halo: body = HaloSpot(i.halo_scheme, body=queues.pop(i)) queues.setdefault(i.parent, []).append(body) assert False
def map(self,coordinates, filelist, strand=True, col=5): # read coordinates mapto = OrderedDict() with open(coordinates) as coor: for lin in coor: line_split = lin.split('\t') if strand: mapto.setdefault(line_split[0]+line_split[1]+line_split[2]+line_split[5].strip('\n'),[]).append(lin.strip('\n')) else: mapto.setdefault(line_split[0]+line_split[1]+line_split[2],[]).append(lin.strip('\n')) for fname in filelist: run_mapto = deepcopy(mapto) with open(fname) as f: for lin in f: line_split = lin.split('\t') if strand: cor = line_split[0]+line_split[1]+line_split[2]+line_split[5].strip('\n') else: cor = line_split[0]+line_split[1]+line_split[2] run_mapto[cor].append(line_split[col-1]) with open(fname+'mapped','w') as fout: for key in run_mapto: if len(run_mapto[key]) == 1: run_mapto[key].append('0') fout.write('\t'.join(run_mapto[key])+'\n')
class ArgsCollection(object): def __init__(self, *args, **kwargs): self._items = OrderedDict() for arg in args: k, _, v = arg.partition('=') k = k.lstrip('-') if not kwargs.get('flatten'): self._items.setdefault(k, []).append(v) else: self._items[k] = v def get(self, k, default=None): return self._items.get(k, default) def items(self): return self._items.items() def __getattr__(self, k, default=None): return self._items.get(k, default) def __contains__(self, k): return k in self._items def __getitem__(self, k): return self._items['k']
def normalize_urls(urls, platform=None, offline_only=False): platform = platform or subdir defaults = tuple(x.rstrip('/') + '/' for x in get_default_urls()) alias = binstar_channel_alias(channel_alias) def normalize_(url): url = url.rstrip('/') if is_url(url): url_s = canonical_channel_name(url, True) else: url_s = url url = alias + url return url_s, url newurls = OrderedDict() priority = 0 while urls: url = urls[0] urls = urls[1:] if url == "system" and rc_path: urls = get_rc_urls() + urls continue elif url in ("defaults", "system"): t_urls = defaults elif url == "local": t_urls = get_local_urls() else: t_urls = [url] priority += 1 for url0 in t_urls: url_s, url0 = normalize_(url0) if offline_only and not url0.startswith('file:'): continue for plat in (platform, 'noarch'): newurls.setdefault('%s/%s/' % (url0, plat), (url_s, priority)) return newurls
def get_mx(mail): """ returns ordered dict of MX records for email's domain :param mail: email adress :return: OrderedDict """ domain = getdomain(mail) try: mx_list = dns.resolver.query(domain, 'MX') except dns.exception.DNSException as error: if isinstance(error, dns.resolver.NoAnswer): raise LackOfMxDomainException('Can\'t get MX from DNS server.') elif isinstance(error, dns.resolver.Timeout): raise DnsTimeoutException('Timeout on DNS query') else: raise Exception('Unknown DNS exception.') servers = dict() ordered_response = OrderedDict() for server in mx_list: priority, mx_domain = str(server).split(' ') mx_domain = sub(r'\.$', '', mx_domain) servers.setdefault(int(priority), list()).append(mx_domain) for priority in sorted(servers): ordered_response.setdefault(priority, servers.get(priority, list())) return ordered_response
def generate_types(self, types): f = self._f_h self.loader.write_header(f) self.write_api_header(f) dedup_types = OrderedDict() for type in types: dedup_types.setdefault(type.name, []).append(type) for types in dedup_types.values(): type = types[0] output_string = (type.raw + '\n').lstrip().replace(' ', ' ') if self.omit_khrplatform: output_string = replace_khr_types(output_string) if output_string == '#include <KHR/khrplatform.h>\n': if self.omit_khrplatform: continue elif self.local_files: output_string = '#include "khrplatform.h"\n' if 'ptrdiff_t' in output_string: # 10.6 is the last version supporting more than 64 bit (>1060) output_string = \ '#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) ' +\ '&& (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ > 1060)\n' +\ output_string.replace('ptrdiff_t', 'long') + '#else\n' + output_string + '#endif\n' f.write(output_string)
def visualize(model, filename): values = model.all_params() cols = OrderedDict() for key, value in values.items(): if isinstance(value, np.ndarray): # TODO group by prefix, showing W and b side by side cols.setdefault(key[:-2] + key[-1] if len(key) > 1 and key[-2] in "bW" else key, []).append((key, value)) _, axes = plt.subplots(2, len(cols), figsize=(5*len(cols), 10)) # TODO https://stackoverflow.com/a/13784887/223267 plt.tight_layout() for j, col in enumerate(tqdm(cols.values(), unit="param", desc=filename)): for i in range(2): axis = axes[i, j] if len(values) > 1 else axes if len(col) <= i: plt.delaxes(axis) else: plt.sca(axis) key, value = col[i] plt.colorbar(plt.pcolormesh(smooth(value))) for pattern, repl in REPLACEMENTS: # TODO map 0123->ifoc key = re.sub(pattern, repl, key) plt.title(key) for axis in (plt.gca().xaxis, plt.gca().yaxis): axis.set_major_locator(MaxNLocator(integer=True)) output_file = filename + ".png" plt.savefig(output_file) plt.clf() print("Saved '%s'." % output_file)
def results(self, identifier, is_public): """ Return the results created by the job with the given identifier that were uploaded to S3. :param identifier: Unique identifier of the Spark job. :param is_public: Whether the Spark job is public or not. :return: A mapping of result prefixes to lists of results. :rtype: dict """ if is_public: bucket = self.config['PUBLIC_DATA_BUCKET'] else: bucket = self.config['PRIVATE_DATA_BUCKET'] params = { 'Prefix': '%s/' % identifier, 'Bucket': bucket, } results = OrderedDict() list_objects_v2_paginator = self.s3.get_paginator('list_objects_v2') for page in list_objects_v2_paginator.paginate(**params): for item in page.get('Contents', []): try: prefix = item['Key'].split('/')[1] except IndexError: continue results.setdefault(prefix, []).append(item['Key']) return results
def read_bed(file_path, restricted_genes=None): """Reads BED file and populates a dictionary separating genes by chromosome. Parameters ---------- file_path : str path to BED file filtered_genes: list list of gene names to not use Returns ------- bed_dict: dict dictionary mapping chromosome keys to a list of BED lines """ # read in entire bed file into a dict with keys as chromsomes bed_dict = OrderedDict() for bed_row in bed_generator(file_path): is_restrict_flag = restricted_genes is None or bed_row.gene_name in restricted_genes if is_restrict_flag: bed_dict.setdefault(bed_row.chrom, []) bed_dict[bed_row.chrom].append(bed_row) sort_chroms = sorted(bed_dict.keys(), key=lambda x: len(bed_dict[x]), reverse=True) bed_dict = OrderedDict((chrom, bed_dict[chrom]) for chrom in sort_chroms) return bed_dict
def _update_tiered_policy(self, ep_id): """ Sends an updated list of tiered policy to an endpoint. Recalculates the list. :param ep_id: ID of the endpoint to send an update to. """ _log.debug("Updating policies for %s from %s", ep_id, self.pol_ids_by_ep_id) # Order the profiles by tier and profile order, using the name of the # tier and profile as a tie-breaker if the orders are the same. profiles = [] for pol_id in self.pol_ids_by_ep_id.iter_values(ep_id): try: tier_order = self.tier_orders[pol_id.tier] except KeyError: _log.warn("Ignoring profile %s because its tier metadata is " "missing.") continue profile_order = self.profile_orders[pol_id] profiles.append((tier_order, pol_id.tier, profile_order, pol_id.policy_id, pol_id)) profiles.sort() # Convert to an ordered dict from tier to list of profiles. pols_by_tier = OrderedDict() for _, tier, _, _, pol_id in profiles: pols_by_tier.setdefault(tier, []).append(pol_id) endpoint = self.objects_by_id[ep_id] endpoint.on_tiered_policy_update(pols_by_tier, async=True)
def prizes_view(request, key=None): givings = request.contest.prizegiving_set \ .filter(state='SUCCESS', prize__prizeforuser__isnull=False) \ .distinct() groups = OrderedDict() for id, name in givings.values_list('id', 'name'): groups.setdefault(name, []).append(id) Group = namedtuple('Group', 'name ids') groups = list(map(Group._make, list(groups.items()))) key = int(key) if key is not None else groups[0].ids[0] group = [group for group in groups if group.ids[0] == key] if not group: raise Http404 group = group[0] pfus = PrizeForUser.objects.select_related('user', 'prize') \ .filter(prize__prize_giving__in=group.ids) return TemplateResponse(request, 'prizes/prizes.html', {'groups': groups, 'key': key, 'pfus': pfus, 'users_on_page': getattr(settings, 'PRIZES_ON_PAGE', 100)})
class Questionnaire: """Class with methods for adding questions to a questionnaire, and running the questionnaire. Additional keyword args are passed to the prompter method when it is called. """ def __init__(self, show_answers=True, can_go_back=True): self.questions = OrderedDict() # key -> list of Question instances self.answers = OrderedDict() # key -> answer self.show_answers = show_answers self.can_go_back = can_go_back def add(self, *args, **kwargs): """Add a Question instance to the questions dict. Each key points to a list of Question instances with that key. Use the `question` kwarg to pass a Question instance if you want, or pass in the same args you would pass to instantiate a question. """ if 'question' in kwargs and isinstance(kwargs['question'], Question): question = kwargs['question'] else: question = Question(*args, **kwargs) self.questions.setdefault(question.key, []).append(question) return question def one(self, *args, **kwargs): kwargs['prompter'] = 'one' return self.add(*args, **kwargs) def many(self, *args, **kwargs): kwargs['prompter'] = 'many' return self.add(*args, **kwargs) def raw(self, *args, **kwargs): kwargs['prompter'] = 'raw' return self.add(*args, **kwargs) def remove(self, key): """Remove all questions associated with `key`. Raises exception if `key` doesn't exist. """ return self.questions.pop(key) def run(self): """Asks all remaining questions in the questionnaire, returns the answers. """ while not self.done: self.ask() return self.answers @exit_on_keyboard_interrupt def ask(self, error=None): """Asks the next question in the questionnaire and returns the answer, unless user goes back. """ q = self.next_question if q is None: return try: answer = q.prompter(self.get_prompt(q, error), *q.prompter_args, **q.prompter_kwargs) except QuestionnaireGoBack as e: steps = e.args[0] if e.args else 1 if steps == 0: self.ask( ) # user can redo current question even if `can_go_back` is `False` return self.go_back(steps) else: if q._validate: error = q._validate(answer) if error: self.ask(error) return if q._transform: answer = q._transform(answer) self.answers[q.key] = answer return answer def get_prompt(self, question, error=None): parts = [] if self.show_answers: parts.append(self.answer_display()) if error: parts.append(error) parts.append(question.prompt) return '\n\n'.join(str(p) for p in parts) @property def next_question(self): """Returns the next `Question` in the questionnaire, or `None` if there are no questions left. Returns first question for whose key there is no answer and for which condition is satisfied, or for which there is no condition. """ for key, questions in self.questions.items(): if key in self.answers: continue for question in questions: if self.check_condition(question._condition): return question return None def check_condition(self, condition): """Helper that returns True if condition is satisfied/doesn't exist. """ if not condition: return True for c in condition.conditions: key, value, operator = c if not operator(self.answers[key], value): return False return True def go_back(self, n=1): """Move `n` questions back in the questionnaire by removing the last `n` answers. """ if not self.can_go_back: return N = max(len(self.answers) - abs(n), 0) self.answers = OrderedDict(islice(self.answers.items(), N)) @property def done(self): return self.next_question is None def reset(self): self.answers = OrderedDict() def format_answers(self, fmt='obj'): """Formats answers depending on `fmt`. """ fmts = ('obj', 'array', 'plain') if fmt not in fmts: eprint("Error: '{}' not in {}".format(fmt, fmts)) return def stringify(val): if type(val) in (list, tuple): return ', '.join(str(e) for e in val) return val if fmt == 'obj': return json.dumps(self.answers) elif fmt == 'array': answers = [[k, v] for k, v in self.answers.items()] return json.dumps(answers) elif fmt == 'plain': answers = '\n'.join('{}: {}'.format(k, stringify(v)) for k, v in self.answers.items()) return answers def answer_display(self, s=''): """Helper method for displaying the answers so far. """ padding = len(max(self.questions.keys(), key=len)) + 5 for key in list(self.answers.keys()): s += '{:>{}} : {}\n'.format(key, padding, self.answers[key]) return s
def ordered_default_dict_sample(): from collections import OrderedDict result = OrderedDict() for key, val in custom_generater(): result.setdefault(key, []).append(val)
class HTTPHeaderDict(MutableMapping): """ :param headers: An iterable of field-value pairs. Must not contain multiple field names when compared case-insensitively. :param kwargs: Additional field-value pairs to pass in to ``dict.update``. A ``dict`` like container for storing HTTP Headers. Field names are stored and compared case-insensitively in compliance with RFC 7230. Iteration provides the first case-sensitive key seen for each case-insensitive pair. Using ``__setitem__`` syntax overwrites fields that compare equal case-insensitively in order to maintain ``dict``'s api. For fields that compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add`` in a loop. If multiple fields that are equal case-insensitively are passed to the constructor or ``.update``, the behavior is undefined and some will be lost. >>> headers = HTTPHeaderDict() >>> headers.add('Set-Cookie', 'foo=bar') >>> headers.add('set-cookie', 'baz=quxx') >>> headers['content-length'] = '7' >>> headers['SET-cookie'] 'foo=bar, baz=quxx' >>> headers['Content-Length'] '7' """ def __init__(self, headers=None, **kwargs): super(HTTPHeaderDict, self).__init__() self._container = OrderedDict() if headers is not None: if isinstance(headers, HTTPHeaderDict): self._copy_from(headers) else: self.extend(headers) if kwargs: self.extend(kwargs) def __setitem__(self, key, val): self._container[key.lower()] = [key, val] return self._container[key.lower()] def __getitem__(self, key): val = self._container[key.lower()] return ', '.join(val[1:]) def __delitem__(self, key): del self._container[key.lower()] def __contains__(self, key): return key.lower() in self._container def __eq__(self, other): if not isinstance(other, Mapping) and not hasattr(other, 'keys'): return False if not isinstance(other, type(self)): other = type(self)(other) return (dict((k.lower(), v) for k, v in self.itermerged()) == dict((k.lower(), v) for k, v in other.itermerged())) def __ne__(self, other): return not self.__eq__(other) if not PY3: # Python 2 iterkeys = MutableMapping.iterkeys itervalues = MutableMapping.itervalues __marker = object() def __len__(self): return len(self._container) def __iter__(self): # Only provide the originally cased names for vals in self._container.values(): yield vals[0] def pop(self, key, default=__marker): '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised. ''' # Using the MutableMapping function directly fails due to the private marker. # Using ordinary dict.pop would expose the internal structures. # So let's reinvent the wheel. try: value = self[key] except KeyError: if default is self.__marker: raise return default else: del self[key] return value def discard(self, key): try: del self[key] except KeyError: pass def add(self, key, val): """Adds a (name, value) pair, doesn't overwrite the value if it already exists. >>> headers = HTTPHeaderDict(foo='bar') >>> headers.add('Foo', 'baz') >>> headers['foo'] 'bar, baz' """ key_lower = key.lower() new_vals = [key, val] # Keep the common case aka no item present as fast as possible vals = self._container.setdefault(key_lower, new_vals) if new_vals is not vals: vals.append(val) def extend(self, *args, **kwargs): """Generic import function for any type of header-like object. Adapted version of MutableMapping.update in order to insert items with self.add instead of self.__setitem__ """ if len(args) > 1: raise TypeError("extend() takes at most 1 positional " "arguments ({0} given)".format(len(args))) other = args[0] if len(args) >= 1 else () if isinstance(other, HTTPHeaderDict): for key, val in other.iteritems(): self.add(key, val) elif isinstance(other, Mapping): for key in other: self.add(key, other[key]) elif hasattr(other, "keys"): for key in other.keys(): self.add(key, other[key]) else: for key, value in other: self.add(key, value) for key, value in kwargs.items(): self.add(key, value) def getlist(self, key, default=__marker): """Returns a list of all the values for the named field. Returns an empty list if the key doesn't exist.""" try: vals = self._container[key.lower()] except KeyError: if default is self.__marker: return [] return default else: return vals[1:] # Backwards compatibility for httplib getheaders = getlist getallmatchingheaders = getlist iget = getlist # Backwards compatibility for http.cookiejar get_all = getlist def __repr__(self): return "%s(%s)" % (type(self).__name__, dict(self.itermerged())) def _copy_from(self, other): for key in other: val = other.getlist(key) if isinstance(val, list): # Don't need to convert tuples val = list(val) self._container[key.lower()] = [key] + val def copy(self): clone = type(self)() clone._copy_from(self) return clone def iteritems(self): """Iterate over all header lines, including duplicate ones.""" for key in self: vals = self._container[key.lower()] for val in vals[1:]: yield vals[0], val def itermerged(self): """Iterate over all headers, merging duplicate ones together.""" for key in self: val = self._container[key.lower()] yield val[0], ', '.join(val[1:]) def items(self): return list(self.iteritems()) @classmethod def from_httplib(cls, message): # Python 2 """Read headers from a Python 2 httplib message object.""" # python2.7 does not expose a proper API for exporting multiheaders # efficiently. This function re-reads raw lines from the message # object and extracts the multiheaders properly. obs_fold_continued_leaders = (' ', '\t') headers = [] for line in message.headers: if line.startswith(obs_fold_continued_leaders): if not headers: # We received a header line that starts with OWS as described # in RFC-7230 S3.2.4. This indicates a multiline header, but # there exists no previous header to which we can attach it. raise InvalidHeader( 'Header continuation with no previous header: %s' % line ) else: key, value = headers[-1] headers[-1] = (key, value + ' ' + line.strip()) continue key, value = line.split(':', 1) headers.append((key, value.strip())) return cls(headers)
class HashedFilesMixin(object): default_template = """url("%s")""" patterns = (("*.css", ( r"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""", (r"""(@import\s*["']\s*(.*?)["'])""", """@import url("%s")"""), )), ) def __init__(self, *args, **kwargs): super(HashedFilesMixin, self).__init__(*args, **kwargs) self._patterns = OrderedDict() self.hashed_files = {} for extension, patterns in self.patterns: for pattern in patterns: if isinstance(pattern, (tuple, list)): pattern, template = pattern else: template = self.default_template compiled = re.compile(pattern, re.IGNORECASE) self._patterns.setdefault(extension, []).append( (compiled, template)) def file_hash(self, name, content=None): """ Returns a hash of the file with the given name and optional content. """ if content is None: return None md5 = hashlib.md5() for chunk in content.chunks(): md5.update(chunk) return md5.hexdigest()[:12] def hashed_name(self, name, content=None): parsed_name = urlsplit(unquote(name)) clean_name = parsed_name.path.strip() opened = False if content is None: if not self.exists(clean_name): raise ValueError("The file '%s' could not be found with %r." % (clean_name, self)) try: content = self.open(clean_name) except IOError: # Handle directory paths and fragments return name opened = True try: file_hash = self.file_hash(clean_name, content) finally: if opened: content.close() path, filename = os.path.split(clean_name) root, ext = os.path.splitext(filename) if file_hash is not None: file_hash = ".%s" % file_hash hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext)) unparsed_name = list(parsed_name) unparsed_name[2] = hashed_name # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax if '?#' in name and not unparsed_name[3]: unparsed_name[2] += '?' return urlunsplit(unparsed_name) def url(self, name, force=False): """ Returns the real URL in DEBUG mode. """ if settings.DEBUG and not force: hashed_name, fragment = name, '' else: clean_name, fragment = urldefrag(name) if urlsplit(clean_name).path.endswith('/'): # don't hash paths hashed_name = name else: hashed_name = self.stored_name(clean_name) final_url = super(HashedFilesMixin, self).url(hashed_name) # Special casing for a @font-face hack, like url(myfont.eot?#iefix") # http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax query_fragment = '?#' in name # [sic!] if fragment or query_fragment: urlparts = list(urlsplit(final_url)) if fragment and not urlparts[4]: urlparts[4] = fragment if query_fragment and not urlparts[3]: urlparts[2] += '?' final_url = urlunsplit(urlparts) return unquote(final_url) def url_converter(self, name, template=None): """ Returns the custom URL converter for the given file name. """ if template is None: template = self.default_template def converter(matchobj): """ Converts the matched URL depending on the parent level (`..`) and returns the normalized and hashed URL using the url method of the storage. """ matched, url = matchobj.groups() # Completely ignore http(s) prefixed URLs, # fragments and data-uri URLs if url.startswith(('#', 'http:', 'https:', 'data:', '//')): return matched name_parts = name.split(os.sep) # Using posix normpath here to remove duplicates url = posixpath.normpath(url) url_parts = url.split('/') parent_level, sub_level = url.count('..'), url.count('/') if url.startswith('/'): sub_level -= 1 url_parts = url_parts[1:] if parent_level or not url.startswith('/'): start, end = parent_level + 1, parent_level else: if sub_level: if sub_level == 1: parent_level -= 1 start, end = parent_level, 1 else: start, end = 1, sub_level - 1 joined_result = '/'.join(name_parts[:-start] + url_parts[end:]) hashed_url = self.url(unquote(joined_result), force=True) file_name = hashed_url.split('/')[-1:] relative_url = '/'.join(url.split('/')[:-1] + file_name) # Return the hashed version to the file return template % unquote(relative_url) return converter def post_process(self, paths, dry_run=False, **options): """ Post process the given OrderedDict of files (called from collectstatic). Processing is actually two separate operations: 1. renaming files to include a hash of their content for cache-busting, and copying those files to the target storage. 2. adjusting files which contain references to other files so they refer to the cache-busting filenames. If either of these are performed on a file, then that file is considered post-processed. """ # don't even dare to process the files if we're in dry run mode if dry_run: return # where to store the new paths hashed_files = OrderedDict() # build a list of adjustable files matches = lambda path: matches_patterns(path, self._patterns.keys()) adjustable_paths = [path for path in paths if matches(path)] # then sort the files by the directory level path_level = lambda name: len(name.split(os.sep)) for name in sorted(paths.keys(), key=path_level, reverse=True): # use the original, local file, not the copied-but-unprocessed # file, which might be somewhere far away, like S3 storage, path = paths[name] with storage.open(path) as original_file: # generate the hash with the original content, even for # adjustable files. hashed_name = self.hashed_name(name, original_file) # then get the original's file content.. if hasattr(original_file, 'seek'): original_file.seek(0) hashed_file_exists = self.exists(hashed_name) processed = False # ..to apply each replacement pattern to the content if name in adjustable_paths: content = original_file.read().decode( settings.FILE_CHARSET) for patterns in self._patterns.values(): for pattern, template in patterns: converter = self.url_converter(name, template) try: content = pattern.sub(converter, content) except ValueError as exc: yield name, None, exc if hashed_file_exists: self.delete(hashed_name) # then save the processed result content_file = ContentFile(force_bytes(content)) saved_name = self._save(hashed_name, content_file) hashed_name = force_text(self.clean_name(saved_name)) processed = True else: # or handle the case in which neither processing nor # a change to the original file happened if not hashed_file_exists: processed = True saved_name = self._save(hashed_name, original_file) hashed_name = force_text(self.clean_name(saved_name)) # and then set the cache accordingly hashed_files[self.hash_key(name)] = hashed_name yield name, hashed_name, processed # Finally store the processed paths self.hashed_files.update(hashed_files) def clean_name(self, name): return name.replace('\\', '/') def hash_key(self, name): return name def stored_name(self, name): hash_key = self.hash_key(name) cache_name = self.hashed_files.get(hash_key) if cache_name is None: cache_name = self.clean_name(self.hashed_name(name)) # store the hashed name if there was a miss, e.g. # when the files are still processed self.hashed_files[hash_key] = cache_name return cache_name
class DestroyHandler(toolbox.Bookkeeper): # noqa """ The DestroyHandler class detects when a graph is impossible to evaluate because of aliasing and destructive operations. Several data structures are used to do this. An Op can use its view_map property to declare that an output may be aliased to an input. If that output is destroyed, the input is also considered to be destroyed. The view_maps of several Ops can feed into one another and form a directed graph. The consequence of destroying any variable in such a graph is that all variables in the graph must be considered to be destroyed, because they could all be referring to the same underlying storage. In the current implementation, that graph is a tree, and the root of that tree is called the foundation. TODO: why "in the current implementation" ? is there another implementation planned? TODO: why is the graph a tree? isn't it possible that one variable could be aliased to many variables? for example, don't switch and ifelse have to do this? The original DestroyHandler (if 0'ed out above) computed several data structures from scratch each time it was asked to validate the graph. Because this happens potentially thousands of times and each graph to validate is extremely similar to the previous one, computing the data structures from scratch repeatedly was wasteful and resulted in high compile times for large graphs. This implementation computes the data structures once at initialization and then incrementally updates them. It is a work in progress. The following data structures have been converted to use the incremental strategy: <none> The following data structures remain to be converted: <unknown> """ pickle_rm_attr = ["destroyers", "has_destroyers"] def __init__(self, do_imports_on_attach=True, algo=None): self.fgraph = None self.do_imports_on_attach = do_imports_on_attach """ Maps every variable in the graph to its "foundation" (deepest ancestor in view chain). TODO: change name to var_to_vroot. """ self.droot = OrderedDict() """ Maps a variable to all variables that are indirect or direct views of it (including itself) essentially the inverse of droot. TODO: do all variables appear in this dict, or only those that are foundations? TODO: do only destroyed variables go in here? one old docstring said so. TODO: rename to x_to_views after reverse engineering what x is """ self.impact = OrderedDict() """ If a var is destroyed, then this dict will map droot[var] to the apply node that destroyed var TODO: rename to vroot_to_destroyer """ self.root_destroyer = OrderedDict() if algo is None: algo = config.cycle_detection self.algo = algo self.fail_validate = OrderedDict() def on_attach(self, fgraph): """ When attaching to a new fgraph, check that 1) This DestroyHandler wasn't already attached to some fgraph (its data structures are only set up to serve one). 2) The FunctionGraph doesn't already have a DestroyHandler. This would result in it validating everything twice, causing compilation to be slower. Give the FunctionGraph instance: 1) A new method "destroyers(var)" TODO: what does this do exactly? 2) A new attribute, "destroy_handler" TODO: WRITEME: what does this do besides the checks? """ # Do the checking # already_there = False if self.fgraph is fgraph: already_there = True if self.fgraph is not None: raise Exception("A DestroyHandler instance can only serve one" " FunctionGraph. (Matthew 6:24)") for attr in ("destroyers", "destroy_handler"): if hasattr(fgraph, attr): already_there = True if already_there: # FunctionGraph.attach_feature catches AlreadyThere and cancels the attachment raise toolbox.AlreadyThere( "DestroyHandler feature is already present" " or in conflict with another plugin.") # Annotate the FunctionGraph # self.unpickle(fgraph) fgraph.destroy_handler = self self.fgraph = fgraph self.destroyers = ( OrderedSet()) # set of Apply instances with non-null destroy_map self.view_i = {} # variable -> variable used in calculation self.view_o = ( {} ) # variable -> set of variables that use this one as a direct input # clients: how many times does an apply use a given variable self.clients = OrderedDict() # variable -> apply -> ninputs self.stale_droot = True self.debug_all_apps = set() if self.do_imports_on_attach: toolbox.Bookkeeper.on_attach(self, fgraph) def unpickle(self, fgraph): def get_destroyers_of(r): droot, _, root_destroyer = self.refresh_droot_impact() try: return [root_destroyer[droot[r]]] except Exception: return [] fgraph.destroyers = get_destroyers_of def has_destroyers(protected_list): if self.algo != "fast": droot, _, root_destroyer = self.refresh_droot_impact() for protected_var in protected_list: try: root_destroyer[droot[protected_var]] return True except KeyError: pass return False def recursive_destroys_finder(protected_var): # protected_var is the idx'th input of app. for (app, idx) in protected_var.clients: if app == "output": continue destroy_maps = getattr(app.op, "destroy_map", {}).values() # If True means that the apply node, destroys the protected_var. if idx in [ dmap for sublist in destroy_maps for dmap in sublist ]: return True for var_idx in getattr(app.op, "view_map", {}).keys(): if idx in app.op.view_map[var_idx]: # We need to recursivly check the destroy_map of all the # outputs that we have a view_map on. if recursive_destroys_finder(app.outputs[var_idx]): return True return False for protected_var in protected_list: if recursive_destroys_finder(protected_var): return True return False fgraph.has_destroyers = has_destroyers def refresh_droot_impact(self): """ Makes sure self.droot, self.impact, and self.root_destroyer are up to date, and returns them (see docstrings for these properties above). """ if self.stale_droot: self.droot, self.impact, self.root_destroyer = _build_droot_impact( self) self.stale_droot = False return self.droot, self.impact, self.root_destroyer def on_detach(self, fgraph): if fgraph is not self.fgraph: raise Exception("detaching wrong fgraph", fgraph) del self.destroyers del self.view_i del self.view_o del self.clients del self.stale_droot assert self.fgraph.destroyer_handler is self delattr(self.fgraph, "destroyers") delattr(self.fgraph, "has_destroyers") delattr(self.fgraph, "destroy_handler") self.fgraph = None def fast_destroy(self, app, reason): """ Do the check for only 1 level. For now: - Destroyed variables can have only 1 clients. - Allow view to have multiple clients. - Allow sequence of view. - But don't allow to destroy view """ dm = getattr(app.op, "destroy_map", None) if not dm: return inputs = set(itertools.chain.from_iterable( dm.values())) # list of app's destroyed inputs for inp_idx in inputs: inp = app.inputs[inp_idx] if getattr(inp.tag, "indestructible", False) or isinstance( inp, graph.Constant): self.fail_validate[app] = InconsistencyError( "Attempting to destroy indestructible variables: %s" % inp) elif len(inp.clients) > 1: self.fail_validate[app] = aesara.gof.InconsistencyError( "Destroyed variable has more than one client. " + str(reason)) elif inp.owner: app2 = inp.owner inp_idx2 = app2.outputs.index(inp) v = getattr(app2.op, "view_map", {}) d = getattr(app2.op, "destroy_map", {}) if v: v = v.get(inp_idx2, []) if len(v) > 0: self.fail_validate[ app] = aesara.gof.InconsistencyError( "Destroyed variable has view_map. " + str(reason)) elif d: d = d.get(inp_idx2, []) if len(d) > 0: self.fail_validate[ app] = aesara.gof.InconsistencyError( "Destroyed variable has destroy_map. " + str(reason)) # These 2 assertions are commented since this function is called so many times # but they should be true. # assert len(v) <= 1 # assert len(d) <= 1 def on_import(self, fgraph, app, reason): """ Add Apply instance to set which must be computed. """ if app in self.debug_all_apps: raise ProtocolError("double import") self.debug_all_apps.add(app) # print 'DH IMPORT', app, id(app), id(self), len(self.debug_all_apps) # If it's a destructive op, add it to our watch list dmap = getattr(app.op, "destroy_map", None) vmap = getattr(app.op, "view_map", {}) if dmap: self.destroyers.add(app) if self.algo == "fast": self.fast_destroy(app, reason) # add this symbol to the forward and backward maps for o_idx, i_idx_list in vmap.items(): if len(i_idx_list) > 1: raise NotImplementedError( "destroying this output invalidates multiple inputs", (app.op)) o = app.outputs[o_idx] i = app.inputs[i_idx_list[0]] self.view_i[o] = i self.view_o.setdefault(i, OrderedSet()).add(o) # update self.clients for i, input in enumerate(app.inputs): self.clients.setdefault(input, OrderedDict()).setdefault(app, 0) self.clients[input][app] += 1 for i, output in enumerate(app.outputs): self.clients.setdefault(output, OrderedDict()) self.stale_droot = True def on_prune(self, fgraph, app, reason): """ Remove Apply instance from set which must be computed. """ if app not in self.debug_all_apps: raise ProtocolError("prune without import") self.debug_all_apps.remove(app) # UPDATE self.clients for input in set(app.inputs): del self.clients[input][app] if getattr(app.op, "destroy_map", OrderedDict()): self.destroyers.remove(app) # Note: leaving empty client dictionaries in the struct. # Why? It's a pain to remove them. I think they aren't doing any harm, they will be # deleted on_detach(). # UPDATE self.view_i, self.view_o for o_idx, i_idx_list in getattr(app.op, "view_map", OrderedDict()).items(): if len(i_idx_list) > 1: # destroying this output invalidates multiple inputs raise NotImplementedError() o = app.outputs[o_idx] i = app.inputs[i_idx_list[0]] del self.view_i[o] self.view_o[i].remove(o) if not self.view_o[i]: del self.view_o[i] self.stale_droot = True if app in self.fail_validate: del self.fail_validate[app] def on_change_input(self, fgraph, app, i, old_r, new_r, reason): """ app.inputs[i] changed from old_r to new_r. """ if app == "output": # app == 'output' is special key that means FunctionGraph is redefining which nodes are being # considered 'outputs' of the graph. pass else: if app not in self.debug_all_apps: raise ProtocolError("change without import") # UPDATE self.clients self.clients[old_r][app] -= 1 if self.clients[old_r][app] == 0: del self.clients[old_r][app] self.clients.setdefault(new_r, OrderedDict()).setdefault(app, 0) self.clients[new_r][app] += 1 # UPDATE self.view_i, self.view_o for o_idx, i_idx_list in getattr(app.op, "view_map", OrderedDict()).items(): if len(i_idx_list) > 1: # destroying this output invalidates multiple inputs raise NotImplementedError() i_idx = i_idx_list[0] output = app.outputs[o_idx] if i_idx == i: if app.inputs[i_idx] is not new_r: raise ProtocolError("wrong new_r on change") self.view_i[output] = new_r self.view_o[old_r].remove(output) if not self.view_o[old_r]: del self.view_o[old_r] self.view_o.setdefault(new_r, OrderedSet()).add(output) if self.algo == "fast": if app in self.fail_validate: del self.fail_validate[app] self.fast_destroy(app, reason) self.stale_droot = True def validate(self, fgraph): """ Return None. Raise InconsistencyError when a) orderings() raises an error b) orderings cannot be topologically sorted. """ if self.destroyers: if self.algo == "fast": if self.fail_validate: app_err_pairs = self.fail_validate self.fail_validate = OrderedDict() # self.fail_validate can only be a hint that maybe/probably # there is a cycle.This is because inside replace() we could # record many reasons to not accept a change, but we don't # know which one will fail first inside validate(). Thus,the # graph might have already changed when we raise the # self.fail_validate error. So before raising the error, we # double check here. for app in app_err_pairs: if app in fgraph.apply_nodes: self.fast_destroy(app, "validate") if self.fail_validate: self.fail_validate = app_err_pairs raise app_err_pairs[app] else: ords = self.orderings(fgraph, ordered=False) if _contains_cycle(fgraph, ords): raise InconsistencyError( "Dependency graph contains cycles") else: # James's Conjecture: # If there are no destructive ops, then there can be no cycles. # FB: This isn't always True. It can happened that # optimization introduce node that depend on itself. This # is very rare and should not happen in general. It will be # caught later. The error will be far from the source. But # doing this conjecture should speed up compilation most of # the time. The user should create such dependency except # if he mess too much with the internal. pass return True def orderings(self, fgraph, ordered=True): """ Return orderings induced by destructive operations. Raise InconsistencyError when a) attempting to destroy indestructable variable, or b) attempting to destroy a value multiple times, or c) an Apply destroys (illegally) one of its own inputs by aliasing """ if ordered: set_type = OrderedSet rval = OrderedDict() else: set_type = set rval = dict() if self.destroyers: # BUILD DATA STRUCTURES # CHECK for multiple destructions during construction of variables droot, impact, __ignore = self.refresh_droot_impact() # check for destruction of constants illegal_destroy = [ r for r in droot if getattr(r.tag, "indestructible", False) or isinstance(r, graph.Constant) ] if illegal_destroy: raise InconsistencyError( "Attempting to destroy indestructible variables: %s" % illegal_destroy) # add destroyed variable clients as computational dependencies for app in self.destroyers: # keep track of clients that should run before the current Apply root_clients = set_type() # for each destroyed input... for output_idx, input_idx_list in app.op.destroy_map.items(): destroyed_idx = input_idx_list[0] destroyed_variable = app.inputs[destroyed_idx] root = droot[destroyed_variable] root_impact = impact[root] # we generally want to put all clients of things which depend on root # as pre-requisites of app. # But, app is itself one such client! # App will always be a client of the node we're destroying # (destroyed_variable, but the tricky thing is when it is also a client of # *another variable* viewing on the root. Generally this is illegal, (e.g., # add_inplace(x, x.T). In some special cases though, the in-place op will # actually be able to work properly with multiple destroyed inputs (e.g, # add_inplace(x, x). An Op that can still work in this case should declare # so via the 'destroyhandler_tolerate_same' attribute or # 'destroyhandler_tolerate_aliased' attribute. # # destroyhandler_tolerate_same should be a list of pairs of the form # [(idx0, idx1), (idx0, idx2), ...] # The first element of each pair is the input index of a destroyed # variable. # The second element of each pair is the index of a different input where # we will permit exactly the same variable to appear. # For example, add_inplace.tolerate_same might be [(0,1)] if the destroyed # input is also allowed to appear as the second argument. # # destroyhandler_tolerate_aliased is the same sort of list of # pairs. # op.destroyhandler_tolerate_aliased = [(idx0, idx1)] tells the # destroyhandler to IGNORE an aliasing between a destroyed # input idx0 and another input idx1. # This is generally a bad idea, but it is safe in some # cases, such as # - the op reads from the aliased idx1 before modifying idx0 # - the idx0 and idx1 are guaranteed not to overlap (e.g. # they are pointed at different rows of a matrix). # # CHECK FOR INPUT ALIASING # OPT: pre-compute this on import tolerate_same = getattr(app.op, "destroyhandler_tolerate_same", []) assert isinstance(tolerate_same, list) tolerated = { idx1 for idx0, idx1 in tolerate_same if idx0 == destroyed_idx } tolerated.add(destroyed_idx) tolerate_aliased = getattr( app.op, "destroyhandler_tolerate_aliased", []) assert isinstance(tolerate_aliased, list) ignored = { idx1 for idx0, idx1 in tolerate_aliased if idx0 == destroyed_idx } for i, input in enumerate(app.inputs): if i in ignored: continue if input in root_impact and ( i not in tolerated or input is not destroyed_variable): raise InconsistencyError( "Input aliasing: %s (%i, %i)" % (app, destroyed_idx, i)) # add the rule: app must be preceded by all other Apply instances that # depend on destroyed_input for r in root_impact: assert not [ a for a, c in self.clients[r].items() if not c ] root_clients.update( [a for a, c in self.clients[r].items() if c]) # app itself is a client of the destroyed inputs, # but should not run before itself root_clients.remove(app) if root_clients: rval[app] = root_clients return rval
class BackgroundLogger(BaseLogger): """A logger which can be used from multiple threads. The messages from main thread will go to robot logging api (or Python logging if Robot is not running). Messages from other threads are saved to memory and can be later logged with ``log_background_messages()``. This will also remove the messages from memory. Example:: from robotbackgroundlogger import BackgroundLogger logger = BackgroundLogger() After that logger can be used mostly like ``robot.api.logger`:: logger.debug('Hello, world!') logger.info('<b>HTML</b> example', html=True) """ LOGGING_THREADS = logger.librarylogger.LOGGING_THREADS def __init__(self): self.lock = threading.RLock() self._messages = OrderedDict() def write(self, msg, level, html=False): with self.lock: thread = threading.currentThread().getName() if thread in self.LOGGING_THREADS: logger.write(msg, level, html) else: message = BackgroundMessage(msg, level, html) self._messages.setdefault(thread, []).append(message) def log_background_messages(self, name=None): """Forwards messages logged on background to Robot Framework log. By default forwards all messages logged by all threads, but can be limited to a certain thread by passing thread's name as an argument. Logged messages are removed from the message storage. This method must be called from the main thread. """ thread = threading.currentThread().getName() if thread not in self.LOGGING_THREADS: raise RuntimeError( "Logging background messages is only allowed from the main " "thread. Current thread name: %s" % thread) with self.lock: if name: self._log_messages_by_thread(name) else: self._log_all_messages() def _log_messages_by_thread(self, name): for message in self._messages.pop(name, []): print(message.format()) def _log_all_messages(self): for thread in list(self._messages): # Only way to get custom timestamps currently is with print print("*HTML* <b>Messages by '%s'</b>" % thread) for message in self._messages.pop(thread): print(message.format()) def reset_background_messages(self, name=None): with self.lock: if name: self._messages.pop(name) else: self._messages.clear()
def push_comic_book(self, username, user, book, opts=None): if not opts: opts = getOpts(user.device, "comic") oeb = CreateOeb(main.log, None, opts) pubtype = 'book:book:KindleEar' language = 'zh-cn' setMetaData( oeb, book.title, language, local_time("%Y-%m-%d", user.timezone), pubtype=pubtype, ) oeb.container = ServerContainer(main.log) #guide id_, href = oeb.manifest.generate('masthead', DEFAULT_MASTHEAD) # size:600*60 oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_MASTHEAD)) oeb.guide.add('masthead', 'Masthead Image', href) id_, href = oeb.manifest.generate('cover', DEFAULT_COVER) item = oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_COVER)) oeb.guide.add('cover', 'Cover', href) oeb.metadata.add('cover', id_) itemcnt, imgindex = 0, 0 sections = OrderedDict() toc_thumbnails = {} #map img-url -> manifest-href chapters = book.ParseFeedUrls() if not chapters: self.deliverlog( username, str(user.kindle_email), book.title, 0, status="nonews", tz=user.timezone, ) return for ( bookname, chapter_title, img_list, chapter_url, next_chapter_index, ) in chapters: try: image_count = 0 for ( mime_or_section, url, filename, content, brief, thumbnail, ) in book.gen_image_items(img_list, chapter_url): if not mime_or_section or not filename or not content: continue if mime_or_section.startswith(r"image/"): id_, href = oeb.manifest.generate(id="img", href=filename) item = oeb.manifest.add(id_, href, mime_or_section, data=content) if thumbnail: toc_thumbnails[url] = href else: sections.setdefault(mime_or_section, []) sections[mime_or_section].append( (filename, brief, thumbnail, content)) image_count += 1 title = book.title + " " + chapter_title if not image_count: self.deliverlog( username, str(user.kindle_email), title, 0, status="can't download image", tz=user.timezone, ) rs = "No new feeds.2" main.log.info(rs) continue insertHtmlToc = False insertThumbnail = False oeb.metadata.clear("title") oeb.metadata.add("title", title) InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc, insertThumbnail) oIO = byteStringIO() o = EPUBOutput() if user.book_type == "epub" else MOBIOutput() o.convert(oeb, oIO, opts, main.log) try: ultima_log = DeliverLog.all().order("-time").get() except: ultima_log = sorted(DeliverLog.all(), key=attrgetter("time"), reverse=True) ultima_log = ultima_log[0] if ultima_log else None if ultima_log: diff = datetime.datetime.utcnow() - ultima_log.datetime if diff.days * 86400 + diff.seconds < 10: time.sleep(8) self.SendToKindle( username, user.kindle_email, title, user.book_type, str(oIO.getvalue()), user.timezone, ) book.UpdateLastDelivered(bookname, chapter_title, next_chapter_index) rs = "%s(%s).%s Sent!" % ( title, local_time(tz=user.timezone), user.book_type, ) main.log.info(rs) except: main.log.exception(u"Failed to push {} {}".format( bookname, chapter_title))
def GET(self): username = web.input().get("u") bookid = web.input().get("id") feedsId = web.input().get("feedsId") user = KeUser.all().filter("name = ", username).get() if not user: return "User not exist!<br />" to = user.kindle_email if (';' in to) or (',' in to): to = to.replace(',', ';').replace(' ', '').split(';') to = list(filter(lambda x: x.find('@', 1, len(x) - 1) > 0, to)) #最简单的判断是否是EMAIL booktype = user.book_type #mobi,epub bookmode = user.book_mode or 'periodical' #periodical,comic titlefmt = user.titlefmt tz = user.timezone bookid = bookid.split(',') if ',' in bookid else [bookid] bks = [] for id_ in bookid: try: bks.append(Book.get_by_id(int(id_))) except: main.log.warn("Can't find book {}".format(id_)) continue #return "id of book is invalid or book not exist!<br />" # Deliver only some feeds in custom rss if feedsId: feedsId = [ int(item) for item in feedsId.split('|') if item.isdigit() ] feedsId = [ Feed.get_by_id(item) for item in feedsId if Feed.get_by_id(item) ] book4meta = None if len(bks) == 0: return "No have book to push!" elif len(bks) == 1: if bks[0].builtin: book4meta = BookClass(bks[0].title) mhfile = book4meta.mastheadfile coverfile = book4meta.coverfile if issubclass(book4meta, BaseComicBook ): #如果单独推送一个继承自BaseComicBook的书籍,则自动设置为漫画模式 bookmode = 'comic' else: #单独的推送自定义RSS book4meta = bks[0] mhfile = DEFAULT_MASTHEAD coverfile = DEFAULT_COVER else: #多本书合并推送时使用“自定义RSS”的元属性 book4meta = user.ownfeeds mhfile = DEFAULT_MASTHEAD coverfile = DEFAULT_COVER_BV if user.merge_books else DEFAULT_COVER if not book4meta: return "No have book to push.<br />" # 创建 OEB #global log opts = getOpts(user.device, bookmode) oeb = CreateOeb(main.log, None, opts) bookTitle = "%s %s" % (book4meta.title, local_time( titlefmt, tz)) if titlefmt else book4meta.title if bookmode == 'comic': pubtype = 'book:book:KindleEar' else: pubtype = 'periodical:magazine:KindleEar' setMetaData(oeb, bookTitle, book4meta.language, local_time("%Y-%m-%d", tz), pubtype=pubtype) oeb.container = ServerContainer(main.log) #guide if mhfile: id_, href = oeb.manifest.generate('masthead', mhfile) # size:600*60 oeb.manifest.add(id_, href, MimeFromFilename(mhfile)) oeb.guide.add('masthead', 'Masthead Image', href) if coverfile: imgData = None imgMime = '' #使用保存在数据库的用户上传的封面 if coverfile == DEFAULT_COVER and user.cover: imgData = user.cover imgMime = 'image/jpeg' #保存在数据库中的只可能是jpeg格式 elif callable(coverfile): #如果封面需要回调的话 try: imgData = book4meta().coverfile() if imgData: imgType = imghdr.what(None, imgData) if imgType: #如果是合法图片 imgMime = r"image/" + imgType else: main.log.warn( u'content of cover is invalid : [%s].' % bookTitle) imgData = None except: main.log.exception(u"Failed to fetch cover for book [%s]" % bookTitle) coverfile = DEFAULT_COVER imgData = None imgMime = '' if imgData and imgMime: id_, href = oeb.manifest.generate('cover', 'cover.jpg') item = oeb.manifest.add(id_, href, imgMime, data=imgData) else: id_, href = oeb.manifest.generate('cover', coverfile) item = oeb.manifest.add(id_, href, MimeFromFilename(coverfile)) oeb.guide.add('cover', 'Cover', href) oeb.metadata.add('cover', id_) elif len(bks) > 1 and DEFAULT_COVER: #将所有书籍的封面拼贴成一个 #如果DEFAULT_COVER=None说明用户不需要封面 id_, href = oeb.manifest.generate('cover', 'cover.jpg') item = oeb.manifest.add(id_, href, 'image/jpeg', data=self.MergeCovers(bks, opts, user)) oeb.guide.add('cover', 'Cover', href) oeb.metadata.add('cover', id_) itemcnt, imgindex = 0, 0 sections = OrderedDict() toc_thumbnails = {} #map img-url -> manifest-href for bk in bks: if bk.builtin: cbook = BookClass(bk.title) if not cbook: main.log.warn('not exist book <%s>' % bk.title) continue book = cbook(imgindex=imgindex, opts=opts, user=user) book.url_filters = [flt.url for flt in user.urlfilter] if bk.needs_subscription: #需要登录 subs_info = user.subscription_info(bk.title) if subs_info: book.account = subs_info.account book.password = subs_info.password if issubclass(cbook, BaseComicBook): self.push_comic_book(username, user, book) continue else: # 自定义RSS if bk.feedscount == 0: continue #return "the book has no feed!<br />" book = BaseFeedBook(imgindex=imgindex, opts=opts, user=user) book.title = bk.title book.description = bk.description book.language = bk.language book.keep_image = bk.keep_image book.oldest_article = bk.oldest_article book.fulltext_by_readability = True feeds = feedsId if feedsId else bk.feeds book.feeds = [] for feed in feeds: if feed.url.startswith(comic_domains): self.ProcessComicRSS(username, user, feed) else: book.feeds.append( (feed.title, feed.url, feed.isfulltext)) book.url_filters = [flt.url for flt in user.urlfilter] # 对于html文件,变量名字自文档,thumbnail为文章第一个img的url # 对于图片文件,section为图片mime,url为原始链接,title为文件名,content为二进制内容, # img的thumbail仅当其为article的第一个img为True # 对于CSS文件,sec_or_media 为 'text/css',url 和 title 都为文件名 try: #书的质量可能不一,一本书的异常不能影响其他书籍的推送 for sec_or_media, url, title, content, brief, thumbnail in book.Items( ): if not sec_or_media or not title or not content: continue if sec_or_media.startswith(r'image/'): id_, href = oeb.manifest.generate(id='img', href=title) item = oeb.manifest.add(id_, href, sec_or_media, data=content) if thumbnail: toc_thumbnails[url] = href imgindex += 1 elif sec_or_media == 'text/css': if url not in oeb.manifest.hrefs: #Only one css needed oeb.manifest.add('css', url, sec_or_media, data=content) else: #id, href = oeb.manifest.generate(id='feed', href='feed%d.html'%itemcnt) #item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content) #oeb.spine.add(item, True) sections.setdefault(sec_or_media, []) sections[sec_or_media].append( (title, brief, thumbnail, content)) itemcnt += 1 except Exception as e: main.log.exception(u"Failed to push <%s>, Err:%s" % (book.title, str(e))) continue volumeTitle = '' if itemcnt > 0: insertHtmlToc = GENERATE_HTML_TOC insertThumbnail = GENERATE_TOC_THUMBNAIL InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc, insertThumbnail) oIO = byteStringIO() o = EPUBOutput() if booktype == "epub" else MOBIOutput() o.convert(oeb, oIO, opts, main.log) try: ultima_log = DeliverLog.all().order('-time').get() except: ultima_log = sorted(DeliverLog.all(), key=attrgetter('time'), reverse=True) ultima_log = ultima_log[0] if ultima_log else None if ultima_log: diff = datetime.datetime.utcnow() - ultima_log.datetime if diff.days * 86400 + diff.seconds < 10: time.sleep(8) self.SendToKindle(username, to, book4meta.title + volumeTitle, booktype, str(oIO.getvalue()), tz) rs = "%s(%s).%s Sent!" % (book4meta.title, local_time(tz=tz), booktype) main.log.info(rs) return rs else: self.deliverlog(username, str(to), book4meta.title + volumeTitle, 0, status='nonews', tz=tz) rs = "No new feeds.1" main.log.info(rs) return rs
# Enter your code here. Read input from STDIN. Print output to STDOUT from collections import OrderedDict if __name__ == '__main__': # Takes less time to execute n = int(input()) dic = OrderedDict() for _ in range(n): s = str(input()) dic.setdefault(s, 0) dic[s] += 1 print(len(dic)) print(*dic.values(), end=' ') # Takes more time to execute arr = [] for _ in range(n): arr.append(str(input())) arr_wo_dup = [] for x in arr: if x not in arr_wo_dup: arr_wo_dup.append(x) print(len(arr_wo_dup)) occurance = []
def _op_to_template(op: BaseOp): """Generate template given an operator inherited from BaseOp.""" # NOTE in-place update to BaseOp # replace all PipelineParams with template var strings processed_op = _process_base_ops(op) if isinstance(op, dsl.ContainerOp): # default output artifacts output_artifact_paths = OrderedDict(op.output_artifact_paths) output_artifact_paths.setdefault('mlpipeline-ui-metadata', '/mlpipeline-ui-metadata.json') output_artifact_paths.setdefault('mlpipeline-metrics', '/mlpipeline-metrics.json') output_artifacts = [ _build_conventional_artifact(name, path) for name, path in output_artifact_paths.items() ] # workflow template template = { 'name': processed_op.name, 'container': K8sHelper.convert_k8s_obj_to_json( processed_op.container ) } elif isinstance(op, dsl.ResourceOp): # no output artifacts output_artifacts = [] # workflow template processed_op.resource["manifest"] = yaml.dump( K8sHelper.convert_k8s_obj_to_json(processed_op.k8s_resource), default_flow_style=False ) template = { 'name': processed_op.name, 'resource': K8sHelper.convert_k8s_obj_to_json( processed_op.resource ) } # inputs inputs = _inputs_to_json(processed_op.inputs) if inputs: template['inputs'] = inputs # outputs if isinstance(op, dsl.ContainerOp): param_outputs = processed_op.file_outputs elif isinstance(op, dsl.ResourceOp): param_outputs = processed_op.attribute_outputs template['outputs'] = _outputs_to_json(op, processed_op.outputs, param_outputs, output_artifacts) # node selector if processed_op.node_selector: template['nodeSelector'] = processed_op.node_selector # metadata if processed_op.pod_annotations or processed_op.pod_labels: template['metadata'] = {} if processed_op.pod_annotations: template['metadata']['annotations'] = processed_op.pod_annotations if processed_op.pod_labels: template['metadata']['labels'] = processed_op.pod_labels # retries if processed_op.num_retries: template['retryStrategy'] = {'limit': processed_op.num_retries} # sidecars if processed_op.sidecars: template['sidecars'] = processed_op.sidecars return template
def _intermediate_bubbles_2(cls, brackets): """Operates in-place. This implements the second intermediate bubbles method, where all debates in the same intermediate bubble have the same number of wins, but there might be multiple intermediate bubbles between brackets. """ new = OrderedDict() unfilled = OrderedDict() intermediates = OrderedDict( ) # Values are lists of {"aff", "neg"} dicts for points, pool in brackets.items(): to_delete_from_unfilled = [] # First, check for unfilled intermediate brackets for unfilled_points, unfilled_pool in unfilled.items(): intermediates.setdefault(unfilled_points, list()) if unfilled_pool["aff"] and unfilled_pool["neg"]: raise DrawFatalError( "An unfilled pool unexpectedly had both affirmative and negative teams." ) elif unfilled_pool["aff"]: # In a new bracket, take the lesser of how many excess affirmative # teams there are, and how many negative teams in the pool we have. num_teams = min(len(unfilled_pool["aff"]), len(pool["neg"])) intermediates[unfilled_points].append({ "aff": unfilled_pool["aff"][:num_teams], "neg": pool["neg"][:num_teams] }) del unfilled_pool["aff"][:num_teams] del pool["neg"][:num_teams] elif unfilled_pool["neg"]: # Take the top teams from affirmative pool as appropriate. num_teams = min(len(unfilled_pool["neg"]), len(pool["aff"])) intermediates[unfilled_points].append({ "aff": pool["aff"][:num_teams], "neg": unfilled_pool["neg"][:num_teams] }) del pool["aff"][:num_teams] del unfilled_pool["neg"][:num_teams] # If we've exhausted the unfilled pool, add all these # intermediate brackets to the main list of brackets and mark # them for deletion from the unfilled buffer. if not unfilled_pool["aff"] and not unfilled_pool["neg"]: num_brackets = len(intermediates[unfilled_points]) for i, intermediate_pool in enumerate( intermediates[unfilled_points], start=1): intermediate_points = unfilled_points - i / ( num_brackets + 1) new[intermediate_points] = intermediate_pool to_delete_from_unfilled.append(unfilled_points) # Delete the unfilled brackets tht were marked for deletion for unfilled_points in to_delete_from_unfilled: del unfilled[unfilled_points] # Find lesser and greater of number of aff and neg teams. nums_teams = list(map(len, list(pool.values()))) n = min(nums_teams) m = max(nums_teams) # Assign the main bracket new[points] = {"aff": pool["aff"][:n], "neg": pool["neg"][:n]} # Take note of the excess teams, if any if m > n: unfilled[points] = { "aff": pool["aff"][n:], "neg": pool["neg"][n:] } if unfilled: raise DrawFatalError( "There are still unfilled intermediate brackets!\n" + repr(unfilled)) # Currently, the brackets are out of order, since e.g. 3.5 would have been # inserted after 3 (or maybe even after 2). Let's change that: new_sorted = sorted(list(new.items()), key=lambda x: x[0], reverse=True) brackets.clear() brackets.update(new_sorted)
def _descriptor_components(raw_contents, validate, extra_keywords=(), non_ascii_fields=()): """ Initial breakup of the server descriptor contents to make parsing easier. A descriptor contains a series of 'keyword lines' which are simply a keyword followed by an optional value. Lines can also be followed by a signature block. To get a sub-listing with just certain keywords use extra_keywords. This can be useful if we care about their relative ordering with respect to each other. For instance, we care about the ordering of 'accept' and 'reject' entries because this influences the resulting exit policy, but for everything else in server descriptors the order does not matter. :param str raw_contents: descriptor content provided by the relay :param bool validate: checks the validity of the descriptor's content if True, skips these checks otherwise :param list extra_keywords: entity keywords to put into a separate listing with ordering intact :param list non_ascii_fields: fields containing non-ascii content :returns: **collections.OrderedDict** with the 'keyword => (value, pgp key) entries' mappings. If a extra_keywords was provided then this instead provides a two value tuple, the second being a list of those entries. """ if isinstance(raw_contents, bytes): raw_contents = stem.util.str_tools._to_unicode(raw_contents) entries = OrderedDict() extra_entries = [] # entries with a keyword in extra_keywords remaining_lines = raw_contents.split('\n') while remaining_lines: line = remaining_lines.pop(0) # V2 network status documents explicitly can contain blank lines... # # "Implementations MAY insert blank lines for clarity between sections; # these blank lines are ignored." # # ... and server descriptors end with an extra newline. But other documents # don't say how blank lines should be handled so globally ignoring them. if not line: continue # Some lines have an 'opt ' for backward compatibility. They should be # ignored. This prefix is being removed in... # https://trac.torproject.org/projects/tor/ticket/5124 if line.startswith('opt '): line = line[4:] line_match = KEYWORD_LINE.match(line) if not line_match: if not validate: continue raise ValueError('Line contains invalid characters: %s' % line) keyword, value = line_match.groups() if value is None: value = '' try: block_attr = _get_pseudo_pgp_block(remaining_lines) if block_attr: block_type, block_contents = block_attr else: block_type, block_contents = None, None except ValueError: if not validate: continue raise if validate and keyword not in non_ascii_fields: try: value.encode('ascii') except UnicodeError: replaced = ''.join([(char if char in string.printable else '?') for char in value]) raise ValueError("'%s' line had non-ascii content: %s" % (keyword, replaced)) if keyword in extra_keywords: extra_entries.append('%s %s' % (keyword, value)) else: entries.setdefault(keyword, []).append( (value, block_type, block_contents)) if extra_keywords: return entries, extra_entries else: return entries
import random import sys from collections import OrderedDict txt = "the theremin is theirs, ok? yes, it is. this is a theremin." #Your sample text goes here. order = 3 #You can change the order of the ngram here ngrams = [] count = OrderedDict() for i in range(0, len(txt) - order + 1): gram = txt[i:i + order] ngrams.append(gram) if gram not in count: count.setdefault(gram, []) if gram in ngrams: #count[gram][0]+=1 if i is not (len(txt) - order): count.setdefault(gram, []).append(txt[i + order]) currentGram = txt[0:order] for i in range(20): if (len(count[currentGram])) is 0: sys.exit(0) possibilities = random.choice(count[currentGram]) newGram = currentGram + possibilities print(newGram, end='') currentGram = newGram[-order:]
class Transient(Module): """Structure to store transient data.""" _REFERENCES = [{SOURCE.BIBCODE: '2017arXiv171002145G'}] _EX_REPS = {'rad': 'radio', 'xray': 'x-ray'} _OBS_KEYS = [ 'times', 'telescopes', 'systems', 'modes', 'instruments', 'bandsets', 'bands', 'frequencies', 'u_frequencies', 'zeropoints', 'measures' ] def __init__(self, **kwargs): """Initialize module.""" super(Transient, self).__init__(**kwargs) self._keys = kwargs.get('keys', '') self._data_determined_parameters = [] def process(self, **kwargs): """Process module.""" return self._data def set_data(self, all_data, req_key_values={}, subtract_minimum_keys=[], smooth_times=-1, extrapolate_time=0.0, limit_fitting_mjds=False, exclude_bands=[], exclude_instruments=[], exclude_systems=[], exclude_sources=[], exclude_kinds=[], time_unit=None, time_list=[], band_list=[], band_telescopes=[], band_systems=[], band_instruments=[], band_modes=[], band_bandsets=[]): """Set transient data.""" prt = self._printer self._all_data = all_data self._data = OrderedDict() if not self._all_data: return name = list(self._all_data.keys())[0] self._data['name'] = name numeric_keys = set() ex_kinds = [ self._EX_REPS.get(x.lower(), x.lower()) for x in exclude_kinds ] # Construct some source dictionaries for exclusion rules src_dict = OrderedDict() sources = self._all_data[name].get('sources', []) for src in sources: if SOURCE.BIBCODE in src: src_dict[src[SOURCE.ALIAS]] = src[SOURCE.BIBCODE] if SOURCE.ARXIVID in src: src_dict[src[SOURCE.ALIAS]] = src[SOURCE.ARXIVID] if SOURCE.NAME in src: src_dict[src[SOURCE.ALIAS]] = src[SOURCE.NAME] for key in self._keys: subkeys = self._keys[key] req_subkeys = [ x for x in subkeys if not isinstance(subkeys, dict) or 'required' in listify(subkeys[x]) ] num_subkeys = [ x for x in subkeys if 'numeric' in listify(subkeys[x]) ] boo_subkeys = [ x for x in subkeys if 'boolean' in listify(subkeys[x]) ] exc_subkeys = [ x for x in subkeys if 'exclude' in listify(subkeys[x]) ] if (key not in self._all_data[name] and not self._model.is_parameter_fixed_by_user(key)): if subkeys.get('value', None) == 'recommended': self._unset_recommended_keys.add(key) continue subdata = self._all_data[name].get(key) if subdata is None: continue # Only include data that contains all subkeys for entry in subdata: if any([x not in entry for x in req_subkeys]): continue if any([x in entry for x in exc_subkeys]): continue if any([ x in entry and ((isinstance(entry[x], list) and any([ not is_number(y) or np.isnan(float(y)) for y in entry[x] ])) or not isinstance(entry[x], list) and (not is_number(entry[x]) or np.isnan(float(entry[x])))) for x in num_subkeys ]): continue skip_key = False if 'frequency' not in entry: for qkey in req_key_values: if qkey in entry and entry[qkey] != '': if entry[qkey] not in req_key_values[qkey]: skip_key = True break if key == 'photometry': if ('fluxdensity' in entry and 'magnitude' not in entry and 'countrate' not in entry): self._kinds_needed.add('radio') if ('radio' in ex_kinds or (not len(ex_kinds) or 'none' not in ex_kinds) and 'radio' not in self._model._kinds_supported): continue if (('countrate' in entry or 'unabsorbedflux' in entry or 'flux' in entry) and 'magnitude' not in entry and 'fluxdensity' not in entry): self._kinds_needed.add('x-ray') if ('x-ray' in ex_kinds or (not len(ex_kinds) or 'none' not in ex_kinds) and 'x-ray' not in self._model._kinds_supported): continue if 'magnitude' in entry: # For now, magnitudes are not excludable. self._kinds_needed |= set( ['infrared', 'optical', 'ultraviolet']) skip_entry = False for x in subkeys: if limit_fitting_mjds is not False and x == 'time': val = np.mean([ float(x) for x in listify(entry.get(x, None)) ]) if (val < limit_fitting_mjds[0] or val > limit_fitting_mjds[1]): skip_entry = True break if exclude_bands is not False and x == 'band': if (entry.get(x, '') in exclude_bands and (not exclude_instruments or entry.get( 'instrument', '') in exclude_instruments) and (not exclude_systems or entry.get( 'system', '') in exclude_systems)): skip_entry = True break if (exclude_instruments is not False and x == 'instrument'): if (entry.get(x, '') in exclude_instruments and (not exclude_bands or entry.get('band', '') in exclude_bands) and (not exclude_systems or entry.get( 'system', '') in exclude_systems)): skip_entry = True break if (exclude_systems is not False and x == 'system'): if (entry.get(x, '') in exclude_systems and (not exclude_bands or entry.get('band', '') in exclude_bands) and (not exclude_instruments or entry.get( 'instrument', '') in exclude_instruments)): skip_entry = True break if (exclude_sources is not False and x == 'source'): val = entry.get(x, '') if (any( [x in exclude_sources for x in val.split(',')]) or any([ src_dict.get(x, '') in exclude_sources for x in val.split(',') ])): skip_entry = True break if skip_entry: continue if ((('magnitude' in entry) != ('band' in entry)) or ((('fluxdensity' in entry) != ('frequency' in entry)) and ('magnitude' not in entry)) or (('countrate' in entry) and ('magnitude' not in entry) and ('instrument' not in entry))): continue for x in subkeys: falseval = (False if x in boo_subkeys else None if x in num_subkeys else '') if x == 'value': if not skip_key: self._data[key] = entry.get(x, falseval) else: plural = self._model.plural(x) val = entry.get(x, falseval) if x in num_subkeys: val = None if val is None else np.mean( [float(x) for x in listify(val)]) if not skip_key: self._data.setdefault(plural, []).append(val) if x in num_subkeys: numeric_keys.add(plural) else: self._data.setdefault('unmatched_' + plural, []).append(val) if 'times' not in self._data or not any([ x in self._data for x in ['magnitudes', 'frequencies', 'countrates'] ]): prt.message('no_fittable_data', [name]) return False for key in list(self._data.keys()): if isinstance(self._data[key], list): self._data[key] = np.array(self._data[key]) if key not in numeric_keys: continue num_values = [ x for x in self._data[key] if isinstance(x, float) ] if len(num_values): self._data['min_' + key] = min(num_values) self._data['max_' + key] = max(num_values) else: if is_number(self._data[key]): self._data[key] = float(self._data[key]) self._data_determined_parameters.append(key) if any(x in self._data for x in ['magnitudes', 'countrates', 'fluxdensities']): # Add a list of tags for each observation to indicate what unit # observation is provided in. self._data['measures'] = [ ((['magnitude'] if x else []) + (['countrate'] if y else []) + (['fluxdensity'] if x else [])) for x, y, z in zip(*(self._data['magnitudes'], self._data['countrates'], self._data['fluxdensities'])) ] if 'times' in self._data and (smooth_times >= 0 or time_list): # Build an observation array out of the real data first. obs = list( zip(*(self._data[x] for x in self._OBS_KEYS if x != 'times'))) # Append extra observations if requested. if len(band_list): b_teles = band_telescopes if len(band_telescopes) == len( band_list) else ( [band_telescopes[0] for x in band_list] if len(band_telescopes) else ['' for x in band_list]) b_systs = band_systems if len(band_systems) == len( band_list) else ([band_systems[0] for x in band_list] if len(band_systems) else ['' for x in band_list]) b_modes = band_modes if len(band_modes) == len( band_list) else ([band_modes[0] for x in band_list] if len(band_modes) else ['' for x in band_list]) b_insts = band_instruments if len(band_instruments) == len( band_list) else ( [band_instruments[0] for x in band_list] if len(band_instruments) else ['' for x in band_list]) b_bsets = band_bandsets if len(band_bandsets) == len( band_list) else ([band_bandsets[0] for x in band_list] if len(band_bandsets) else ['' for x in band_list]) b_freqs = [None for x in band_list] b_u_freqs = ['' for x in band_list] b_zerops = [None for x in band_list] b_measures = [[] for x in band_list] obs.extend( list( zip(*(b_teles, b_systs, b_modes, b_insts, b_bsets, band_list, b_freqs, b_u_freqs, b_zerops, b_measures)))) # Prune extra observations if they are duplicitous to existing. uniqueobs = [] for o in obs: to = tuple(o) if to not in uniqueobs: uniqueobs.append(to) # Preprend times to real observations list. minet, maxet = (extrapolate_time, extrapolate_time) if isinstance( extrapolate_time, (float, int)) else ( (tuple(extrapolate_time) if len(extrapolate_time) == 2 else (extrapolate_time[0], extrapolate_time[0]))) mint, maxt = (min(self._data['times']) - minet, max(self._data['times']) + maxet) if time_unit is None: alltimes = time_list + [x for x in self._data['times']] elif time_unit == 'mjd': alltimes = [x - min(self._data['times']) for x in time_list ] + [x for x in self._data['times']] elif time_unit == 'phase': if 'maxdate' not in self._data: raise (prt.message('no_maxdate', name)) max_mjd = astrotime(self._data['maxdate'].replace('/', '-')).mjd alltimes = [x + max_mjd for x in time_list ] + [x for x in self._data['times']] else: raise ('Unknown `time_unit`.') if smooth_times >= 0: alltimes += list(np.linspace(mint, maxt, max(smooth_times, 2))) alltimes = list(sorted(set(alltimes))) # Create additional fake observations. currobslist = list(zip(*(self._data[x] for x in self._OBS_KEYS))) obslist = [] for ti, t in enumerate(alltimes): new_per = np.round(100.0 * float(ti) / len(alltimes), 1) prt.message('construct_obs_array', [new_per], inline=True) for o in uniqueobs: newobs = tuple([t] + list(o)) if newobs not in currobslist: obslist.append(newobs) obslist.sort() # Save these fake observations under keys with `extra_` prefix. if len(obslist): for x, y in zip(self._OBS_KEYS, zip(*obslist)): self._data['extra_' + x] = y for qkey in subtract_minimum_keys: if 'upperlimits' in self._data: new_vals = np.array(self._data[qkey])[np.array( self._data['upperlimits']) != True] # noqa E712 if len(new_vals): self._data['min_' + qkey] = min(new_vals) self._data['max_' + qkey] = max(new_vals) minv = self._data['min_' + qkey] self._data[qkey] = [x - minv for x in self._data[qkey]] if 'extra_' + qkey in self._data: self._data['extra_' + qkey] = [ x - minv for x in self._data['extra_' + qkey] ] return True def get_data_determined_parameters(self): """Return list of parameters determined by data.""" return self._data_determined_parameters def send_request(self, request): """Send requests to other modules.""" if request == 'min_times': return self._data['min_times'] return []
def _draw_text_as_text(self, gc, x, y, s, prop, angle, ismath, mtext=None): writer = self.writer color = rgb2hex(gc.get_rgb()) style = {} if color != '#000000': style['fill'] = color if gc.get_alpha() != 1.0: style['opacity'] = short_float_fmt(gc.get_alpha()) if not ismath: font = self._get_font(prop) font.set_text(s, 0.0, flags=LOAD_NO_HINTING) fontsize = prop.get_size_in_points() fontfamily = font.family_name fontstyle = prop.get_style() attrib = {} # Must add "px" to workaround a Firefox bug style['font-size'] = short_float_fmt(fontsize) + 'px' style['font-family'] = six.text_type(fontfamily) style['font-style'] = prop.get_style().lower() style['font-weight'] = six.text_type(prop.get_weight()).lower() attrib['style'] = generate_css(style) if mtext and (angle == 0 or mtext.get_rotation_mode() == "anchor"): # If text anchoring can be supported, get the original # coordinates and add alignment information. # Get anchor coordinates. transform = mtext.get_transform() ax, ay = transform.transform_point(mtext.get_position()) ay = self.height - ay # Don't do vertical anchor alignment. Most applications do not # support 'alignment-baseline' yet. Apply the vertical layout # to the anchor point manually for now. angle_rad = np.deg2rad(angle) dir_vert = np.array([np.sin(angle_rad), np.cos(angle_rad)]) v_offset = np.dot(dir_vert, [(x - ax), (y - ay)]) ax = ax + v_offset * dir_vert[0] ay = ay + v_offset * dir_vert[1] ha_mpl_to_svg = { 'left': 'start', 'right': 'end', 'center': 'middle' } style['text-anchor'] = ha_mpl_to_svg[mtext.get_ha()] attrib['x'] = short_float_fmt(ax) attrib['y'] = short_float_fmt(ay) attrib['style'] = generate_css(style) attrib['transform'] = "rotate(%s, %s, %s)" % (short_float_fmt( -angle), short_float_fmt(ax), short_float_fmt(ay)) writer.element('text', s, attrib=attrib) else: attrib['transform'] = generate_transform([ ('translate', (x, y)), ('rotate', (-angle, )) ]) writer.element('text', s, attrib=attrib) if rcParams['svg.fonttype'] == 'svgfont': fontset = self._fonts.setdefault(font.fname, set()) for c in s: fontset.add(ord(c)) else: writer.comment(s) width, height, descent, svg_elements, used_characters = \ self.mathtext_parser.parse(s, 72, prop) svg_glyphs = svg_elements.svg_glyphs svg_rects = svg_elements.svg_rects attrib = {} attrib['style'] = generate_css(style) attrib['transform'] = generate_transform([('translate', (x, y)), ('rotate', (-angle, ))]) # Apply attributes to 'g', not 'text', because we likely # have some rectangles as well with the same style and # transformation writer.start('g', attrib=attrib) writer.start('text') # Sort the characters by font, and output one tspan for # each spans = OrderedDict() for font, fontsize, thetext, new_x, new_y, metrics in svg_glyphs: style = generate_css({ 'font-size': short_float_fmt(fontsize) + 'px', 'font-family': font.family_name, 'font-style': font.style_name.lower(), 'font-weight': font.style_name.lower() }) if thetext == 32: thetext = 0xa0 # non-breaking space spans.setdefault(style, []).append((new_x, -new_y, thetext)) if rcParams['svg.fonttype'] == 'svgfont': for font, fontsize, thetext, new_x, new_y, metrics in svg_glyphs: fontset = self._fonts.setdefault(font.fname, set()) fontset.add(thetext) for style, chars in six.iteritems(spans): chars.sort() same_y = True if len(chars) > 1: last_y = chars[0][1] for i in xrange(1, len(chars)): if chars[i][1] != last_y: same_y = False break if same_y: ys = six.text_type(chars[0][1]) else: ys = ' '.join(six.text_type(c[1]) for c in chars) attrib = { 'style': style, 'x': ' '.join(short_float_fmt(c[0]) for c in chars), 'y': ys } writer.element('tspan', ''.join(unichr(c[2]) for c in chars), attrib=attrib) writer.end('text') if len(svg_rects): for x, y, width, height in svg_rects: writer.element('rect', x=short_float_fmt(x), y=short_float_fmt(-y + height), width=short_float_fmt(width), height=short_float_fmt(height)) writer.end('g')
class GeneData: def __init__(self, genes_hdf5_file, worker_index=None, workers=None, read_1hot=True, read_targets=True): # open HDF5 self.genes_hdf5_in = h5py.File(genes_hdf5_file) # simple stats self.num_seqs, self.seq_length, self.seq_depth = self.genes_hdf5_in[ 'seqs_1hot'].shape self.pool_width = int(np.array(self.genes_hdf5_in['pool_width'])) ######################################### # gene sequences self.gene_seqs = [] for si in range(len(self.genes_hdf5_in['seq_chrom'])): gene_seq = GeneSeq( self.genes_hdf5_in['seq_chrom'][si].decode('UTF-8'), self.genes_hdf5_in['seq_start'][si], self.genes_hdf5_in['seq_end'][si]) self.gene_seqs.append(gene_seq) if read_1hot: self.seqs_1hot = np.array(self.genes_hdf5_in['seqs_1hot']) else: self.seqs_1hot = None ######################################### # TSS information self.tss = [] for tss_i in range(len(self.genes_hdf5_in['tss_id'])): # map to gene seq seq_i = self.genes_hdf5_in['tss_seq'][tss_i] tss_seq = self.gene_seqs[seq_i] # read in TSS tss = TSS(self.genes_hdf5_in['tss_id'][tss_i].decode('UTF-8'), self.genes_hdf5_in['tss_gene'][tss_i].decode('UTF-8'), self.genes_hdf5_in['tss_chrom'][tss_i].decode('UTF-8'), self.genes_hdf5_in['tss_pos'][tss_i], tss_seq) # tss_seq, # self.genes_hdf5_in['tss_strand'][tss_i].decode('UTF-8')) self.tss.append(tss) # append to GeneSeq tss_seq.append_tss(tss) self.num_tss = len(self.tss) ######################################### # gene information self.gene_tss = OrderedDict() self.gene_index = OrderedDict() for tss_i in range(len(self.tss)): gene_id = self.tss[tss_i].gene_id self.gene_tss.setdefault(gene_id, []).append(tss_i) if gene_id not in self.gene_index: self.gene_index[gene_id] = len(self.gene_index) self.num_genes = len(self.gene_tss) ######################################### # determine genes split across sequences gene_seqs = {} for seq_i in range(self.num_seqs): for tss in self.gene_seqs[seq_i].tss_list: gene_seqs.setdefault(tss.gene_id, set()).add(seq_i) self.multi_seq_genes = set() for gene_id in gene_seqs: if len(gene_seqs[gene_id]) > 1: self.multi_seq_genes.add(gene_id) ######################################### # target information if 'tss_targets' in self.genes_hdf5_in and read_targets: self.tss_targets = self.genes_hdf5_in['tss_targets'] self.target_labels = [ tl.decode('UTF-8') for tl in self.genes_hdf5_in['target_labels'] ] if 'target_ids' in self.genes_hdf5_in: # TEMP self.target_ids = [ tl.decode('UTF-8') for tl in self.genes_hdf5_in['target_ids'] ] else: self.target_ids = [''] * len(self.target_labels) self.num_targets = len(self.target_labels) else: self.tss_targets = None self.target_ids = None self.target_labels = None self.num_targets = None def gene_ids(self): """ Return a list of gene identifiers """ return list(self.gene_tss.keys()) def subset_genes(self, gene_ids): ''' Limit the sequences to a subset containing the given transcripts. ''' if type(gene_ids) != set: gene_ids = set(gene_ids) seq_mask = np.zeros(self.num_seqs, dtype='bool') tss_mask = [] for si in range(self.num_seqs): # determine TSSs matching given genes. seq_tss_list = [ tss for tss in self.gene_seqs[si].tss_list if tss.gene_id in gene_ids ] seq_tss_mask = [ tss.gene_id in gene_ids for tss in self.gene_seqs[si].tss_list ] tss_mask += seq_tss_mask # filter TSSs to those matching given genes. if len(seq_tss_list) > 0: seq_mask[si] = True self.gene_seqs[si].tss_list = seq_tss_list # filter sequences for those with a match self.gene_seqs = [ self.gene_seqs[si] for si in range(self.num_seqs) if seq_mask[si] ] if self.seqs_1hot is not None: self.seqs_1hot = self.seqs_1hot[seq_mask] self.num_seqs = len(self.gene_seqs) if self.tss_targets is not None: tss_mask = np.array(tss_mask, dtype='bool') self.tss_targets = self.tss_targets[tss_mask, :] def tss_ids(self): """ Return a list of TSS identifiers """ return [tss.identifier for tss in self.tss] def worker(self, wi, worker_num): """ Limit the sequences to one worker's share. """ worker_mask = np.array( [si % worker_num == wi for si in range(self.num_seqs)]) # filter gene sequences if self.seqs_1hot is not None: self.seqs_1hot = self.seqs_1hot[worker_mask, :, :] self.gene_seqs = [ self.gene_seqs[si] for si in range(self.num_seqs) if worker_mask[si] ] self.num_seqs = len(self.gene_seqs) # hash remaining gene sequences seq_hash = set() for gene_seq in self.gene_seqs: seq_hash.add((gene_seq.chrom, gene_seq.start)) # filter TSSs worker_tss = [] for tss in self.tss: tss_seq_key = (tss.gene_seq.chrom, tss.gene_seq.start) if tss_seq_key in seq_hash: worker_tss.append(tss) self.tss = worker_tss # tss_targets isn't relevant for SED def __exit__(self): # close HDF5 self.genes_hdf5_in.close()
def handle(self, *app_labels, **options): format = options.get('format') indent = options.get('indent') using = options.get('database') excludes = options.get('exclude') output = options.get('output') show_traceback = options.get('traceback') use_natural_foreign_keys = options.get('use_natural_foreign_keys') use_natural_primary_keys = options.get('use_natural_primary_keys') use_base_manager = options.get('use_base_manager') pks = options.get('primary_keys') if pks: primary_keys = pks.split(',') else: primary_keys = [] excluded_apps = set() excluded_models = set() for exclude in excludes: if '.' in exclude: try: model = apps.get_model(exclude) except LookupError: raise CommandError('Unknown model in excludes: %s' % exclude) excluded_models.add(model) else: try: app_config = apps.get_app_config(exclude) except LookupError: raise CommandError('Unknown app in excludes: %s' % exclude) excluded_apps.add(app_config) if len(app_labels) == 0: if primary_keys: raise CommandError( "You can only use --pks option with one model") app_list = OrderedDict((app_config, None) for app_config in apps.get_app_configs() if app_config.models_module is not None and app_config not in excluded_apps) else: if len(app_labels) > 1 and primary_keys: raise CommandError( "You can only use --pks option with one model") app_list = OrderedDict() for label in app_labels: try: app_label, model_label = label.split('.') try: app_config = apps.get_app_config(app_label) except LookupError: raise CommandError("Unknown application: %s" % app_label) if app_config.models_module is None or app_config in excluded_apps: continue try: model = app_config.get_model(model_label) except LookupError: raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) app_list_value = app_list.setdefault(app_config, []) # We may have previously seen a "all-models" request for # this app (no model qualifier was given). In this case # there is no need adding specific models to the list. if app_list_value is not None: if model not in app_list_value: app_list_value.append(model) except ValueError: if primary_keys: raise CommandError( "You can only use --pks option with one model") # This is just an app - no model qualifier app_label = label try: app_config = apps.get_app_config(app_label) except LookupError: raise CommandError("Unknown application: %s" % app_label) if app_config.models_module is None or app_config in excluded_apps: continue app_list[app_config] = None # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): try: serializers.get_serializer(format) except serializers.SerializerDoesNotExist: pass raise CommandError("Unknown serialization format: %s" % format) def get_objects(): # Collate the objects to be serialized. for model in serializers.sort_dependencies(app_list.items()): if model in excluded_models: continue if not model._meta.proxy and router.allow_migrate( using, model): if use_base_manager: objects = model._base_manager else: objects = model._default_manager queryset = objects.using(using).order_by( model._meta.pk.name) if primary_keys: queryset = queryset.filter(pk__in=primary_keys) for obj in queryset.iterator(): yield obj try: self.stdout.ending = None stream = open(output, 'w') if output else None try: serializers.serialize( format, get_objects(), indent=indent, use_natural_foreign_keys=use_natural_foreign_keys, use_natural_primary_keys=use_natural_primary_keys, stream=stream or self.stdout) finally: if stream: stream.close() except Exception as e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
class RendererSVG(RendererBase): FONT_SCALE = 100.0 fontd = maxdict(50) def __init__(self, width, height, svgwriter, basename=None, image_dpi=72): self.width = width self.height = height self.writer = XMLWriter(svgwriter) self.image_dpi = image_dpi # the actual dpi we want to rasterize stuff with self._groupd = {} if not rcParams['svg.image_inline']: assert basename is not None self.basename = basename self._imaged = {} self._clipd = OrderedDict() self._char_defs = {} self._markers = {} self._path_collection_id = 0 self._imaged = {} self._hatchd = OrderedDict() self._has_gouraud = False self._n_gradients = 0 self._fonts = OrderedDict() self.mathtext_parser = MathTextParser('SVG') RendererBase.__init__(self) self._glyph_map = dict() str_height = short_float_fmt(height) str_width = short_float_fmt(width) svgwriter.write(svgProlog) self._start_id = self.writer.start( 'svg', width='%spt' % str_width, height='%spt' % str_height, viewBox='0 0 %s %s' % (str_width, str_height), xmlns="http://www.w3.org/2000/svg", version="1.1", attrib={'xmlns:xlink': "http://www.w3.org/1999/xlink"}) self._write_default_style() def finalize(self): self._write_clips() self._write_hatches() self._write_svgfonts() self.writer.close(self._start_id) self.writer.flush() def _write_default_style(self): writer = self.writer default_style = generate_css({ 'stroke-linejoin': 'round', 'stroke-linecap': 'butt' }) writer.start('defs') writer.start('style', type='text/css') writer.data('*{%s}\n' % default_style) writer.end('style') writer.end('defs') def _make_id(self, type, content): content = str(content) if rcParams['svg.hashsalt'] is None: salt = str(uuid.uuid4()) else: salt = rcParams['svg.hashsalt'] if six.PY3: content = content.encode('utf8') salt = salt.encode('utf8') m = hashlib.md5() m.update(salt) m.update(content) return '%s%s' % (type, m.hexdigest()[:10]) def _make_flip_transform(self, transform): return (transform + Affine2D().scale(1.0, -1.0).translate(0.0, self.height)) def _get_font(self, prop): fname = findfont(prop) font = get_font(fname) font.clear() size = prop.get_size_in_points() font.set_size(size, 72.0) return font def _get_hatch(self, gc, rgbFace): """ Create a new hatch pattern """ if rgbFace is not None: rgbFace = tuple(rgbFace) edge = gc.get_hatch_color() if edge is not None: edge = tuple(edge) dictkey = (gc.get_hatch(), rgbFace, edge) oid = self._hatchd.get(dictkey) if oid is None: oid = self._make_id('h', dictkey) self._hatchd[dictkey] = ((gc.get_hatch_path(), rgbFace, edge), oid) else: _, oid = oid return oid def _write_hatches(self): if not len(self._hatchd): return HATCH_SIZE = 72 writer = self.writer writer.start('defs') for ((path, face, stroke), oid) in six.itervalues(self._hatchd): writer.start('pattern', id=oid, patternUnits="userSpaceOnUse", x="0", y="0", width=six.text_type(HATCH_SIZE), height=six.text_type(HATCH_SIZE)) path_data = self._convert_path(path, Affine2D().scale(HATCH_SIZE).scale( 1.0, -1.0).translate(0, HATCH_SIZE), simplify=False) if face is None: fill = 'none' else: fill = rgb2hex(face) writer.element('rect', x="0", y="0", width=six.text_type(HATCH_SIZE + 1), height=six.text_type(HATCH_SIZE + 1), fill=fill) writer.element('path', d=path_data, style=generate_css({ 'fill': rgb2hex(stroke), 'stroke': rgb2hex(stroke), 'stroke-width': six.text_type(rcParams['hatch.linewidth']), 'stroke-linecap': 'butt', 'stroke-linejoin': 'miter' })) writer.end('pattern') writer.end('defs') def _get_style_dict(self, gc, rgbFace): """ return the style string. style is generated from the GraphicsContext and rgbFace """ attrib = {} forced_alpha = gc.get_forced_alpha() if gc.get_hatch() is not None: attrib['fill'] = "url(#%s)" % self._get_hatch(gc, rgbFace) if rgbFace is not None and len( rgbFace) == 4 and rgbFace[3] != 1.0 and not forced_alpha: attrib['fill-opacity'] = short_float_fmt(rgbFace[3]) else: if rgbFace is None: attrib['fill'] = 'none' else: if tuple(rgbFace[:3]) != (0, 0, 0): attrib['fill'] = rgb2hex(rgbFace) if len(rgbFace ) == 4 and rgbFace[3] != 1.0 and not forced_alpha: attrib['fill-opacity'] = short_float_fmt(rgbFace[3]) if forced_alpha and gc.get_alpha() != 1.0: attrib['opacity'] = short_float_fmt(gc.get_alpha()) offset, seq = gc.get_dashes() if seq is not None: attrib['stroke-dasharray'] = ','.join( [short_float_fmt(val) for val in seq]) attrib['stroke-dashoffset'] = short_float_fmt(float(offset)) linewidth = gc.get_linewidth() if linewidth: rgb = gc.get_rgb() attrib['stroke'] = rgb2hex(rgb) if not forced_alpha and rgb[3] != 1.0: attrib['stroke-opacity'] = short_float_fmt(rgb[3]) if linewidth != 1.0: attrib['stroke-width'] = short_float_fmt(linewidth) if gc.get_joinstyle() != 'round': attrib['stroke-linejoin'] = gc.get_joinstyle() if gc.get_capstyle() != 'butt': attrib['stroke-linecap'] = _capstyle_d[gc.get_capstyle()] return attrib def _get_style(self, gc, rgbFace): return generate_css(self._get_style_dict(gc, rgbFace)) def _get_clip(self, gc): cliprect = gc.get_clip_rectangle() clippath, clippath_trans = gc.get_clip_path() if clippath is not None: clippath_trans = self._make_flip_transform(clippath_trans) dictkey = (id(clippath), str(clippath_trans)) elif cliprect is not None: x, y, w, h = cliprect.bounds y = self.height - (y + h) dictkey = (x, y, w, h) else: return None clip = self._clipd.get(dictkey) if clip is None: oid = self._make_id('p', dictkey) if clippath is not None: self._clipd[dictkey] = ((clippath, clippath_trans), oid) else: self._clipd[dictkey] = (dictkey, oid) else: clip, oid = clip return oid def _write_clips(self): if not len(self._clipd): return writer = self.writer writer.start('defs') for clip, oid in six.itervalues(self._clipd): writer.start('clipPath', id=oid) if len(clip) == 2: clippath, clippath_trans = clip path_data = self._convert_path(clippath, clippath_trans, simplify=False) writer.element('path', d=path_data) else: x, y, w, h = clip writer.element('rect', x=short_float_fmt(x), y=short_float_fmt(y), width=short_float_fmt(w), height=short_float_fmt(h)) writer.end('clipPath') writer.end('defs') def _write_svgfonts(self): if not rcParams['svg.fonttype'] == 'svgfont': return writer = self.writer writer.start('defs') for font_fname, chars in six.iteritems(self._fonts): font = get_font(font_fname) font.set_size(72, 72) sfnt = font.get_sfnt() writer.start('font', id=sfnt[1, 0, 0, 4].decode("mac_roman")) writer.element( 'font-face', attrib={ 'font-family': font.family_name, 'font-style': font.style_name.lower(), 'units-per-em': '72', 'bbox': ' '.join(short_float_fmt(x / 64.0) for x in font.bbox) }) for char in chars: glyph = font.load_char(char, flags=LOAD_NO_HINTING) verts, codes = font.get_path() path = Path(verts, codes) path_data = self._convert_path(path) # name = font.get_glyph_name(char) writer.element( 'glyph', d=path_data, attrib={ # 'glyph-name': name, 'unicode': unichr(char), 'horiz-adv-x': short_float_fmt(glyph.linearHoriAdvance / 65536.0) }) writer.end('font') writer.end('defs') def open_group(self, s, gid=None): """ Open a grouping element with label *s*. If *gid* is given, use *gid* as the id of the group. """ if gid: self.writer.start('g', id=gid) else: self._groupd[s] = self._groupd.get(s, 0) + 1 self.writer.start('g', id="%s_%d" % (s, self._groupd[s])) def close_group(self, s): self.writer.end('g') def option_image_nocomposite(self): """ return whether to generate a composite image from multiple images on a set of axes """ return not rcParams['image.composite_image'] def _convert_path(self, path, transform=None, clip=None, simplify=None, sketch=None): if clip: clip = (0.0, 0.0, self.width, self.height) else: clip = None return _path.convert_to_string(path, transform, clip, simplify, sketch, 6, [b'M', b'L', b'Q', b'C', b'z'], False).decode('ascii') def draw_path(self, gc, path, transform, rgbFace=None): trans_and_flip = self._make_flip_transform(transform) clip = (rgbFace is None and gc.get_hatch_path() is None) simplify = path.should_simplify and clip path_data = self._convert_path(path, trans_and_flip, clip=clip, simplify=simplify, sketch=gc.get_sketch_params()) attrib = {} attrib['style'] = self._get_style(gc, rgbFace) clipid = self._get_clip(gc) if clipid is not None: attrib['clip-path'] = 'url(#%s)' % clipid if gc.get_url() is not None: self.writer.start('a', {'xlink:href': gc.get_url()}) self.writer.element('path', d=path_data, attrib=attrib) if gc.get_url() is not None: self.writer.end('a') def draw_markers(self, gc, marker_path, marker_trans, path, trans, rgbFace=None): if not len(path.vertices): return writer = self.writer path_data = self._convert_path(marker_path, marker_trans + Affine2D().scale(1.0, -1.0), simplify=False) style = self._get_style_dict(gc, rgbFace) dictkey = (path_data, generate_css(style)) oid = self._markers.get(dictkey) style = generate_css( {k: v for k, v in six.iteritems(style) if k.startswith('stroke')}) if oid is None: oid = self._make_id('m', dictkey) writer.start('defs') writer.element('path', id=oid, d=path_data, style=style) writer.end('defs') self._markers[dictkey] = oid attrib = {} clipid = self._get_clip(gc) if clipid is not None: attrib['clip-path'] = 'url(#%s)' % clipid writer.start('g', attrib=attrib) trans_and_flip = self._make_flip_transform(trans) attrib = {'xlink:href': '#%s' % oid} clip = (0, 0, self.width * 72, self.height * 72) for vertices, code in path.iter_segments(trans_and_flip, clip=clip, simplify=False): if len(vertices): x, y = vertices[-2:] attrib['x'] = short_float_fmt(x) attrib['y'] = short_float_fmt(y) attrib['style'] = self._get_style(gc, rgbFace) writer.element('use', attrib=attrib) writer.end('g') def draw_path_collection(self, gc, master_transform, paths, all_transforms, offsets, offsetTrans, facecolors, edgecolors, linewidths, linestyles, antialiaseds, urls, offset_position): # Is the optimization worth it? Rough calculation: # cost of emitting a path in-line is # (len_path + 5) * uses_per_path # cost of definition+use is # (len_path + 3) + 9 * uses_per_path len_path = len(paths[0].vertices) if len(paths) > 0 else 0 uses_per_path = self._iter_collection_uses_per_path( paths, all_transforms, offsets, facecolors, edgecolors) should_do_optimization = \ len_path + 9 * uses_per_path + 3 < (len_path + 5) * uses_per_path if not should_do_optimization: return RendererBase.draw_path_collection( self, gc, master_transform, paths, all_transforms, offsets, offsetTrans, facecolors, edgecolors, linewidths, linestyles, antialiaseds, urls, offset_position) writer = self.writer path_codes = [] writer.start('defs') for i, (path, transform) in enumerate( self._iter_collection_raw_paths(master_transform, paths, all_transforms)): transform = Affine2D(transform.get_matrix()).scale(1.0, -1.0) d = self._convert_path(path, transform, simplify=False) oid = 'C%x_%x_%s' % (self._path_collection_id, i, self._make_id('', d)) writer.element('path', id=oid, d=d) path_codes.append(oid) writer.end('defs') for xo, yo, path_id, gc0, rgbFace in self._iter_collection( gc, master_transform, all_transforms, path_codes, offsets, offsetTrans, facecolors, edgecolors, linewidths, linestyles, antialiaseds, urls, offset_position): clipid = self._get_clip(gc0) url = gc0.get_url() if url is not None: writer.start('a', attrib={'xlink:href': url}) if clipid is not None: writer.start('g', attrib={'clip-path': 'url(#%s)' % clipid}) attrib = { 'xlink:href': '#%s' % path_id, 'x': short_float_fmt(xo), 'y': short_float_fmt(self.height - yo), 'style': self._get_style(gc0, rgbFace) } writer.element('use', attrib=attrib) if clipid is not None: writer.end('g') if url is not None: writer.end('a') self._path_collection_id += 1 def draw_gouraud_triangle(self, gc, points, colors, trans): # This uses a method described here: # # http://www.svgopen.org/2005/papers/Converting3DFaceToSVG/index.html # # that uses three overlapping linear gradients to simulate a # Gouraud triangle. Each gradient goes from fully opaque in # one corner to fully transparent along the opposite edge. # The line between the stop points is perpendicular to the # opposite edge. Underlying these three gradients is a solid # triangle whose color is the average of all three points. writer = self.writer if not self._has_gouraud: self._has_gouraud = True writer.start('filter', id='colorAdd') writer.element('feComposite', attrib={'in': 'SourceGraphic'}, in2='BackgroundImage', operator='arithmetic', k2="1", k3="1") writer.end('filter') avg_color = np.sum(colors[:, :], axis=0) / 3.0 # Just skip fully-transparent triangles if avg_color[-1] == 0.0: return trans_and_flip = self._make_flip_transform(trans) tpoints = trans_and_flip.transform(points) writer.start('defs') for i in range(3): x1, y1 = tpoints[i] x2, y2 = tpoints[(i + 1) % 3] x3, y3 = tpoints[(i + 2) % 3] c = colors[i][:] if x2 == x3: xb = x2 yb = y1 elif y2 == y3: xb = x1 yb = y2 else: m1 = (y2 - y3) / (x2 - x3) b1 = y2 - (m1 * x2) m2 = -(1.0 / m1) b2 = y1 - (m2 * x1) xb = (-b1 + b2) / (m1 - m2) yb = m2 * xb + b2 writer.start('linearGradient', id="GR%x_%d" % (self._n_gradients, i), x1=short_float_fmt(x1), y1=short_float_fmt(y1), x2=short_float_fmt(xb), y2=short_float_fmt(yb)) writer.element('stop', offset='0', style=generate_css({ 'stop-color': rgb2hex(c), 'stop-opacity': short_float_fmt(c[-1]) })) writer.element('stop', offset='1', style=generate_css({ 'stop-color': rgb2hex(c), 'stop-opacity': "0" })) writer.end('linearGradient') writer.element('polygon', id='GT%x' % self._n_gradients, points=" ".join([ short_float_fmt(x) for x in (x1, y1, x2, y2, x3, y3) ])) writer.end('defs') avg_color = np.sum(colors[:, :], axis=0) / 3.0 href = '#GT%x' % self._n_gradients writer.element('use', attrib={ 'xlink:href': href, 'fill': rgb2hex(avg_color), 'fill-opacity': short_float_fmt(avg_color[-1]) }) for i in range(3): writer.element('use', attrib={ 'xlink:href': href, 'fill': 'url(#GR%x_%d)' % (self._n_gradients, i), 'fill-opacity': '1', 'filter': 'url(#colorAdd)' }) self._n_gradients += 1 def draw_gouraud_triangles(self, gc, triangles_array, colors_array, transform): attrib = {} clipid = self._get_clip(gc) if clipid is not None: attrib['clip-path'] = 'url(#%s)' % clipid self.writer.start('g', attrib=attrib) transform = transform.frozen() for tri, col in zip(triangles_array, colors_array): self.draw_gouraud_triangle(gc, tri, col, transform) self.writer.end('g') def option_scale_image(self): return True def get_image_magnification(self): return self.image_dpi / 72.0 def draw_image(self, gc, x, y, im, transform=None): h, w = im.shape[:2] if w == 0 or h == 0: return attrib = {} clipid = self._get_clip(gc) if clipid is not None: # Can't apply clip-path directly to the image because the # image has a transformation, which would also be applied # to the clip-path self.writer.start('g', attrib={'clip-path': 'url(#%s)' % clipid}) oid = gc.get_gid() url = gc.get_url() if url is not None: self.writer.start('a', attrib={'xlink:href': url}) if rcParams['svg.image_inline']: bytesio = io.BytesIO() _png.write_png(im, bytesio) oid = oid or self._make_id('image', bytesio.getvalue()) attrib['xlink:href'] = ( "data:image/png;base64,\n" + base64.b64encode(bytesio.getvalue()).decode('ascii')) else: self._imaged[self.basename] = self._imaged.get(self.basename, 0) + 1 filename = '%s.image%d.png' % (self.basename, self._imaged[self.basename]) _log.info('Writing image file for inclusion: %s', filename) _png.write_png(im, filename) oid = oid or 'Im_' + self._make_id('image', filename) attrib['xlink:href'] = filename attrib['id'] = oid if transform is None: w = 72.0 * w / self.image_dpi h = 72.0 * h / self.image_dpi self.writer.element('image', transform=generate_transform([ ('scale', (1, -1)), ('translate', (0, -h)) ]), x=short_float_fmt(x), y=short_float_fmt(-(self.height - y - h)), width=short_float_fmt(w), height=short_float_fmt(h), attrib=attrib) else: alpha = gc.get_alpha() if alpha != 1.0: attrib['opacity'] = short_float_fmt(alpha) flipped = (Affine2D().scale(1.0 / w, 1.0 / h) + transform + Affine2D().translate(x, y).scale(1.0, -1.0).translate( 0.0, self.height)) attrib['transform'] = generate_transform([('matrix', flipped.frozen())]) self.writer.element('image', width=short_float_fmt(w), height=short_float_fmt(h), attrib=attrib) if url is not None: self.writer.end('a') if clipid is not None: self.writer.end('g') def _adjust_char_id(self, char_id): return char_id.replace("%20", "_") def _draw_text_as_path(self, gc, x, y, s, prop, angle, ismath, mtext=None): """ draw the text by converting them to paths using textpath module. Parameters ---------- prop : `matplotlib.font_manager.FontProperties` font property s : str text to be converted usetex : bool If True, use matplotlib usetex mode. ismath : bool If True, use mathtext parser. If "TeX", use *usetex* mode. """ writer = self.writer writer.comment(s) glyph_map = self._glyph_map text2path = self._text2path color = rgb2hex(gc.get_rgb()) fontsize = prop.get_size_in_points() style = {} if color != '#000000': style['fill'] = color if gc.get_alpha() != 1.0: style['opacity'] = short_float_fmt(gc.get_alpha()) if not ismath: font = text2path._get_font(prop) _glyphs = text2path.get_glyphs_with_font( font, s, glyph_map=glyph_map, return_new_glyphs_only=True) glyph_info, glyph_map_new, rects = _glyphs if glyph_map_new: writer.start('defs') for char_id, glyph_path in six.iteritems(glyph_map_new): path = Path(*glyph_path) path_data = self._convert_path(path, simplify=False) writer.element('path', id=char_id, d=path_data) writer.end('defs') glyph_map.update(glyph_map_new) attrib = {} attrib['style'] = generate_css(style) font_scale = fontsize / text2path.FONT_SCALE attrib['transform'] = generate_transform([ ('translate', (x, y)), ('rotate', (-angle, )), ('scale', (font_scale, -font_scale)) ]) writer.start('g', attrib=attrib) for glyph_id, xposition, yposition, scale in glyph_info: attrib = {'xlink:href': '#%s' % glyph_id} if xposition != 0.0: attrib['x'] = short_float_fmt(xposition) if yposition != 0.0: attrib['y'] = short_float_fmt(yposition) writer.element('use', attrib=attrib) writer.end('g') else: if ismath == "TeX": _glyphs = text2path.get_glyphs_tex(prop, s, glyph_map=glyph_map, return_new_glyphs_only=True) else: _glyphs = text2path.get_glyphs_mathtext( prop, s, glyph_map=glyph_map, return_new_glyphs_only=True) glyph_info, glyph_map_new, rects = _glyphs # we store the character glyphs w/o flipping. Instead, the # coordinate will be flipped when this characters are # used. if glyph_map_new: writer.start('defs') for char_id, glyph_path in six.iteritems(glyph_map_new): char_id = self._adjust_char_id(char_id) # Some characters are blank if not len(glyph_path[0]): path_data = "" else: path = Path(*glyph_path) path_data = self._convert_path(path, simplify=False) writer.element('path', id=char_id, d=path_data) writer.end('defs') glyph_map.update(glyph_map_new) attrib = {} font_scale = fontsize / text2path.FONT_SCALE attrib['style'] = generate_css(style) attrib['transform'] = generate_transform([ ('translate', (x, y)), ('rotate', (-angle, )), ('scale', (font_scale, -font_scale)) ]) writer.start('g', attrib=attrib) for char_id, xposition, yposition, scale in glyph_info: char_id = self._adjust_char_id(char_id) writer.element('use', transform=generate_transform([ ('translate', (xposition, yposition)), ('scale', (scale, )), ]), attrib={'xlink:href': '#%s' % char_id}) for verts, codes in rects: path = Path(verts, codes) path_data = self._convert_path(path, simplify=False) writer.element('path', d=path_data) writer.end('g') def _draw_text_as_text(self, gc, x, y, s, prop, angle, ismath, mtext=None): writer = self.writer color = rgb2hex(gc.get_rgb()) style = {} if color != '#000000': style['fill'] = color if gc.get_alpha() != 1.0: style['opacity'] = short_float_fmt(gc.get_alpha()) if not ismath: font = self._get_font(prop) font.set_text(s, 0.0, flags=LOAD_NO_HINTING) fontsize = prop.get_size_in_points() fontfamily = font.family_name fontstyle = prop.get_style() attrib = {} # Must add "px" to workaround a Firefox bug style['font-size'] = short_float_fmt(fontsize) + 'px' style['font-family'] = six.text_type(fontfamily) style['font-style'] = prop.get_style().lower() style['font-weight'] = six.text_type(prop.get_weight()).lower() attrib['style'] = generate_css(style) if mtext and (angle == 0 or mtext.get_rotation_mode() == "anchor"): # If text anchoring can be supported, get the original # coordinates and add alignment information. # Get anchor coordinates. transform = mtext.get_transform() ax, ay = transform.transform_point(mtext.get_position()) ay = self.height - ay # Don't do vertical anchor alignment. Most applications do not # support 'alignment-baseline' yet. Apply the vertical layout # to the anchor point manually for now. angle_rad = np.deg2rad(angle) dir_vert = np.array([np.sin(angle_rad), np.cos(angle_rad)]) v_offset = np.dot(dir_vert, [(x - ax), (y - ay)]) ax = ax + v_offset * dir_vert[0] ay = ay + v_offset * dir_vert[1] ha_mpl_to_svg = { 'left': 'start', 'right': 'end', 'center': 'middle' } style['text-anchor'] = ha_mpl_to_svg[mtext.get_ha()] attrib['x'] = short_float_fmt(ax) attrib['y'] = short_float_fmt(ay) attrib['style'] = generate_css(style) attrib['transform'] = "rotate(%s, %s, %s)" % (short_float_fmt( -angle), short_float_fmt(ax), short_float_fmt(ay)) writer.element('text', s, attrib=attrib) else: attrib['transform'] = generate_transform([ ('translate', (x, y)), ('rotate', (-angle, )) ]) writer.element('text', s, attrib=attrib) if rcParams['svg.fonttype'] == 'svgfont': fontset = self._fonts.setdefault(font.fname, set()) for c in s: fontset.add(ord(c)) else: writer.comment(s) width, height, descent, svg_elements, used_characters = \ self.mathtext_parser.parse(s, 72, prop) svg_glyphs = svg_elements.svg_glyphs svg_rects = svg_elements.svg_rects attrib = {} attrib['style'] = generate_css(style) attrib['transform'] = generate_transform([('translate', (x, y)), ('rotate', (-angle, ))]) # Apply attributes to 'g', not 'text', because we likely # have some rectangles as well with the same style and # transformation writer.start('g', attrib=attrib) writer.start('text') # Sort the characters by font, and output one tspan for # each spans = OrderedDict() for font, fontsize, thetext, new_x, new_y, metrics in svg_glyphs: style = generate_css({ 'font-size': short_float_fmt(fontsize) + 'px', 'font-family': font.family_name, 'font-style': font.style_name.lower(), 'font-weight': font.style_name.lower() }) if thetext == 32: thetext = 0xa0 # non-breaking space spans.setdefault(style, []).append((new_x, -new_y, thetext)) if rcParams['svg.fonttype'] == 'svgfont': for font, fontsize, thetext, new_x, new_y, metrics in svg_glyphs: fontset = self._fonts.setdefault(font.fname, set()) fontset.add(thetext) for style, chars in six.iteritems(spans): chars.sort() same_y = True if len(chars) > 1: last_y = chars[0][1] for i in xrange(1, len(chars)): if chars[i][1] != last_y: same_y = False break if same_y: ys = six.text_type(chars[0][1]) else: ys = ' '.join(six.text_type(c[1]) for c in chars) attrib = { 'style': style, 'x': ' '.join(short_float_fmt(c[0]) for c in chars), 'y': ys } writer.element('tspan', ''.join(unichr(c[2]) for c in chars), attrib=attrib) writer.end('text') if len(svg_rects): for x, y, width, height in svg_rects: writer.element('rect', x=short_float_fmt(x), y=short_float_fmt(-y + height), width=short_float_fmt(width), height=short_float_fmt(height)) writer.end('g') def draw_tex(self, gc, x, y, s, prop, angle, ismath='TeX!', mtext=None): self._draw_text_as_path(gc, x, y, s, prop, angle, ismath="TeX") def draw_text(self, gc, x, y, s, prop, angle, ismath=False, mtext=None): clipid = self._get_clip(gc) if clipid is not None: # Cannot apply clip-path directly to the text, because # is has a transformation self.writer.start('g', attrib={'clip-path': 'url(#%s)' % clipid}) if gc.get_url() is not None: self.writer.start('a', {'xlink:href': gc.get_url()}) if rcParams['svg.fonttype'] == 'path': self._draw_text_as_path(gc, x, y, s, prop, angle, ismath, mtext) else: self._draw_text_as_text(gc, x, y, s, prop, angle, ismath, mtext) if gc.get_url() is not None: self.writer.end('a') if clipid is not None: self.writer.end('g') def flipy(self): return True def get_canvas_width_height(self): return self.width, self.height def get_text_width_height_descent(self, s, prop, ismath): return self._text2path.get_text_width_height_descent(s, prop, ismath)
StdBlckLst.extend(get_standard_funtions(csv_file_to_table(open(StdFuncsCSV)))) # Dictionary to speedup block type fetching by name StdBlckDct = OrderedDict() for section in StdBlckLst: for desc in section["list"]: words = desc["comment"].split('"') if len(words) > 1: desc["comment"] = words[1] desc["usage"] = ("\n (%s) => (%s)" % (", ".join([ "%s:%s" % (input[1], input[0]) for input in desc["inputs"] ]), ", ".join( ["%s:%s" % (output[1], output[0]) for output in desc["outputs"]]))) BlkLst = StdBlckDct.setdefault(desc["name"], []) BlkLst.append((section["name"], desc)) # ------------------------------------------------------------------------------- # Languages Keywords # ------------------------------------------------------------------------------- # Keywords for Pou Declaration POU_BLOCK_START_KEYWORDS = ["FUNCTION", "FUNCTION_BLOCK", "PROGRAM"] POU_BLOCK_END_KEYWORDS = ["END_FUNCTION", "END_FUNCTION_BLOCK", "END_PROGRAM"] POU_KEYWORDS = ["EN", "ENO", "F_EDGE", "R_EDGE" ] + POU_BLOCK_START_KEYWORDS + POU_BLOCK_END_KEYWORDS for category in StdBlckLst: for block in category["list"]: if block["name"] not in POU_KEYWORDS: POU_KEYWORDS.append(block["name"])
parser = argparse.ArgumentParser() parser.add_argument("--key", help="input key") parser.add_argument("--val", help="input value") args = parser.parse_args() key = args.key value = args.val storage_path = os.path.join(tempfile.gettempdir(), 'storage.data') data = None if os.path.isfile(storage_path): with open(storage_path, "r+") as f: data = json.load(f) elif value is None: print(None) exit() else: with open(storage_path, "w+") as f: data = OrderedDict() data[key] = value json.dump(data, f) #writing... with open(storage_path, "w+") as f: if value is not None: data.setdefault(key, value) if data[key] != value: data[key] = data[key], value else: print(data.get(key, None)) json.dump(data, f)
def test_awxcollector(setup_environment): ''' Efforts to improve the performance of cleanup_jobs involved sub-classing the django Collector class. This unit test will check for parity between the django Collector and the modified AWXCollector class. AWXCollector is used in cleanup_jobs to bulk-delete old jobs from the database. Specifically, Collector has four dictionaries to check: .dependencies, .data, .fast_deletes, and .field_updates These tests will convert each dictionary from AWXCollector (after running .collect on jobs), from querysets to sets of objects. The final result should be a dictionary that is equivalent to django's Collector. ''' (old_jobs, new_jobs, days_str) = setup_environment collector = Collector('default') collector.collect(old_jobs) awx_col = AWXCollector('default') # awx_col accepts a queryset as input awx_col.collect(Job.objects.filter(pk__in=[obj.pk for obj in old_jobs])) # check that dependencies are the same assert awx_col.dependencies == collector.dependencies # check that objects to delete are the same awx_del_dict = OrderedDict() for model, instances in awx_col.data.items(): awx_del_dict.setdefault(model, set()) for inst in instances: # .update() will put each object in a queryset into the set awx_del_dict[model].update(inst) assert awx_del_dict == collector.data # check that field updates are the same awx_del_dict = OrderedDict() for model, instances_for_fieldvalues in awx_col.field_updates.items(): awx_del_dict.setdefault(model, {}) for (field, value), instances in instances_for_fieldvalues.items(): awx_del_dict[model].setdefault((field, value), set()) for inst in instances: awx_del_dict[model][(field, value)].update(inst) # collector field updates don't use the base (polymorphic parent) model, e.g. # it will use JobTemplate instead of UnifiedJobTemplate. Therefore, # we need to rebuild the dictionary and grab the model from the field collector_del_dict = OrderedDict() for model, instances_for_fieldvalues in collector.field_updates.items(): for (field, value), instances in instances_for_fieldvalues.items(): collector_del_dict.setdefault(field.model, {}) collector_del_dict[field.model][( field, value)] = collector.field_updates[model][(field, value)] assert awx_del_dict == collector_del_dict # check that fast deletes are the same collector_fast_deletes = set() for q in collector.fast_deletes: collector_fast_deletes.update(q) awx_col_fast_deletes = set() for q in awx_col.fast_deletes: awx_col_fast_deletes.update(q) assert collector_fast_deletes == awx_col_fast_deletes
def compress(self, log=None, **options): """ Searches templates containing 'compress' nodes and compresses them "offline" -- outside of the request/response cycle. The result is cached with a cache-key derived from the content of the compress nodes (not the content of the possibly linked files!). """ extensions = options.get('extensions') extensions = self.handle_extensions(extensions or ['html']) verbosity = int(options.get("verbosity", 0)) if not log: log = StringIO() if not settings.TEMPLATE_LOADERS: raise OfflineGenerationError("No template loaders defined. You " "must set TEMPLATE_LOADERS in your " "settings.") paths = set() for loader in self.get_loaders(): try: module = import_module(loader.__module__) get_template_sources = getattr(module, 'get_template_sources', None) if get_template_sources is None: get_template_sources = loader.get_template_sources paths.update(list(get_template_sources(''))) except (ImportError, AttributeError, TypeError): # Yeah, this didn't work out so well, let's move on pass if not paths: raise OfflineGenerationError("No template paths found. None of " "the configured template loaders " "provided template paths. See " "https://docs.djangoproject.com/en/1.8/topics/templates/ " "for more information on template " "loaders.") if verbosity > 1: log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n") templates = set() for path in paths: for root, dirs, files in os.walk(path, followlinks=options.get('followlinks', False)): templates.update(os.path.join(root, name) for name in files if not name.startswith('.') and any(fnmatch(name, "*%s" % glob) for glob in extensions)) if not templates: raise OfflineGenerationError("No templates found. Make sure your " "TEMPLATE_LOADERS and TEMPLATE_DIRS " "settings are correct.") if verbosity > 1: log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n") engine = options.get("engine", "django") parser = self.__get_parser(engine) compressor_nodes = OrderedDict() for template_name in templates: try: template = parser.parse(template_name) except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) continue except TemplateSyntaxError as e: # broken template -> ignore if verbosity > 0: log.write("Invalid template %s: %s\n" % (template_name, e)) continue except TemplateDoesNotExist: # non existent template -> ignore if verbosity > 0: log.write("Non-existent template at: %s\n" % template_name) continue except UnicodeDecodeError: if verbosity > 0: log.write("UnicodeDecodeError while trying to read " "template %s\n" % template_name) try: nodes = list(parser.walk_nodes(template)) except (TemplateDoesNotExist, TemplateSyntaxError) as e: # Could be an error in some base template if verbosity > 0: log.write("Error parsing template %s: %s\n" % (template_name, e)) continue if nodes: template.template_name = template_name compressor_nodes.setdefault(template, []).extend(nodes) if not compressor_nodes: raise OfflineGenerationError( "No 'compress' template tags found in templates." "Try running compress command with --follow-links and/or" "--extension=EXTENSIONS") if verbosity > 0: log.write("Found 'compress' tags in:\n\t" + "\n\t".join((t.template_name for t in compressor_nodes.keys())) + "\n") contexts = settings.COMPRESS_OFFLINE_CONTEXT if isinstance(contexts, six.string_types): try: module, function = get_mod_func(contexts) contexts = getattr(import_module(module), function)() except (AttributeError, ImportError, TypeError) as e: raise ImportError("Couldn't import offline context function %s: %s" % (settings.COMPRESS_OFFLINE_CONTEXT, e)) elif not isinstance(contexts, (list, tuple)): contexts = [contexts] log.write("Compressing... ") block_count = context_count = 0 results = [] offline_manifest = OrderedDict() for context_dict in contexts: context_count += 1 init_context = parser.get_init_context(context_dict) for template, nodes in compressor_nodes.items(): context = Context(init_context) template._log = log template._log_verbosity = verbosity if not parser.process_template(template, context): continue for node in nodes: context.push() parser.process_node(template, context, node) rendered = parser.render_nodelist(template, context, node) key = get_offline_hexdigest(rendered) if key in offline_manifest: continue try: result = parser.render_node(template, context, node) except Exception as e: raise CommandError("An error occurred during rendering %s: " "%s" % (template.template_name, e)) offline_manifest[key] = result context.pop() results.append(result) block_count += 1 write_offline_manifest(offline_manifest) log.write("done\nCompressed %d block(s) from %d template(s) for %d context(s).\n" % (block_count, len(compressor_nodes), context_count)) return block_count, results
class PoserPmdFile(object): # Author of the class: Fredi Hartmann, Germany, aka ADP # Email : [email protected] def __init__(self, fh=None): if isinstance(fh, str): fh = open(fh, "rb") if fh is not None: self.filename = fh.name self.fh = fh # Filehandle to read from. self.read_header() # Check file-signature else: self.filename = self.fh = None self.num_targets = None # No target read yet. self.targets = OrderedDict() # storage for Morph-Structure def read_header(self): """ Check file-signature. Raise IOError if signature is not "PZMD". :return: True """ self.fh.seek(0) s = self.fh.read(4) if s != b"PZMD": raise IOError("Not a Poser PMD file") return True def _unpack(self, fmt, size): result = struct.unpack("!" + fmt, self.fh.read(size)) return result[0] def read_all(self): """ Calls read_target() and read_deltas() :return: None """ self.read_targets() self.read_deltas() def read_targets(self): """ Read all morphtargets from open file (filehandle in self.fh) :return: <OrderedDict>self.targets """ if self.fh.closed: raise IOError("File '%s' is closed" % self.fh.name) self.fh.seek(0x10) self.num_targets = self._read_uint(4) self.targets.clear() for i in range(self.num_targets): morphname = self._read_str() actorname = self._read_str() target = self.targets.setdefault(morphname, OrderedDict()) actor = target.setdefault(actorname, OrderedDict()) actor["numb_deltas"] = self._read_uint(4) actor["morphpos"] = self._read_uint(4) self.read_uuids(self.fh.tell()) return self.targets def read_deltas(self): """ Read deltas from open file (filehandle in self.fh) into self.targets. :return: None """ if self.targets is None: self.read_targets() def read_one(pos): result = list() self.fh.seek(pos) indexes = self._read_uint(4) for i in range(indexes): result.append( DELTA(self._read_uint(4), VERTEX(*self._read_vector()))) return result for morphname, v in self.targets.items(): for actor, data in v.items(): data["deltas"] = read_one(data["morphpos"]) def read_uuids(self, pos): """ Read all uuids starting at file-position <pos>. :param pos: fileposition :return: None """ self.fh.seek(pos) for name, target in self.targets.items(): for actorname, actordata in target.items(): actordata["UUID"] = self._read_str() def _read_str(self, length=1): strlen = self._read_uint(length) return self.fh.read(strlen).decode("latin1") def _read_int(self, size): if size == 1: return self._unpack("b", size) if size == 2: return self._unpack("h", size) if size == 4: return self._unpack("i", size) raise ValueError("invalid int size: " + size) def _read_uint(self, size): if size == 1: return self._unpack("B", size) if size == 2: return self._unpack("H", size) if size == 4: return self._unpack("I", size) raise ValueError("invalid int size: " + size) def _read_vector(self): return struct.unpack("!fff", self.fh.read(12)) def _read_float(self): return self._unpack("f", 4)
def setdefault(self, key, default=None): """ Override ``.setdefault()`` to perform case-insensitively. """ return OrderedDict.setdefault(self, key.lower(), default)
class Blueprint(FlaskBlueprint, ArgumentsMixin, ResponseMixin, PaginationMixin, EtagMixin): """Blueprint that registers info in API documentation""" def __init__(self, *args, **kwargs): self.description = kwargs.pop('description', '') super().__init__(*args, **kwargs) # _docs is an ordered dict storing endpoints documentation: # {endpoint: { # 'get': documentation, # 'post': documentation, # ... # } # } self._docs = OrderedDict() def route(self, rule, **options): """Decorator to register url rule in application Also stores doc info for later registration Use this to decorate a :class:`MethodView <flask.views.MethodView>` or a resource function """ def decorator(func): # By default, endpoint name is function name endpoint = options.pop('endpoint', func.__name__) # MethodView (class) if isinstance(func, MethodViewType): # This decorator may be called multiple times on the same # MethodView, but Flask will complain if different views are # mapped to the same endpoint, so we should call 'as_view' only # once and keep the result in MethodView._view_func if not getattr(func, '_view_func', None): func._view_func = func.as_view(endpoint) view_func = func._view_func # Function else: view_func = func # Add URL rule in Flask and store endpoint documentation self.add_url_rule(rule, endpoint, view_func, **options) self._store_endpoint_docs(endpoint, func, **options) return func return decorator def _store_endpoint_docs(self, endpoint, obj, **kwargs): """Store view or function doc info""" endpoint_doc = self._docs.setdefault(endpoint, OrderedDict()) def store_method_docs(method, function): # Get summary/description from docstring docstring = function.__doc__ doc = load_info_from_docstring(docstring) if docstring else {} # Update doc with description from @doc decorator doc.update(getattr(function, '_apidoc', {})) # Add function doc to table for later registration method_l = method.lower() # Check another doc was not already registed for endpoint/method if method_l in endpoint_doc and endpoint_doc[method_l] is not doc: # If multiple routes point to the same endpoint, the doc may # be already registered. # Only trigger exception if a different doc is passed. raise EndpointMethodDocAlreadyRegistedError( "Another doc is already registered for endpoint '{}' " "method {}".format(endpoint, method_l.upper())) endpoint_doc[method_l] = doc # MethodView (class) if isinstance(obj, MethodViewType): for method in HTTP_METHODS: if method in obj.methods: func = getattr(obj, method.lower()) store_method_docs(method, func) # Function else: methods = kwargs.pop('methods', None) or ['GET'] for method in methods: store_method_docs(method, obj) def register_views_in_doc(self, app, spec): """Register views information in documentation If a schema in a parameter or a response appears in the spec `definitions` section, it is replaced by a reference to its definition in the parameter or response documentation: "schema":{"$ref": "#/definitions/MySchema"} """ # This method uses the documentation information associated with the # endpoint (in self._docs) to provide documentation for the route to # the spec object. # for endpoint, doc in self._docs.items(): # doc is a dict of documentation per method for the endpoint # {'get': documentation, 'post': documentation,...} # Prepend Blueprint name to endpoint endpoint = '.'.join((self.name, endpoint)) # Tag all operations with Blueprint name # Format operations documentation in OpenAPI structure for operation in doc.values(): operation['tags'] = [self.name] self._prepare_doc(operation, spec.openapi_version) # TODO: Do we support multiple rules per endpoint? # https://github.com/marshmallow-code/apispec/issues/181 for rule in app.url_map.iter_rules(endpoint): # We need to deepcopy operations here # because it can be modified in APISpec.path(), which causes # issues if there are multiple rules for the same endpoint if APISPEC_VERSION_MAJOR < 1: spec.add_path(rule=rule, operations=deepcopy(doc)) else: spec.path(rule=rule, operations=deepcopy(doc)) @staticmethod def _prepare_doc(operation, openapi_version): """Format operation documentation in OpenAPI structure The decorators store all documentation information in a dict structure that is close to OpenAPI doc structure, so this information could _almost_ be copied as is. Yet, some adjustemnts may have to be performed, especially if the spec structure differs between OpenAPI versions: the OpenAPI version is not known when the decorators are applied but only at registration time when this method is called. """ if openapi_version.major >= 3: if 'responses' in operation: for resp in operation['responses'].values(): if 'schema' in resp: resp['content'] = { 'application/json': { 'schema': resp.pop('schema') } } if 'parameters' in operation: for param in operation['parameters']: if param['in'] == 'body': request_body = { **{ 'content': { 'application/json': { 'schema': param['schema'] } } }, **{ x: param[x] for x in ('description', 'required') if x in param } } operation['requestBody'] = request_body # There can be only one requestBody continue parameters = [ param for param in operation['parameters'] if not param['in'] == 'body' ] if parameters: operation['parameters'] = parameters else: del operation['parameters'] @staticmethod def doc(**kwargs): """Decorator adding description attributes to a view function Values passed as kwargs are copied verbatim in the docs Example: :: @blp.doc(description="Return pets based on ID", summary="Find pets by ID" ) def get(...): ... """ def decorator(func): func._apidoc = deepupdate(getattr(func, '_apidoc', {}), kwargs) return func return decorator
def _compare_namelists(gold_namelists, comp_namelists, case): ############################################################################### """ Compare two namelists. Print diff information if any. Returns comments Note there will only be comments if the namelists were not an exact match Expect args in form: {namelist -> {key -> value} }. value can be an int, string, list, or dict >>> teststr = '''&nml ... val = 'foo' ... aval = 'one','two', 'three' ... maval = 'one', 'two', 'three', 'four' ... dval = 'one -> two', 'three -> four' ... mdval = 'one -> two', 'three -> four', 'five -> six' ... nval = 1850 ... / ... &nml2 ... val2 = .false. ... / ... ''' >>> _compare_namelists(_parse_namelists(teststr.splitlines(), 'foo'), _parse_namelists(teststr.splitlines(), 'bar'), None) '' >>> teststr1 = '''&nml1 ... val11 = 'foo' ... / ... &nml2 ... val21 = 'foo' ... val22 = 'foo', 'bar', 'baz' ... val23 = 'baz' ... val24 = '1 -> 2', '2 -> 3', '3 -> 4' ... / ... &nml3 ... val3 = .false. ... /''' >>> teststr2 = '''&nml01 ... val11 = 'foo' ... / ... &nml2 ... val21 = 'foo0' ... val22 = 'foo', 'bar0', 'baz' ... val230 = 'baz' ... val24 = '1 -> 20', '2 -> 3', '30 -> 4' ... / ... &nml3 ... val3 = .false. ... /''' >>> comments = _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'), _parse_namelists(teststr2.splitlines(), 'bar'), None) >>> print(comments) Missing namelist: nml1 Differences in namelist 'nml2': BASE: val21 = 'foo' COMP: val21 = 'foo0' BASE: val22 list item 1 = 'bar' COMP: val22 list item 1 = 'bar0' missing variable: 'val23' BASE: val24 dict item 1 = 2 COMP: val24 dict item 1 = 20 dict variable 'val24' missing key 3 with value 4 dict variable 'val24' has extra key 30 with value 4 found extra variable: 'val230' Found extra namelist: nml01 <BLANKLINE> >>> teststr1 = '''&rad_cnst_nl ... icecldoptics = 'mitchell' ... logfile = 'cpl.log.150514-001533' ... case_name = 'ERB.f19_g16.B1850C5.skybridge_intel.C.150513-230221' ... runid = 'FOO' ... model_version = 'cam5_3_36' ... username = '******' ... iceopticsfile = '/projects/ccsm/inputdata/atm/cam/physprops/iceoptics_c080917.nc' ... liqcldoptics = 'gammadist' ... liqopticsfile = '/projects/ccsm/inputdata/atm/cam/physprops/F_nwvl200_mu20_lam50_res64_t298_c080428.nc' ... mode_defs = 'mam3_mode1:accum:=', 'A:num_a1:N:num_c1:num_mr:+', ... 'A:so4_a1:N:so4_c1:sulfate:/projects/ccsm/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', 'A:pom_a1:N:pom_c1:p-organic:/projects/ccsm/inputdata/atm/cam/physprops/ocpho_rrtmg_c101112.nc:+', ... 'A:soa_a1:N:soa_c1:s-organic:/projects/ccsm/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', 'A:bc_a1:N:bc_c1:black-c:/projects/ccsm/inputdata/atm/cam/physprops/bcpho_rrtmg_c100508.nc:+', ... 'A:dst_a1:N:dst_c1:dust:/projects/ccsm/inputdata/atm/cam/physprops/dust4_rrtmg_c090521.nc:+', 'A:ncl_a1:N:ncl_c1:seasalt:/projects/ccsm/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc', ... 'mam3_mode2:aitken:=', 'A:num_a2:N:num_c2:num_mr:+', ... 'A:so4_a2:N:so4_c2:sulfate:/projects/ccsm/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', 'A:soa_a2:N:soa_c2:s-organic:/projects/ccsm/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', ... 'A:ncl_a2:N:ncl_c2:seasalt:/projects/ccsm/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc', 'mam3_mode3:coarse:=', ... 'A:num_a3:N:num_c3:num_mr:+', 'A:dst_a3:N:dst_c3:dust:/projects/ccsm/inputdata/atm/cam/physprops/dust4_rrtmg_c090521.nc:+', ... 'A:ncl_a3:N:ncl_c3:seasalt:/projects/ccsm/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc:+', 'A:so4_a3:N:so4_c3:sulfate:/projects/ccsm/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc' ... rad_climate = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2', ... 'N:ozone:O3', 'N:N2O:N2O', 'N:CH4:CH4', ... 'N:CFC11:CFC11', 'N:CFC12:CFC12', 'M:mam3_mode1:/projects/ccsm/inputdata/atm/cam/physprops/mam3_mode1_rrtmg_c110318.nc', ... 'M:mam3_mode2:/projects/ccsm/inputdata/atm/cam/physprops/mam3_mode2_rrtmg_c110318.nc', 'M:mam3_mode3:/projects/ccsm/inputdata/atm/cam/physprops/mam3_mode3_rrtmg_c110318.nc' ... /''' >>> teststr2 = '''&rad_cnst_nl ... icecldoptics = 'mitchell' ... logfile = 'cpl.log.150514-2398745' ... case_name = 'ERB.f19_g16.B1850C5.skybridge_intel.C.150513-1274213' ... runid = 'BAR' ... model_version = 'cam5_3_36' ... username = '******' ... iceopticsfile = '/something/else/inputdata/atm/cam/physprops/iceoptics_c080917.nc' ... liqcldoptics = 'gammadist' ... liqopticsfile = '/something/else/inputdata/atm/cam/physprops/F_nwvl200_mu20_lam50_res64_t298_c080428.nc' ... mode_defs = 'mam3_mode1:accum:=', 'A:num_a1:N:num_c1:num_mr:+', ... 'A:so4_a1:N:so4_c1:sulfate:/something/else/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', 'A:pom_a1:N:pom_c1:p-organic:/something/else/inputdata/atm/cam/physprops/ocpho_rrtmg_c101112.nc:+', ... 'A:soa_a1:N:soa_c1:s-organic:/something/else/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', 'A:bc_a1:N:bc_c1:black-c:/something/else/inputdata/atm/cam/physprops/bcpho_rrtmg_c100508.nc:+', ... 'A:dst_a1:N:dst_c1:dust:/something/else/inputdata/atm/cam/physprops/dust4_rrtmg_c090521.nc:+', 'A:ncl_a1:N:ncl_c1:seasalt:/something/else/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc', ... 'mam3_mode2:aitken:=', 'A:num_a2:N:num_c2:num_mr:+', ... 'A:so4_a2:N:so4_c2:sulfate:/something/else/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', 'A:soa_a2:N:soa_c2:s-organic:/something/else/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', ... 'A:ncl_a2:N:ncl_c2:seasalt:/something/else/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc', 'mam3_mode3:coarse:=', ... 'A:num_a3:N:num_c3:num_mr:+', 'A:dst_a3:N:dst_c3:dust:/something/else/inputdata/atm/cam/physprops/dust4_rrtmg_c090521.nc:+', ... 'A:ncl_a3:N:ncl_c3:seasalt:/something/else/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc:+', 'A:so4_a3:N:so4_c3:sulfate:/something/else/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc' ... rad_climate = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2', ... 'N:ozone:O3', 'N:N2O:N2O', 'N:CH4:CH4', ... 'N:CFC11:CFC11', 'N:CFC12:CFC12', 'M:mam3_mode1:/something/else/inputdata/atm/cam/physprops/mam3_mode1_rrtmg_c110318.nc', ... 'M:mam3_mode2:/something/else/inputdata/atm/cam/physprops/mam3_mode2_rrtmg_c110318.nc', 'M:mam3_mode3:/something/else/inputdata/atm/cam/physprops/mam3_mode3_rrtmg_c110318.nc' ... /''' >>> _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'), _parse_namelists(teststr2.splitlines(), 'bar'), 'ERB.f19_g16.B1850C5.skybridge_intel') '' """ different_namelists = OrderedDict() for namelist, gold_names in gold_namelists.items(): if (namelist not in comp_namelists): different_namelists[namelist] = [ "Missing namelist: {}\n".format(namelist) ] else: comp_names = comp_namelists[namelist] for name, gold_value in gold_names.items(): if (name not in comp_names): different_namelists.setdefault(namelist, []).append( " missing variable: '{}'\n".format(name)) else: comp_value = comp_names[name] comments = _compare_values(name, gold_value, comp_value, case) if comments != "": different_namelists.setdefault(namelist, []).append(comments) for name in comp_names: if (name not in gold_names): different_namelists.setdefault(namelist, []).append( " found extra variable: '{}'\n".format(name)) for namelist in comp_namelists: if (namelist not in gold_namelists): different_namelists[namelist] = [ "Found extra namelist: {}\n".format(namelist) ] comments = "" for namelist, nlcomment in different_namelists.items(): if len(nlcomment) == 1: comments += nlcomment[0] else: comments += "Differences in namelist '{}':\n".format(namelist) comments += "".join(nlcomment) return comments
def _traverse_profile_sections(profile_sections, request, start_section_num=0): sections_by_title = OrderedDict() section_topics_by_title = OrderedDict() section_topic_indicators_by_title = OrderedDict() for section_num, profile_section in enumerate(profile_sections, start=start_section_num + 1): (country, profile_title) = (str(profile_section.get_parent()), profile_section.title) \ if isinstance(profile_section, ProfileSectionPage) \ else (str(profile_section), 'Country Overview') default_section = { 'id': 'section-{}'.format(section_num), 'title': profile_title, 'href': 'section-{}-topics-tab'.format(section_num), 'key_issues': [], } section = sections_by_title.setdefault(profile_title.lower(), default_section) topics_by_title = section_topics_by_title.setdefault( profile_title.lower(), OrderedDict()) topic_indicators_by_title = section_topic_indicators_by_title.setdefault( profile_title.lower(), OrderedDict()) start_topic_num = len(topics_by_title.keys()) for topic_num, section_topic in enumerate(profile_section.body, start=start_topic_num + 1): # Topics that have no indicators (key issues) should be # displayed separately. topic_title = section_topic.value['title'] if not section_topic.value['indicators']: section['key_issues'].append({ 'id': '{}-key_issue-{}'.format(section['id'], topic_num), 'title': topic_title, 'country': country, 'href': profile_section.get_url(request), }) else: default_topic = { 'id': '{}-topic-{}'.format(section['id'], topic_num), 'title': topic_title, 'href': '{}-topic-{}-indicators-tab'.format( section['id'], topic_num), } topic = topics_by_title.setdefault(topic_title.lower(), default_topic) indicators_by_title = topic_indicators_by_title.setdefault( topic_title.lower(), OrderedDict()) start_indicator_num = len(indicators_by_title.keys()) for indicator_num, topic_indicator in enumerate( section_topic.value['indicators'], start=start_indicator_num + 1): indicator_title = topic_indicator.value['title'] default_indicator = { 'id': '{}-indicator-{}'.format(topic['id'], indicator_num), 'title': indicator_title, 'href': '{}-indicator-{}-country-selections-tab'.format( topic['id'], indicator_num), 'countries': [], } indicator = indicators_by_title.setdefault( indicator_title.lower(), default_indicator) indicator['countries'].append({ 'title': country, 'href': profile_section.get_url(request), }) topic['indicators'] = indicators_by_title.values() section['topics'] = topics_by_title.values() return sections_by_title.values()
def showHitCoinList(request): context = {} items = StorageReport.objects.all() clientItems = Clients.objects.all() userMap = {} for item in clientItems: node_id = item.node_id username = item.username userMap[node_id] = username rewardAddDict = OrderedDict() onlineTime_all = 0 for item in items: node_id = item.node_id repo_size = item.repo_size storage_size = item.storage_size address = item.address if address == "127.0.0.1" or address == "localhost": continue if node_id not in userMap: continue print(address) hit_per_bit = 2 / 10000000000 reward_repo_size_divide_10G = 0.0 for i in range(len(repo_size) - 1, -1, -1): b = int(repo_size[i]) reward_repo_size_divide_10G += b * hit_per_bit hit_per_bit *= 10 rewardAddDict.setdefault(address, (0, 0.0)) rewardAddDict[address] = (rewardAddDict[address][0] + 1, rewardAddDict[address][1] + reward_repo_size_divide_10G) # hours onlineTime_all += 1 addresses = list(rewardAddDict.keys()) locations, errorIps = getLocByIpList(addresses) if locations == None: context["Error"] = True return render(request, "hitcoinList.html", locals()) print("location is: ") print(locations) for ip in errorIps: rewardAddDict.pop(ip) addresses.remove(ip) longitudes = [] latitudes = [] hours = [] realHours = [] for i in range(len(addresses)): add = addresses[i] loc = locations[i] value = rewardAddDict[add] longitudes.append(loc[0]) latitudes.append(loc[1]) hours.append(math.ceil(value[0] / onlineTime_all * 30)) # only record the online time realHours.append(value[0]) return render(request, "hitcoinList.html", locals())
def parse_optgeo_options(filename): """ Parse an optgeo_options.txt file into specific OptGeoTarget Target Options""" logger.info("Reading optgeo options from file: %s\n" % filename) global_opts = OrderedDict() sys_opts = OrderedDict() section = None section_opts = OrderedDict() with open(filename) as f: for ln, line in enumerate(f, 1): # Anything after "#" is a comment line = line.split("#", maxsplit=1)[0].strip() if not line: continue ls = line.split() key = ls[0].lower() if key[0] == "$": # section sign $ if key == '$end': if section is None: warn_press_key( "Line %i: Encountered $end before any section." % ln) elif section == 'global': # global options read finish global_opts = section_opts elif section == 'system': # check if system section contains name if 'name' not in section_opts: warn_press_key( "Line %i: You need to specify a name for the system section ending." % ln) elif section_opts['name'] in sys_opts: warn_press_key( "Line %i: A system named %s already exists in Systems" % (ln, section_opts['name'])) else: sys_opts[section_opts['name']] = section_opts section = None section_opts = OrderedDict() else: if section is not None: warn_press_key( "Line %i: Encountered section start %s before previous section $end." % (ln, key)) if key == '$global': section = 'global' elif key == '$system': section = 'system' else: warn_press_key( "Line %i: Encountered unsupported section name %s " % (ln, key)) else: # put normal key-value options into section_opts if key in ['name', 'geometry', 'topology']: if len(ls) != 2: warn_press_key( "Line %i: one value expected for key %s" % (ln, key)) if section == 'global': warn_press_key( "Line %i: key %s should not appear in $global section" % (ln, key)) section_opts[key] = ls[1] elif key in [ 'bond_denom', 'angle_denom', 'dihedral_denom', 'improper_denom' ]: if len(ls) != 2: warn_press_key( "Line %i: one value expected for key %s" % (ln, key)) section_opts[key] = float(ls[1]) elif key == 'mol2': # special parsing for mol2 option for SMIRNOFF engine # the value is a list of filenames section_opts[key] = ls[1:] # apply a few default global options global_opts.setdefault('bond_denom', 0.02) global_opts.setdefault('angle_denom', 3) global_opts.setdefault('dihedral_denom', 10.0) global_opts.setdefault('improper_denom', 10.0) # copy global options into each system for sys_name, sys_opt_dict in sys_opts.items(): for k, v in global_opts.items(): # do not overwrite system options sys_opt_dict.setdefault(k, v) for k in ['name', 'geometry', 'topology']: if k not in sys_opt_dict: warn_press_key( "key %s missing in system section named %s" % (k, sys_name)) return sys_opts
else: size = np.zeros(len(x)) # n is the number of nodes n = len(x) # creates node numbers from the nodes file node = np.zeros(n, dtype=np.int32) # to check for duplicate nodes # this piece of code uses OrderedDict to remove duplicate nodes # source "http://stackoverflow.com/questions/12698987" # ################################################################### tmp = OrderedDict() for point in zip(x, y, z, size): tmp.setdefault(point[:2], point) # in python 3 tmp.values() is a view object that needs to be # converted to a list mypoints = list(tmp.values()) # ################################################################### n_rev = len(mypoints) # replace x,y,z,size and n with their unique equivalents if (duplicates_flag == 1): for i in range(n_rev): x[i] = mypoints[i][0] y[i] = mypoints[i][1] z[i] = mypoints[i][2] size[i] = mypoints[i][3] n = n_rev