class DownloaderSimulator(BaseDownloaderSimulator):
def __init__(self, rate):
self._requests_per_slot = rate
self.slots = OrderedDict()
super(DownloaderSimulator, self).__init__()
def update(self, requests):
for request in requests:
hostname = urlparse(request.url).hostname or ""
self.slots.setdefault(hostname, deque()).append(request)
def download(self):
output = []
_trash_can = []
for key, requests in six.iteritems(self.slots):
for i in range(min(len(requests), self._requests_per_slot)):
output.append(requests.popleft())
if not requests:
_trash_can.append(key)
for key in _trash_can:
del self.slots[key]
return output
def downloader_info(self):
info = {"key_type": "domain", "overused_keys": []}
for key, requests in six.iteritems(self.slots):
if len(requests) > self._requests_per_slot:
info["overused_keys"].append(key)
return info
def idle(self):
return len(self.slots) == 0
def get(content, keep_order=False):
"""Get dataentry from *content*. *keep_order* indicates whether to
return an ordered dictionnay."""
if keep_order:
from collections import OrderedDict
dataentry = OrderedDict()
else:
dataentry = {}
found = False
for line in content.split('\n'):
if line.strip().startswith('---- dataentry'):
found = True
continue
elif line == '----':
break
elif not found:
continue
line_split = line.split(':')
key = line_split[0].strip()
value = re.sub('#.*$', '', ':'.join(line_split[1:])).strip()
dataentry.setdefault(key, value)
if not found:
raise DokuWikiError('no dataentry found')
return dataentry
def list_releases(moniker=None, limit=0):
"""
List currently available releases.
Arguments:
moniker -- kernel release moniker (e.g., mainline, stable, longterm, etc.).
Defaults to everything
limit -- maximum number of releases to list per moniker. Defaults to 0,
which is no limit
"""
releases = get_releases()
if moniker is None:
releases_by_moniker = OrderedDict()
for release in releases['releases']:
releases_by_moniker.setdefault(release['moniker'], []).append(release)
first = True
for moniker, r in releases_by_moniker.items():
if not first:
print()
first = False
print('%s:' % moniker)
for release in r[:limit] if limit > 0 else r:
print(release['version'])
else:
r = get_releases_by_moniker(releases, moniker)
for release in r[:limit] if limit > 0 else r:
print(release['version'])
def _parse(self, block):
""" Translate Tiger XML structure to the fields of export format,
so that tree writing code for export format can be used. """
nodes = OrderedDict()
root = block.find('graph').get('root')
for term in block.find('graph').find('terminals'):
fields = nodes.setdefault(term.get('id'), 6 * [None])
fields[WORD] = term.get('word')
fields[LEMMA] = term.get('lemma')
fields[TAG] = term.get('pos')
fields[MORPH] = term.get('morph')
nodes[term.get('id')] = fields
for nt in block.find('graph').find('nonterminals'):
if nt.get('id') == root:
ntid = '0'
else:
fields = nodes.setdefault(nt.get('id'), 6 * [None])
ntid = nt.get('id').split('_')[-1]
fields[WORD] = '#' + ntid
fields[TAG] = nt.get('cat')
fields[LEMMA] = fields[MORPH] = '--'
for edge in nt:
idref = edge.get('idref')
nodes.setdefault(idref, 6 * [None])
if nodes[idref][FUNC] is None:
nodes[idref][FUNC] = edge.get('label')
nodes[idref][PARENT] = ntid
else: # secondary edge
nodes[idref].extend((edge.get('label'), ntid))
tree = exportparse(list(nodes.values()), self.morphology, self.lemmas)
sent = self._word(block, orig=True)
return tree, sent
def get_stacktrace_processing_task(infos, processors):
"""Returns a list of all tasks for the processors. This can skip over
processors that seem to not handle any frames.
"""
by_processor = {}
to_lookup = {}
# by_stacktrace_info requires stable sorting as it is used in
# StacktraceProcessingTask.iter_processable_stacktraces. This is important
# to guarantee reproducible symbolicator requests.
by_stacktrace_info = OrderedDict()
for info in infos:
processable_frames = get_processable_frames(info, processors)
for processable_frame in processable_frames:
processable_frame.processor.preprocess_frame(processable_frame)
by_processor.setdefault(processable_frame.processor, []) \
.append(processable_frame)
by_stacktrace_info.setdefault(processable_frame.stacktrace_info, []) \
.append(processable_frame)
if processable_frame.cache_key is not None:
to_lookup[processable_frame.cache_key] = processable_frame
frame_cache = lookup_frame_cache(to_lookup)
for cache_key, processable_frame in six.iteritems(to_lookup):
processable_frame.cache_value = frame_cache.get(cache_key)
return StacktraceProcessingTask(
processable_stacktraces=by_stacktrace_info, processors=by_processor
)
def sum_factorise(variable, tail_ordering, monomial_sum):
if tail_ordering:
key_ordering = OrderedDict()
sub_monosums = defaultdict(MonomialSum)
for sum_indices, atomics, rest in monomial_sum:
# Pull out those sum indices that are not contained in the
# tail ordering, together with those atomics which do not
# share free indices with the tail ordering.
#
# Based on this, split the monomial sum, then recursively
# optimise each sub monomial sum with the first tail index
# removed.
tail_indices = tuple(i for i in sum_indices if i in tail_ordering)
tail_atomics = tuple(a for a in atomics
if set(tail_indices) & set(a.free_indices))
head_indices = tuple(i for i in sum_indices if i not in tail_ordering)
head_atomics = tuple(a for a in atomics if a not in tail_atomics)
key = (head_indices, head_atomics)
key_ordering.setdefault(key)
sub_monosums[key].add(tail_indices, tail_atomics, rest)
sub_monosums = [(k, sub_monosums[k]) for k in key_ordering]
monomial_sum = MonomialSum()
for (sum_indices, atomics), monosum in sub_monosums:
new_rest = sum_factorise(variable, tail_ordering[1:], monosum)
monomial_sum.add(sum_indices, atomics, new_rest)
# Use COFFEE algorithm to optimise the monomial sum
return optimise_monomial_sum(monomial_sum, variable.index_ordering())
def get_all_kplex(graph, k):
triangles = find_triangles(graph)
kplexes=OrderedDict()
while len(triangles) > 0:
t = triangles.pop()
[kplex, peripheral] = get_kplex(graph, t, k)
kplex.sort()
peripheral.sort()
kplex = tuple(kplex)
kplexes.setdefault(kplex,set())
kplexes[kplex]=kplexes[kplex].union(set(peripheral))
#here we remove triangles containing a vertex in the kplex to prevent duplicates
#not sure what we should do about vertex belonging to multiple kplexes?
#new_triangles = []
#for t in triangles:
# is_valid = True
# for v in kplex:
# if v in t:
# is_valid = False
# break
# if is_valid:
# new_triangles.append(t)
#
#triangles = new_triangles
return kplexes
def _groupResults(self, results, formatter):
"""
Group tests together based on their results.
@param results: An iterable of tuples of two or more elements. The
first element of each tuple is a test case. The remaining
elements describe the outcome of that test case.
@param formatter: A callable which turns a test case result into a
string. The elements after the first of the tuples in
C{results} will be passed as positional arguments to
C{formatter}.
@return: A C{list} of two-tuples. The first element of each tuple
is a unique string describing one result from at least one of
the test cases in C{results}. The second element is a list of
the test cases which had that result.
"""
groups = OrderedDict()
for content in results:
case = content[0]
outcome = content[1:]
key = formatter(*outcome)
groups.setdefault(key, []).append(case)
return groups.items()
def print_queues():
"""
Print each queue with waiting or delayed jobs, by priority
"""
queues = OrderedDict()
for q in Queue.collection().sort(by='name', alpha=True).instances():
waiting = q.waiting.llen()
delayed = q.delayed.zcard()
if waiting + delayed == 0:
continue
name, priority = q.hmget('name', 'priority')
queues.setdefault(name, []).append({
'priority': int(priority),
'waiting': waiting,
'delayed': delayed,
})
for name in queues:
sub_queues = sorted(queues[name], key=itemgetter('priority'), reverse=True)
total_waiting = sum([q['waiting'] for q in sub_queues])
total_delayed = sum([q['delayed'] for q in sub_queues])
if len(sub_queues) == 1:
priority_part = sub_queues[0]['priority']
else:
priority_part = '----'
print('%30s %4s %4d %4d' % (name, priority_part, total_waiting, total_delayed))
if len(sub_queues) > 1:
for i, q in enumerate(sub_queues):
print('%30s %4d %4d %4d' % (' ', q['priority'], q['waiting'], q['delayed']))
def query(area, keywords, preRender=True):
transKeywords = transKeyWords(keywords)
colNameDict = getColNameDict()
global redis
rowDict = {}
tableOrderedDict = OrderedDict()
for key in redis.sscan_iter('rowSet',match=area.encode("utf8")+"[0-9][0-9][0-9][0-9]%%&&%%*%s*" % transKeywords[0], count=10000):
#020304 六位，02表示专业，03表示该专业的第三个工作簿， 04表示该工作簿的第四个sheet
flag = True
tabIndex = key.split("%&&%")[0]
rowContent = key.split("%&&%")[1]
for keyword in transKeywords[1:]: #To Do 要分割出来再判断，加了*再后面加%%&&%%，增了差不多一倍的时间。。查询时过滤这个关键词吧。
if rowContent.find(keyword)==-1:
flag = False
break
if flag:
if preRender:
for i in range(len(keywords)): #不使用前端的正则替换，很容易出错。后端把字符替换完即可。注意使用未转换redis特殊字符的关键字列表
rowContent = rowContent.replace(keywords[i],"".join(['<span class="keyword',str(i),'">',keywords[i],'</span>']))
rowDict.setdefault(tabIndex, []).append(rowContent.split("@$$@"))
# ToDo 不能用：区分，因为单元各种会有这种符号
# rowOrderedDict = OrderedDict(sorted(rowDict.items(),cmp=lambda a,b:int(a[0][1:])-int(b[0][1:]) if a[0][0]==b[0][0] else int(a[0][0])-int(b[0][0]), key=lambda d:int(d[0]))) #使用OrderedDict解决显示输出表格排序的问题
rowOrderedDict = OrderedDict(sorted(rowDict.items(),cmp=lambda a,b:int(a[0][2:])-int(b[0][2:]) if a[0][0:2]==b[0][0:2] else int(a[0][0:2])-int(b[0][0:2]))) #使用OrderedDict解决显示输出表格排序的问题
for k, v in rowOrderedDict.items():
# row = OrderedDict(zip(colNameDict[k][0], v))
# 列名一样的话，使用zip会自动删除一列！注意,使用dict会打乱顺序，使用OrderDict解决
colNames = [colName.decode('utf8')[:60].encode('utf8') for colName in colNameDict[k][0]] # 限定列名为公式而导致过长的情况，只限定75字节
tableOrderedDict.setdefault(colNameDict[k][1]+" ---> "+colNameDict[k][2]+"(更新时间:"+colNameDict[k][3]+")", {"colName":colNames,"rowData":v,"tabIndex":k})
# 如何排序，按键值。导入相同的文件，相同表名，字典会被重置
return tableOrderedDict
def parse_recipe(self, response, title, picture):
hxs = HtmlXPathSelector(response)
section = None
sections = OrderedDict()
for node in hxs.select(xp_ingredients):
text = html_to_text(node.extract()).strip()
if not text:
continue
if node.select('strong'):
section = text
continue
sections.setdefault(section, []).append(text)
ingredients = sections.pop(None, None) or sections.pop(sections.keys()[-1])
extra_ingredients = [x for y in sections.values() for x in y]
yield CocktailItem(
title=title,
picture=picture,
url=response.url,
ingredients=ingredients,
extra_ingredients=extra_ingredients
)
def normalizeRAmemberlist(inputstream,filterexpdate=None):
#----------------------------------------------------------------------
'''
Take RunningAHEAD membership list (Export individual membership records), and produce member list.
For a given expiration date, the earliest renewal date is used
This allows "first renewal for year" computations
:param inputstream: open file with csv exported from RunningAHEAD (e.g., from request.files['file'].stream)
:param filterexpdate: yyyy-mm-dd for expiration date to filter on, else None
:rtype: csv file data, string format (e.g., data for make_response(data))
'''
memberships = csv.DictReader(inputstream)
members = OrderedDict({})
# for each membership, unique member can be found from combination of names, dob, gender (should be unique)
# make list of memberships for this member
for mship in memberships:
first = mship['GivenName']
middle = mship['MiddleName']
last = mship['FamilyName']
gender = mship['Gender']
dob = mship['DOB']
# make list of memberships for this member
members.setdefault((last,first,middle,dob,gender),[]).append(mship)
# get ready for output
outdatalist = wlist()
OUT = csv.DictWriter(outdatalist,MINHDR,extrasaction='ignore')
OUT.writeheader()
# for each member, find earliest renew date for each expiration date
for key in members:
renewbyexp = {}
thesememberships = members[key]
for mship in thesememberships:
thisexpdate = mship['ExpirationDate']
thisrenewdate = mship['RenewalDate']
# special case for 11/11/2013, FSRC bulk import
if thisrenewdate =='2013-11-11':
thisrenewdate = mship['JoinDate']
# optional filter
if not filterexpdate or thisexpdate == filterexpdate:
if thisexpdate not in renewbyexp or thisrenewdate < renewbyexp[thisexpdate]:
renewbyexp[thisexpdate] = thisrenewdate
# for each expiration date (already filtered appropriately),
# grab a record, update renewal and expiration dates, then save
for thisexpdate in renewbyexp:
outrec = copy(members[key][-1]) # doesn't matter which one, pick the last
outrec['RenewalDate'] = renewbyexp[thisexpdate]
outrec['ExpirationDate'] = thisexpdate
OUT.writerow(outrec) # note this adds \r\n
# one big string for return data
outputdata = ''.join(outdatalist)
return outputdata
def main():
args = arguments()
motif_sites = load_sitecounts(args.input_file, args.cutoff, args.proxyBED)
if args.output_dir:
outdir = args.output_dir
else:
outdir = ''
double_sitecounts = OrderedDict()
for motif in os.listdir(args.dirname):
motevo_output_file = os.path.join(args.dirname, motif)
double_sitecounts.setdefault(motif, 0)
double_sitecounts[motif], dist = \
count_nonoverlapping_motifs(motif_sites, motevo_output_file, args.cutoff)
fname = os.path.join(outdir, '%s.dist' % motif)
with open(fname, 'w') as outf:
for i, d in enumerate(dist):
outf.write('\t'.join([
str(i),
str(d) + '\n',
]))
with open( os.path.join(outdir, os.path.basename(args.input_file)), 'w') as outf:
outf.write('\t'.join([motif for motif in double_sitecounts.keys()]) + '\n')
for region in motif_sites.keys():
outf.write(region + '\t')
outf.write('\t'.join([str(double_sitecounts[motif][region]) for motif in double_sitecounts.keys()]))
outf.write('\n')
return 0
def get_milestones_choices(self, milestones):
data = OrderedDict()
for milestone in milestones:
data.setdefault(milestone.state, []).append(
(milestone.id, milestone.title)
)
return [('', 'No milestone')] + list(data.items())
def archives(request) :
try:
post_date = Article.objects.dates('pub_date', 'month')
post_date = post_date.reverse()
post_date_article = []
for i in range(len(post_date)):
post_date_article.append([])
for i in range(len(post_date)):
cur_year = post_date[i].year
cur_month = post_date[i].month
temp_arc = Article.objects.filter(pub_date__year=cur_year).filter(pub_date__month = cur_month)
post_date_article[i] = temp_arc
dicts = OrderedDict()
for i in range(len(post_date)):
dicts.setdefault(post_date[i], post_date_article[i])
except Article.DoesNotExist:
raise Http404
return render(request, 'archives.html', {
'select_module': 'archive',
'dicts' : dicts,
'error' : False,
'tag_cloud' : tagCloud()})
def get_data_values(time_series_results): # type: (QuerySet) -> object
result_ids = [result_id[0] for result_id in time_series_results.values_list('pk')]
data_values_queryset = TimeSeriesResultValue.objects.filter(result_id__in=result_ids).order_by('value_datetime').values('value_datetime', 'value_datetime_utc_offset', 'result_id', 'data_value')
data_values_map = OrderedDict()
for value in data_values_queryset:
data_values_map.setdefault(value['value_datetime'], {}).update({
'utc_offset': value['value_datetime_utc_offset'],
value['result_id']: value['data_value']
})
data = []
for timestamp, values in data_values_map.iteritems():
local_timestamp = timestamp + timedelta(hours=values['utc_offset'])
row = [
local_timestamp.strftime(CSVDataApi.date_format), # Local DateTime
'{0}:00'.format(values['utc_offset']), # UTC Offset
timestamp.strftime(CSVDataApi.date_format) # UTC DateTime
]
for result_id in result_ids:
try:
row.append(values[result_id])
except KeyError:
row.append('')
data.append(row)
return data
def compress(cls, operators):
sets = OrderedDict()
incs = OrderedDict()
rval = []
for op in operators:
if isinstance(op, cls):
if op.sets:
assert len(op.sets) == 1 and len(op.incs) == 0
sets.setdefault(op.sets[0], []).append(op)
else:
assert len(op.incs) == 1 and len(op.sets) == 0
incs.setdefault(op.incs[0], []).append(op)
else:
rval.append(op)
# combine incs into sets if on same view
for view, set_ops in iteritems(sets):
set_op, = set_ops
inc_ops = incs.get(view, [])
for inc_op in inc_ops[:]:
set_op.As.extend(inc_op.As)
set_op.Xs.extend(inc_op.Xs)
inc_ops.remove(inc_op)
rval.append(set_op)
# combine remaining incs if on same view
for view, inc_ops in iteritems(incs):
if len(inc_ops) > 0:
inc_op0 = inc_ops[0]
for inc_op in inc_ops[1:]:
inc_op0.As.extend(inc_op.As)
inc_op0.Xs.extend(inc_op.Xs)
rval.append(inc_op0)
return rval
def _parseStats(self,str_statsData,counter=False):
listHeaderData = []
listInterfaces = []
dictInterfaceData = OrderedDict()
listStatsData = str_statsData.split("#")[1].lstrip().rstrip().split("\n")
for interfaceIndex in range(len(listStatsData)):
str_interfaceData = ""
listValues = listStatsData[interfaceIndex].split(",")
if interfaceIndex == 0:
for eachValue in range(len(listValues)-1):
listHeaderData.append(listValues[eachValue])
#print("\n HeaderData \n" + str(listHeaderData))
else:
str_dictKey = str(listValues[0])+'_'+str(listValues[1])
if str_dictKey in listInterfaces:
continue
else:
listInterfaces.append(str_dictKey)
for eachValue in range(len(listValues)-1):
if listHeaderData[eachValue] in dict_reqdMet.keys():
dictInterfaceData.setdefault(str_dictKey+ "_" + str(dict_reqdMet[listHeaderData[eachValue]]),listValues[eachValue])
if (counter == True and (listHeaderData[eachValue] == 'bin' or listHeaderData[eachValue] == 'bout')):
finalValue = ((int(listValues[eachValue]) - int(self.dictCounterValues[str_dictKey+ "_" + str(dict_reqdMet[listHeaderData[eachValue]])]))/5)
dictInterfaceData[str_dictKey+ "_" + str(dict_reqdMet[listHeaderData[eachValue]])] = str(finalValue)
else:
pass
if counter==False:
self.dictCounterValues = dictInterfaceData
else:
dictInterfaceData['plugin_version'] = PLUGIN_VERSION
dictInterfaceData['heartbeat_required']=HEARTBEAT
print(str(json.dumps(dictInterfaceData)))
def fatlink_statistics_view(request, year=datetime.date.today().year, month=datetime.date.today().month):
year = int(year)
month = int(month)
start_of_month = datetime.datetime(year, month, 1)
start_of_next_month = first_day_of_next_month(year, month)
start_of_previous_month = first_day_of_previous_month(year, month)
fatStats = OrderedDict()
if settings.IS_CORP:
fatStats[settings.CORP_NAME] = CorpStat(settings.CORP_ID)
else:
alliance_corps = EveCorporationInfo.objects.filter(alliance__alliance_id=settings.ALLIANCE_ID)
for corp in alliance_corps:
fatStats[corp.corporation_name] = CorpStat(corp.corporation_id, corp=corp)
fatlinks_in_span = Fatlink.objects.filter(fatdatetime__gte = start_of_month).filter(fatdatetime__lt = start_of_next_month)
for fatlink in fatlinks_in_span:
fats_in_fatlink = Fat.objects.filter(fatlink=fatlink)
for fat in fats_in_fatlink:
fatStats.setdefault(fat.character.corporation_name,
CorpStat(fat.character.corporation_id, blue=True)
).n_fats += 1
fatStatsList = [fatStat for corp_name, fatStat in fatStats.items()]
fatStatsList.sort(key=lambda stat: stat.corp.corporation_name)
fatStatsList.sort(key=lambda stat: (stat.n_fats, stat.n_fats/stat.corp.member_count), reverse=True)
if datetime.datetime.now() > start_of_next_month:
context = {'fatStats':fatStatsList, 'month':start_of_month.strftime("%B"), 'year':year, 'previous_month': start_of_previous_month,'next_month': start_of_next_month}
else:
context = {'fatStats':fatStatsList, 'month':start_of_month.strftime("%B"), 'year':year, 'previous_month': start_of_previous_month}
return render_to_response('registered/fatlinkstatisticsview.html', context, context_instance=RequestContext(request))
def iet_make(stree):
"""Create an IET from a ScheduleTree."""
nsections = 0
queues = OrderedDict()
for i in stree.visit():
if i == stree:
# We hit this handle at the very end of the visit
return List(body=queues.pop(i))
elif i.is_Exprs:
exprs = [Increment(e) if e.is_Increment else Expression(e) for e in i.exprs]
body = ExpressionBundle(i.shape, i.ops, i.traffic, body=exprs)
elif i.is_Conditional:
body = Conditional(i.guard, queues.pop(i))
elif i.is_Iteration:
# Order to ensure deterministic code generation
uindices = sorted(i.sub_iterators, key=lambda d: d.name)
# Generate Iteration
body = Iteration(queues.pop(i), i.dim, i.dim._limits, offsets=i.limits,
direction=i.direction, uindices=uindices)
elif i.is_Section:
body = Section('section%d' % nsections, body=queues.pop(i))
nsections += 1
elif i.is_Halo:
body = HaloSpot(i.halo_scheme, body=queues.pop(i))
queues.setdefault(i.parent, []).append(body)
assert False
def map(self,coordinates, filelist, strand=True, col=5):
# read coordinates
mapto = OrderedDict()
with open(coordinates) as coor:
for lin in coor:
line_split = lin.split('\t')
if strand:
mapto.setdefault(line_split[0]+line_split[1]+line_split[2]+line_split[5].strip('\n'),[]).append(lin.strip('\n'))
else:
mapto.setdefault(line_split[0]+line_split[1]+line_split[2],[]).append(lin.strip('\n'))
for fname in filelist:
run_mapto = deepcopy(mapto)
with open(fname) as f:
for lin in f:
line_split = lin.split('\t')
if strand:
cor = line_split[0]+line_split[1]+line_split[2]+line_split[5].strip('\n')
else:
cor = line_split[0]+line_split[1]+line_split[2]
run_mapto[cor].append(line_split[col-1])
with open(fname+'mapped','w') as fout:
for key in run_mapto:
if len(run_mapto[key]) == 1:
run_mapto[key].append('0')
fout.write('\t'.join(run_mapto[key])+'\n')
class ArgsCollection(object):
def __init__(self, *args, **kwargs):
self._items = OrderedDict()
for arg in args:
k, _, v = arg.partition('=')
k = k.lstrip('-')
if not kwargs.get('flatten'):
self._items.setdefault(k, []).append(v)
else:
self._items[k] = v
def get(self, k, default=None):
return self._items.get(k, default)
def items(self):
return self._items.items()
def __getattr__(self, k, default=None):
return self._items.get(k, default)
def __contains__(self, k):
return k in self._items
def __getitem__(self, k):
return self._items['k']
def normalize_urls(urls, platform=None, offline_only=False):
platform = platform or subdir
defaults = tuple(x.rstrip('/') + '/' for x in get_default_urls())
alias = binstar_channel_alias(channel_alias)
def normalize_(url):
url = url.rstrip('/')
if is_url(url):
url_s = canonical_channel_name(url, True)
else:
url_s = url
url = alias + url
return url_s, url
newurls = OrderedDict()
priority = 0
while urls:
url = urls[0]
urls = urls[1:]
if url == "system" and rc_path:
urls = get_rc_urls() + urls
continue
elif url in ("defaults", "system"):
t_urls = defaults
elif url == "local":
t_urls = get_local_urls()
else:
t_urls = [url]
priority += 1
for url0 in t_urls:
url_s, url0 = normalize_(url0)
if offline_only and not url0.startswith('file:'):
continue
for plat in (platform, 'noarch'):
newurls.setdefault('%s/%s/' % (url0, plat), (url_s, priority))
return newurls
def get_mx(mail):
"""
returns ordered dict of MX records for email's domain
:param mail: email adress
:return: OrderedDict
"""
domain = getdomain(mail)
try:
mx_list = dns.resolver.query(domain, 'MX')
except dns.exception.DNSException as error:
if isinstance(error, dns.resolver.NoAnswer):
raise LackOfMxDomainException('Can\'t get MX from DNS server.')
elif isinstance(error, dns.resolver.Timeout):
raise DnsTimeoutException('Timeout on DNS query')
else:
raise Exception('Unknown DNS exception.')
servers = dict()
ordered_response = OrderedDict()
for server in mx_list:
priority, mx_domain = str(server).split(' ')
mx_domain = sub(r'\.$', '', mx_domain)
servers.setdefault(int(priority), list()).append(mx_domain)
for priority in sorted(servers):
ordered_response.setdefault(priority, servers.get(priority, list()))
return ordered_response
def generate_types(self, types):
f = self._f_h
self.loader.write_header(f)
self.write_api_header(f)
dedup_types = OrderedDict()
for type in types:
dedup_types.setdefault(type.name, []).append(type)
for types in dedup_types.values():
type = types[0]
output_string = (type.raw + '\n').lstrip().replace(' ', ' ')
if self.omit_khrplatform:
output_string = replace_khr_types(output_string)
if output_string == '#include <KHR/khrplatform.h>\n':
if self.omit_khrplatform:
continue
elif self.local_files:
output_string = '#include "khrplatform.h"\n'
if 'ptrdiff_t' in output_string:
# 10.6 is the last version supporting more than 64 bit (>1060)
output_string = \
'#if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) ' +\
'&& (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ > 1060)\n' +\
output_string.replace('ptrdiff_t', 'long') + '#else\n' + output_string + '#endif\n'
f.write(output_string)
def visualize(model, filename):
values = model.all_params()
cols = OrderedDict()
for key, value in values.items():
if isinstance(value, np.ndarray): # TODO group by prefix, showing W and b side by side
cols.setdefault(key[:-2] + key[-1] if len(key) > 1 and key[-2] in "bW" else key, []).append((key, value))
_, axes = plt.subplots(2, len(cols), figsize=(5*len(cols), 10)) # TODO https://stackoverflow.com/a/13784887/223267
plt.tight_layout()
for j, col in enumerate(tqdm(cols.values(), unit="param", desc=filename)):
for i in range(2):
axis = axes[i, j] if len(values) > 1 else axes
if len(col) <= i:
plt.delaxes(axis)
else:
plt.sca(axis)
key, value = col[i]
plt.colorbar(plt.pcolormesh(smooth(value)))
for pattern, repl in REPLACEMENTS: # TODO map 0123->ifoc
key = re.sub(pattern, repl, key)
plt.title(key)
for axis in (plt.gca().xaxis, plt.gca().yaxis):
axis.set_major_locator(MaxNLocator(integer=True))
output_file = filename + ".png"
plt.savefig(output_file)
plt.clf()
print("Saved '%s'." % output_file)
def results(self, identifier, is_public):
"""
Return the results created by the job with the given identifier
that were uploaded to S3.
:param identifier: Unique identifier of the Spark job.
:param is_public: Whether the Spark job is public or not.
:return: A mapping of result prefixes to lists of results.
:rtype: dict
"""
if is_public:
bucket = self.config['PUBLIC_DATA_BUCKET']
else:
bucket = self.config['PRIVATE_DATA_BUCKET']
params = {
'Prefix': '%s/' % identifier,
'Bucket': bucket,
}
results = OrderedDict()
list_objects_v2_paginator = self.s3.get_paginator('list_objects_v2')
for page in list_objects_v2_paginator.paginate(**params):
for item in page.get('Contents', []):
try:
prefix = item['Key'].split('/')[1]
except IndexError:
continue
results.setdefault(prefix, []).append(item['Key'])
return results
def read_bed(file_path, restricted_genes=None):
"""Reads BED file and populates a dictionary separating genes
by chromosome.
Parameters
----------
file_path : str
path to BED file
filtered_genes: list
list of gene names to not use
Returns
-------
bed_dict: dict
dictionary mapping chromosome keys to a list of BED lines
"""
# read in entire bed file into a dict with keys as chromsomes
bed_dict = OrderedDict()
for bed_row in bed_generator(file_path):
is_restrict_flag = restricted_genes is None or bed_row.gene_name in restricted_genes
if is_restrict_flag:
bed_dict.setdefault(bed_row.chrom, [])
bed_dict[bed_row.chrom].append(bed_row)
sort_chroms = sorted(bed_dict.keys(), key=lambda x: len(bed_dict[x]), reverse=True)
bed_dict = OrderedDict((chrom, bed_dict[chrom]) for chrom in sort_chroms)
return bed_dict
def _update_tiered_policy(self, ep_id):
"""
Sends an updated list of tiered policy to an endpoint.
Recalculates the list.
:param ep_id: ID of the endpoint to send an update to.
"""
_log.debug("Updating policies for %s from %s", ep_id,
self.pol_ids_by_ep_id)
# Order the profiles by tier and profile order, using the name of the
# tier and profile as a tie-breaker if the orders are the same.
profiles = []
for pol_id in self.pol_ids_by_ep_id.iter_values(ep_id):
try:
tier_order = self.tier_orders[pol_id.tier]
except KeyError:
_log.warn("Ignoring profile %s because its tier metadata is "
"missing.")
continue
profile_order = self.profile_orders[pol_id]
profiles.append((tier_order, pol_id.tier,
profile_order, pol_id.policy_id,
pol_id))
profiles.sort()
# Convert to an ordered dict from tier to list of profiles.
pols_by_tier = OrderedDict()
for _, tier, _, _, pol_id in profiles:
pols_by_tier.setdefault(tier, []).append(pol_id)
endpoint = self.objects_by_id[ep_id]
endpoint.on_tiered_policy_update(pols_by_tier, async=True)
def prizes_view(request, key=None):
givings = request.contest.prizegiving_set \
.filter(state='SUCCESS', prize__prizeforuser__isnull=False) \
.distinct()
groups = OrderedDict()
for id, name in givings.values_list('id', 'name'):
groups.setdefault(name, []).append(id)
Group = namedtuple('Group', 'name ids')
groups = list(map(Group._make, list(groups.items())))
key = int(key) if key is not None else groups[0].ids[0]
group = [group for group in groups if group.ids[0] == key]
if not group:
raise Http404
group = group[0]
pfus = PrizeForUser.objects.select_related('user', 'prize') \
.filter(prize__prize_giving__in=group.ids)
return TemplateResponse(request, 'prizes/prizes.html',
{'groups': groups, 'key': key, 'pfus': pfus,
'users_on_page': getattr(settings, 'PRIZES_ON_PAGE', 100)})
class Questionnaire:
"""Class with methods for adding questions to a questionnaire, and running
the questionnaire. Additional keyword args are passed to the prompter
method when it is called.
"""
def __init__(self, show_answers=True, can_go_back=True):
self.questions = OrderedDict() # key -> list of Question instances
self.answers = OrderedDict() # key -> answer
self.show_answers = show_answers
self.can_go_back = can_go_back
def add(self, *args, **kwargs):
"""Add a Question instance to the questions dict. Each key points
to a list of Question instances with that key. Use the `question`
kwarg to pass a Question instance if you want, or pass in the same
args you would pass to instantiate a question.
"""
if 'question' in kwargs and isinstance(kwargs['question'], Question):
question = kwargs['question']
else:
question = Question(*args, **kwargs)
self.questions.setdefault(question.key, []).append(question)
return question
def one(self, *args, **kwargs):
kwargs['prompter'] = 'one'
return self.add(*args, **kwargs)
def many(self, *args, **kwargs):
kwargs['prompter'] = 'many'
return self.add(*args, **kwargs)
def raw(self, *args, **kwargs):
kwargs['prompter'] = 'raw'
return self.add(*args, **kwargs)
def remove(self, key):
"""Remove all questions associated with `key`. Raises exception if `key`
doesn't exist.
"""
return self.questions.pop(key)
def run(self):
"""Asks all remaining questions in the questionnaire, returns the answers.
"""
while not self.done:
self.ask()
return self.answers
@exit_on_keyboard_interrupt
def ask(self, error=None):
"""Asks the next question in the questionnaire and returns the answer,
unless user goes back.
"""
q = self.next_question
if q is None:
return
try:
answer = q.prompter(self.get_prompt(q, error), *q.prompter_args,
**q.prompter_kwargs)
except QuestionnaireGoBack as e:
steps = e.args[0] if e.args else 1
if steps == 0:
self.ask(
) # user can redo current question even if `can_go_back` is `False`
return
self.go_back(steps)
else:
if q._validate:
error = q._validate(answer)
if error:
self.ask(error)
return
if q._transform:
answer = q._transform(answer)
self.answers[q.key] = answer
return answer
def get_prompt(self, question, error=None):
parts = []
if self.show_answers:
parts.append(self.answer_display())
if error:
parts.append(error)
parts.append(question.prompt)
return '\n\n'.join(str(p) for p in parts)
@property
def next_question(self):
"""Returns the next `Question` in the questionnaire, or `None` if there
are no questions left. Returns first question for whose key there is no
answer and for which condition is satisfied, or for which there is no
condition.
"""
for key, questions in self.questions.items():
if key in self.answers:
continue
for question in questions:
if self.check_condition(question._condition):
return question
return None
def check_condition(self, condition):
"""Helper that returns True if condition is satisfied/doesn't exist.
"""
if not condition:
return True
for c in condition.conditions:
key, value, operator = c
if not operator(self.answers[key], value):
return False
return True
def go_back(self, n=1):
"""Move `n` questions back in the questionnaire by removing the last `n`
answers.
"""
if not self.can_go_back:
return
N = max(len(self.answers) - abs(n), 0)
self.answers = OrderedDict(islice(self.answers.items(), N))
@property
def done(self):
return self.next_question is None
def reset(self):
self.answers = OrderedDict()
def format_answers(self, fmt='obj'):
"""Formats answers depending on `fmt`.
"""
fmts = ('obj', 'array', 'plain')
if fmt not in fmts:
eprint("Error: '{}' not in {}".format(fmt, fmts))
return
def stringify(val):
if type(val) in (list, tuple):
return ', '.join(str(e) for e in val)
return val
if fmt == 'obj':
return json.dumps(self.answers)
elif fmt == 'array':
answers = [[k, v] for k, v in self.answers.items()]
return json.dumps(answers)
elif fmt == 'plain':
answers = '\n'.join('{}: {}'.format(k, stringify(v))
for k, v in self.answers.items())
return answers
def answer_display(self, s=''):
"""Helper method for displaying the answers so far.
"""
padding = len(max(self.questions.keys(), key=len)) + 5
for key in list(self.answers.keys()):
s += '{:>{}} : {}\n'.format(key, padding, self.answers[key])
return s
def ordered_default_dict_sample():
from collections import OrderedDict
result = OrderedDict()
for key, val in custom_generater():
result.setdefault(key, []).append(val)
class HTTPHeaderDict(MutableMapping):
"""
:param headers:
An iterable of field-value pairs. Must not contain multiple field names
when compared case-insensitively.
:param kwargs:
Additional field-value pairs to pass in to ``dict.update``.
A ``dict`` like container for storing HTTP Headers.
Field names are stored and compared case-insensitively in compliance with
RFC 7230. Iteration provides the first case-sensitive key seen for each
case-insensitive pair.
Using ``__setitem__`` syntax overwrites fields that compare equal
case-insensitively in order to maintain ``dict``'s api. For fields that
compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
in a loop.
If multiple fields that are equal case-insensitively are passed to the
constructor or ``.update``, the behavior is undefined and some will be
lost.
>>> headers = HTTPHeaderDict()
>>> headers.add('Set-Cookie', 'foo=bar')
>>> headers.add('set-cookie', 'baz=quxx')
>>> headers['content-length'] = '7'
>>> headers['SET-cookie']
'foo=bar, baz=quxx'
>>> headers['Content-Length']
'7'
"""
def __init__(self, headers=None, **kwargs):
super(HTTPHeaderDict, self).__init__()
self._container = OrderedDict()
if headers is not None:
if isinstance(headers, HTTPHeaderDict):
self._copy_from(headers)
else:
self.extend(headers)
if kwargs:
self.extend(kwargs)
def __setitem__(self, key, val):
self._container[key.lower()] = [key, val]
return self._container[key.lower()]
def __getitem__(self, key):
val = self._container[key.lower()]
return ', '.join(val[1:])
def __delitem__(self, key):
del self._container[key.lower()]
def __contains__(self, key):
return key.lower() in self._container
def __eq__(self, other):
if not isinstance(other, Mapping) and not hasattr(other, 'keys'):
return False
if not isinstance(other, type(self)):
other = type(self)(other)
return (dict((k.lower(), v) for k, v in self.itermerged()) ==
dict((k.lower(), v) for k, v in other.itermerged()))
def __ne__(self, other):
return not self.__eq__(other)
if not PY3: # Python 2
iterkeys = MutableMapping.iterkeys
itervalues = MutableMapping.itervalues
__marker = object()
def __len__(self):
return len(self._container)
def __iter__(self):
# Only provide the originally cased names
for vals in self._container.values():
yield vals[0]
def pop(self, key, default=__marker):
'''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
If key is not found, d is returned if given, otherwise KeyError is raised.
'''
# Using the MutableMapping function directly fails due to the private marker.
# Using ordinary dict.pop would expose the internal structures.
# So let's reinvent the wheel.
try:
value = self[key]
except KeyError:
if default is self.__marker:
raise
return default
else:
del self[key]
return value
def discard(self, key):
try:
del self[key]
except KeyError:
pass
def add(self, key, val):
"""Adds a (name, value) pair, doesn't overwrite the value if it already
exists.
>>> headers = HTTPHeaderDict(foo='bar')
>>> headers.add('Foo', 'baz')
>>> headers['foo']
'bar, baz'
"""
key_lower = key.lower()
new_vals = [key, val]
# Keep the common case aka no item present as fast as possible
vals = self._container.setdefault(key_lower, new_vals)
if new_vals is not vals:
vals.append(val)
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError("extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args)))
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
def getlist(self, key, default=__marker):
"""Returns a list of all the values for the named field. Returns an
empty list if the key doesn't exist."""
try:
vals = self._container[key.lower()]
except KeyError:
if default is self.__marker:
return []
return default
else:
return vals[1:]
# Backwards compatibility for httplib
getheaders = getlist
getallmatchingheaders = getlist
iget = getlist
# Backwards compatibility for http.cookiejar
get_all = getlist
def __repr__(self):
return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
def _copy_from(self, other):
for key in other:
val = other.getlist(key)
if isinstance(val, list):
# Don't need to convert tuples
val = list(val)
self._container[key.lower()] = [key] + val
def copy(self):
clone = type(self)()
clone._copy_from(self)
return clone
def iteritems(self):
"""Iterate over all header lines, including duplicate ones."""
for key in self:
vals = self._container[key.lower()]
for val in vals[1:]:
yield vals[0], val
def itermerged(self):
"""Iterate over all headers, merging duplicate ones together."""
for key in self:
val = self._container[key.lower()]
yield val[0], ', '.join(val[1:])
def items(self):
return list(self.iteritems())
@classmethod
def from_httplib(cls, message): # Python 2
"""Read headers from a Python 2 httplib message object."""
# python2.7 does not expose a proper API for exporting multiheaders
# efficiently. This function re-reads raw lines from the message
# object and extracts the multiheaders properly.
obs_fold_continued_leaders = (' ', '\t')
headers = []
for line in message.headers:
if line.startswith(obs_fold_continued_leaders):
if not headers:
# We received a header line that starts with OWS as described
# in RFC-7230 S3.2.4. This indicates a multiline header, but
# there exists no previous header to which we can attach it.
raise InvalidHeader(
'Header continuation with no previous header: %s' % line
)
else:
key, value = headers[-1]
headers[-1] = (key, value + ' ' + line.strip())
continue
key, value = line.split(':', 1)
headers.append((key, value.strip()))
return cls(headers)
class HashedFilesMixin(object):
default_template = """url("%s")"""
patterns = (("*.css", (
r"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""",
(r"""(@import\s*["']\s*(.*?)["'])""", """@import url("%s")"""),
)), )
def __init__(self, *args, **kwargs):
super(HashedFilesMixin, self).__init__(*args, **kwargs)
self._patterns = OrderedDict()
self.hashed_files = {}
for extension, patterns in self.patterns:
for pattern in patterns:
if isinstance(pattern, (tuple, list)):
pattern, template = pattern
else:
template = self.default_template
compiled = re.compile(pattern, re.IGNORECASE)
self._patterns.setdefault(extension, []).append(
(compiled, template))
def file_hash(self, name, content=None):
"""
Returns a hash of the file with the given name and optional content.
"""
if content is None:
return None
md5 = hashlib.md5()
for chunk in content.chunks():
md5.update(chunk)
return md5.hexdigest()[:12]
def hashed_name(self, name, content=None):
parsed_name = urlsplit(unquote(name))
clean_name = parsed_name.path.strip()
opened = False
if content is None:
if not self.exists(clean_name):
raise ValueError("The file '%s' could not be found with %r." %
(clean_name, self))
try:
content = self.open(clean_name)
except IOError:
# Handle directory paths and fragments
return name
opened = True
try:
file_hash = self.file_hash(clean_name, content)
finally:
if opened:
content.close()
path, filename = os.path.split(clean_name)
root, ext = os.path.splitext(filename)
if file_hash is not None:
file_hash = ".%s" % file_hash
hashed_name = os.path.join(path, "%s%s%s" % (root, file_hash, ext))
unparsed_name = list(parsed_name)
unparsed_name[2] = hashed_name
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
if '?#' in name and not unparsed_name[3]:
unparsed_name[2] += '?'
return urlunsplit(unparsed_name)
def url(self, name, force=False):
"""
Returns the real URL in DEBUG mode.
"""
if settings.DEBUG and not force:
hashed_name, fragment = name, ''
else:
clean_name, fragment = urldefrag(name)
if urlsplit(clean_name).path.endswith('/'): # don't hash paths
hashed_name = name
else:
hashed_name = self.stored_name(clean_name)
final_url = super(HashedFilesMixin, self).url(hashed_name)
# Special casing for a @font-face hack, like url(myfont.eot?#iefix")
# http://www.fontspring.com/blog/the-new-bulletproof-font-face-syntax
query_fragment = '?#' in name # [sic!]
if fragment or query_fragment:
urlparts = list(urlsplit(final_url))
if fragment and not urlparts[4]:
urlparts[4] = fragment
if query_fragment and not urlparts[3]:
urlparts[2] += '?'
final_url = urlunsplit(urlparts)
return unquote(final_url)
def url_converter(self, name, template=None):
"""
Returns the custom URL converter for the given file name.
"""
if template is None:
template = self.default_template
def converter(matchobj):
"""
Converts the matched URL depending on the parent level (`..`)
and returns the normalized and hashed URL using the url method
of the storage.
"""
matched, url = matchobj.groups()
# Completely ignore http(s) prefixed URLs,
# fragments and data-uri URLs
if url.startswith(('#', 'http:', 'https:', 'data:', '//')):
return matched
name_parts = name.split(os.sep)
# Using posix normpath here to remove duplicates
url = posixpath.normpath(url)
url_parts = url.split('/')
parent_level, sub_level = url.count('..'), url.count('/')
if url.startswith('/'):
sub_level -= 1
url_parts = url_parts[1:]
if parent_level or not url.startswith('/'):
start, end = parent_level + 1, parent_level
else:
if sub_level:
if sub_level == 1:
parent_level -= 1
start, end = parent_level, 1
else:
start, end = 1, sub_level - 1
joined_result = '/'.join(name_parts[:-start] + url_parts[end:])
hashed_url = self.url(unquote(joined_result), force=True)
file_name = hashed_url.split('/')[-1:]
relative_url = '/'.join(url.split('/')[:-1] + file_name)
# Return the hashed version to the file
return template % unquote(relative_url)
return converter
def post_process(self, paths, dry_run=False, **options):
"""
Post process the given OrderedDict of files (called from collectstatic).
Processing is actually two separate operations:
1. renaming files to include a hash of their content for cache-busting,
and copying those files to the target storage.
2. adjusting files which contain references to other files so they
refer to the cache-busting filenames.
If either of these are performed on a file, then that file is considered
post-processed.
"""
# don't even dare to process the files if we're in dry run mode
if dry_run:
return
# where to store the new paths
hashed_files = OrderedDict()
# build a list of adjustable files
matches = lambda path: matches_patterns(path, self._patterns.keys())
adjustable_paths = [path for path in paths if matches(path)]
# then sort the files by the directory level
path_level = lambda name: len(name.split(os.sep))
for name in sorted(paths.keys(), key=path_level, reverse=True):
# use the original, local file, not the copied-but-unprocessed
# file, which might be somewhere far away, like S3
storage, path = paths[name]
with storage.open(path) as original_file:
# generate the hash with the original content, even for
# adjustable files.
hashed_name = self.hashed_name(name, original_file)
# then get the original's file content..
if hasattr(original_file, 'seek'):
original_file.seek(0)
hashed_file_exists = self.exists(hashed_name)
processed = False
# ..to apply each replacement pattern to the content
if name in adjustable_paths:
content = original_file.read().decode(
settings.FILE_CHARSET)
for patterns in self._patterns.values():
for pattern, template in patterns:
converter = self.url_converter(name, template)
try:
content = pattern.sub(converter, content)
except ValueError as exc:
yield name, None, exc
if hashed_file_exists:
self.delete(hashed_name)
# then save the processed result
content_file = ContentFile(force_bytes(content))
saved_name = self._save(hashed_name, content_file)
hashed_name = force_text(self.clean_name(saved_name))
processed = True
else:
# or handle the case in which neither processing nor
# a change to the original file happened
if not hashed_file_exists:
processed = True
saved_name = self._save(hashed_name, original_file)
hashed_name = force_text(self.clean_name(saved_name))
# and then set the cache accordingly
hashed_files[self.hash_key(name)] = hashed_name
yield name, hashed_name, processed
# Finally store the processed paths
self.hashed_files.update(hashed_files)
def clean_name(self, name):
return name.replace('\\', '/')
def hash_key(self, name):
return name
def stored_name(self, name):
hash_key = self.hash_key(name)
cache_name = self.hashed_files.get(hash_key)
if cache_name is None:
cache_name = self.clean_name(self.hashed_name(name))
# store the hashed name if there was a miss, e.g.
# when the files are still processed
self.hashed_files[hash_key] = cache_name
return cache_name
class DestroyHandler(toolbox.Bookkeeper): # noqa
"""
The DestroyHandler class detects when a graph is impossible to evaluate
because of aliasing and destructive operations.
Several data structures are used to do this.
An Op can use its view_map property to declare that an output may be
aliased to an input. If that output is destroyed, the input is also
considered to be destroyed. The view_maps of several Ops can feed into
one another and form a directed graph. The consequence of destroying any
variable in such a graph is that all variables in the graph must be
considered to be destroyed, because they could all be referring to the
same underlying storage.
In the current implementation, that graph is a tree, and the root of that
tree is called the foundation.
TODO: why "in the current implementation" ? is there another implementation
planned?
TODO: why is the graph a tree? isn't it possible that one variable could
be aliased to many variables? for example, don't switch and ifelse
have to do this?
The original DestroyHandler (if 0'ed out above) computed several data
structures from scratch each time it was asked to validate the graph.
Because this happens potentially thousands of times and each graph to
validate is extremely similar to the previous one, computing the
data structures from scratch repeatedly was wasteful and resulted in
high compile times for large graphs.
This implementation computes the data structures once at initialization
and then incrementally updates them.
It is a work in progress. The following data structures have been
converted to use the incremental strategy:
<none>
The following data structures remain to be converted:
<unknown>
"""
pickle_rm_attr = ["destroyers", "has_destroyers"]
def __init__(self, do_imports_on_attach=True, algo=None):
self.fgraph = None
self.do_imports_on_attach = do_imports_on_attach
"""
Maps every variable in the graph to its "foundation" (deepest
ancestor in view chain).
TODO: change name to var_to_vroot.
"""
self.droot = OrderedDict()
"""
Maps a variable to all variables that are indirect or direct views of it
(including itself) essentially the inverse of droot.
TODO: do all variables appear in this dict, or only those that are
foundations?
TODO: do only destroyed variables go in here? one old docstring said so.
TODO: rename to x_to_views after reverse engineering what x is
"""
self.impact = OrderedDict()
"""
If a var is destroyed, then this dict will map
droot[var] to the apply node that destroyed var
TODO: rename to vroot_to_destroyer
"""
self.root_destroyer = OrderedDict()
if algo is None:
algo = config.cycle_detection
self.algo = algo
self.fail_validate = OrderedDict()
def on_attach(self, fgraph):
"""
When attaching to a new fgraph, check that
1) This DestroyHandler wasn't already attached to some fgraph
(its data structures are only set up to serve one).
2) The FunctionGraph doesn't already have a DestroyHandler.
This would result in it validating everything twice, causing
compilation to be slower.
Give the FunctionGraph instance:
1) A new method "destroyers(var)"
TODO: what does this do exactly?
2) A new attribute, "destroy_handler"
TODO: WRITEME: what does this do besides the checks?
"""
# Do the checking #
already_there = False
if self.fgraph is fgraph:
already_there = True
if self.fgraph is not None:
raise Exception("A DestroyHandler instance can only serve one"
" FunctionGraph. (Matthew 6:24)")
for attr in ("destroyers", "destroy_handler"):
if hasattr(fgraph, attr):
already_there = True
if already_there:
# FunctionGraph.attach_feature catches AlreadyThere and cancels the attachment
raise toolbox.AlreadyThere(
"DestroyHandler feature is already present"
" or in conflict with another plugin.")
# Annotate the FunctionGraph #
self.unpickle(fgraph)
fgraph.destroy_handler = self
self.fgraph = fgraph
self.destroyers = (
OrderedSet()) # set of Apply instances with non-null destroy_map
self.view_i = {} # variable -> variable used in calculation
self.view_o = (
{}
) # variable -> set of variables that use this one as a direct input
# clients: how many times does an apply use a given variable
self.clients = OrderedDict() # variable -> apply -> ninputs
self.stale_droot = True
self.debug_all_apps = set()
if self.do_imports_on_attach:
toolbox.Bookkeeper.on_attach(self, fgraph)
def unpickle(self, fgraph):
def get_destroyers_of(r):
droot, _, root_destroyer = self.refresh_droot_impact()
try:
return [root_destroyer[droot[r]]]
except Exception:
return []
fgraph.destroyers = get_destroyers_of
def has_destroyers(protected_list):
if self.algo != "fast":
droot, _, root_destroyer = self.refresh_droot_impact()
for protected_var in protected_list:
try:
root_destroyer[droot[protected_var]]
return True
except KeyError:
pass
return False
def recursive_destroys_finder(protected_var):
# protected_var is the idx'th input of app.
for (app, idx) in protected_var.clients:
if app == "output":
continue
destroy_maps = getattr(app.op, "destroy_map", {}).values()
# If True means that the apply node, destroys the protected_var.
if idx in [
dmap for sublist in destroy_maps
for dmap in sublist
]:
return True
for var_idx in getattr(app.op, "view_map", {}).keys():
if idx in app.op.view_map[var_idx]:
# We need to recursivly check the destroy_map of all the
# outputs that we have a view_map on.
if recursive_destroys_finder(app.outputs[var_idx]):
return True
return False
for protected_var in protected_list:
if recursive_destroys_finder(protected_var):
return True
return False
fgraph.has_destroyers = has_destroyers
def refresh_droot_impact(self):
"""
Makes sure self.droot, self.impact, and self.root_destroyer are up to
date, and returns them (see docstrings for these properties above).
"""
if self.stale_droot:
self.droot, self.impact, self.root_destroyer = _build_droot_impact(
self)
self.stale_droot = False
return self.droot, self.impact, self.root_destroyer
def on_detach(self, fgraph):
if fgraph is not self.fgraph:
raise Exception("detaching wrong fgraph", fgraph)
del self.destroyers
del self.view_i
del self.view_o
del self.clients
del self.stale_droot
assert self.fgraph.destroyer_handler is self
delattr(self.fgraph, "destroyers")
delattr(self.fgraph, "has_destroyers")
delattr(self.fgraph, "destroy_handler")
self.fgraph = None
def fast_destroy(self, app, reason):
"""
Do the check for only 1 level.
For now:
- Destroyed variables can have only 1 clients.
- Allow view to have multiple clients.
- Allow sequence of view.
- But don't allow to destroy view
"""
dm = getattr(app.op, "destroy_map", None)
if not dm:
return
inputs = set(itertools.chain.from_iterable(
dm.values())) # list of app's destroyed inputs
for inp_idx in inputs:
inp = app.inputs[inp_idx]
if getattr(inp.tag, "indestructible", False) or isinstance(
inp, graph.Constant):
self.fail_validate[app] = InconsistencyError(
"Attempting to destroy indestructible variables: %s" % inp)
elif len(inp.clients) > 1:
self.fail_validate[app] = aesara.gof.InconsistencyError(
"Destroyed variable has more than one client. " +
str(reason))
elif inp.owner:
app2 = inp.owner
inp_idx2 = app2.outputs.index(inp)
v = getattr(app2.op, "view_map", {})
d = getattr(app2.op, "destroy_map", {})
if v:
v = v.get(inp_idx2, [])
if len(v) > 0:
self.fail_validate[
app] = aesara.gof.InconsistencyError(
"Destroyed variable has view_map. " +
str(reason))
elif d:
d = d.get(inp_idx2, [])
if len(d) > 0:
self.fail_validate[
app] = aesara.gof.InconsistencyError(
"Destroyed variable has destroy_map. " +
str(reason))
# These 2 assertions are commented since this function is called so many times
# but they should be true.
# assert len(v) <= 1
# assert len(d) <= 1
def on_import(self, fgraph, app, reason):
"""
Add Apply instance to set which must be computed.
"""
if app in self.debug_all_apps:
raise ProtocolError("double import")
self.debug_all_apps.add(app)
# print 'DH IMPORT', app, id(app), id(self), len(self.debug_all_apps)
# If it's a destructive op, add it to our watch list
dmap = getattr(app.op, "destroy_map", None)
vmap = getattr(app.op, "view_map", {})
if dmap:
self.destroyers.add(app)
if self.algo == "fast":
self.fast_destroy(app, reason)
# add this symbol to the forward and backward maps
for o_idx, i_idx_list in vmap.items():
if len(i_idx_list) > 1:
raise NotImplementedError(
"destroying this output invalidates multiple inputs",
(app.op))
o = app.outputs[o_idx]
i = app.inputs[i_idx_list[0]]
self.view_i[o] = i
self.view_o.setdefault(i, OrderedSet()).add(o)
# update self.clients
for i, input in enumerate(app.inputs):
self.clients.setdefault(input, OrderedDict()).setdefault(app, 0)
self.clients[input][app] += 1
for i, output in enumerate(app.outputs):
self.clients.setdefault(output, OrderedDict())
self.stale_droot = True
def on_prune(self, fgraph, app, reason):
"""
Remove Apply instance from set which must be computed.
"""
if app not in self.debug_all_apps:
raise ProtocolError("prune without import")
self.debug_all_apps.remove(app)
# UPDATE self.clients
for input in set(app.inputs):
del self.clients[input][app]
if getattr(app.op, "destroy_map", OrderedDict()):
self.destroyers.remove(app)
# Note: leaving empty client dictionaries in the struct.
# Why? It's a pain to remove them. I think they aren't doing any harm, they will be
# deleted on_detach().
# UPDATE self.view_i, self.view_o
for o_idx, i_idx_list in getattr(app.op, "view_map",
OrderedDict()).items():
if len(i_idx_list) > 1:
# destroying this output invalidates multiple inputs
raise NotImplementedError()
o = app.outputs[o_idx]
i = app.inputs[i_idx_list[0]]
del self.view_i[o]
self.view_o[i].remove(o)
if not self.view_o[i]:
del self.view_o[i]
self.stale_droot = True
if app in self.fail_validate:
del self.fail_validate[app]
def on_change_input(self, fgraph, app, i, old_r, new_r, reason):
"""
app.inputs[i] changed from old_r to new_r.
"""
if app == "output":
# app == 'output' is special key that means FunctionGraph is redefining which nodes are being
# considered 'outputs' of the graph.
pass
else:
if app not in self.debug_all_apps:
raise ProtocolError("change without import")
# UPDATE self.clients
self.clients[old_r][app] -= 1
if self.clients[old_r][app] == 0:
del self.clients[old_r][app]
self.clients.setdefault(new_r, OrderedDict()).setdefault(app, 0)
self.clients[new_r][app] += 1
# UPDATE self.view_i, self.view_o
for o_idx, i_idx_list in getattr(app.op, "view_map",
OrderedDict()).items():
if len(i_idx_list) > 1:
# destroying this output invalidates multiple inputs
raise NotImplementedError()
i_idx = i_idx_list[0]
output = app.outputs[o_idx]
if i_idx == i:
if app.inputs[i_idx] is not new_r:
raise ProtocolError("wrong new_r on change")
self.view_i[output] = new_r
self.view_o[old_r].remove(output)
if not self.view_o[old_r]:
del self.view_o[old_r]
self.view_o.setdefault(new_r, OrderedSet()).add(output)
if self.algo == "fast":
if app in self.fail_validate:
del self.fail_validate[app]
self.fast_destroy(app, reason)
self.stale_droot = True
def validate(self, fgraph):
"""
Return None.
Raise InconsistencyError when
a) orderings() raises an error
b) orderings cannot be topologically sorted.
"""
if self.destroyers:
if self.algo == "fast":
if self.fail_validate:
app_err_pairs = self.fail_validate
self.fail_validate = OrderedDict()
# self.fail_validate can only be a hint that maybe/probably
# there is a cycle.This is because inside replace() we could
# record many reasons to not accept a change, but we don't
# know which one will fail first inside validate(). Thus,the
# graph might have already changed when we raise the
# self.fail_validate error. So before raising the error, we
# double check here.
for app in app_err_pairs:
if app in fgraph.apply_nodes:
self.fast_destroy(app, "validate")
if self.fail_validate:
self.fail_validate = app_err_pairs
raise app_err_pairs[app]
else:
ords = self.orderings(fgraph, ordered=False)
if _contains_cycle(fgraph, ords):
raise InconsistencyError(
"Dependency graph contains cycles")
else:
# James's Conjecture:
# If there are no destructive ops, then there can be no cycles.
# FB: This isn't always True. It can happened that
# optimization introduce node that depend on itself. This
# is very rare and should not happen in general. It will be
# caught later. The error will be far from the source. But
# doing this conjecture should speed up compilation most of
# the time. The user should create such dependency except
# if he mess too much with the internal.
pass
return True
def orderings(self, fgraph, ordered=True):
"""
Return orderings induced by destructive operations.
Raise InconsistencyError when
a) attempting to destroy indestructable variable, or
b) attempting to destroy a value multiple times, or
c) an Apply destroys (illegally) one of its own inputs by aliasing
"""
if ordered:
set_type = OrderedSet
rval = OrderedDict()
else:
set_type = set
rval = dict()
if self.destroyers:
# BUILD DATA STRUCTURES
# CHECK for multiple destructions during construction of variables
droot, impact, __ignore = self.refresh_droot_impact()
# check for destruction of constants
illegal_destroy = [
r for r in droot if getattr(r.tag, "indestructible", False)
or isinstance(r, graph.Constant)
]
if illegal_destroy:
raise InconsistencyError(
"Attempting to destroy indestructible variables: %s" %
illegal_destroy)
# add destroyed variable clients as computational dependencies
for app in self.destroyers:
# keep track of clients that should run before the current Apply
root_clients = set_type()
# for each destroyed input...
for output_idx, input_idx_list in app.op.destroy_map.items():
destroyed_idx = input_idx_list[0]
destroyed_variable = app.inputs[destroyed_idx]
root = droot[destroyed_variable]
root_impact = impact[root]
# we generally want to put all clients of things which depend on root
# as pre-requisites of app.
# But, app is itself one such client!
# App will always be a client of the node we're destroying
# (destroyed_variable, but the tricky thing is when it is also a client of
# *another variable* viewing on the root. Generally this is illegal, (e.g.,
# add_inplace(x, x.T). In some special cases though, the in-place op will
# actually be able to work properly with multiple destroyed inputs (e.g,
# add_inplace(x, x). An Op that can still work in this case should declare
# so via the 'destroyhandler_tolerate_same' attribute or
# 'destroyhandler_tolerate_aliased' attribute.
#
# destroyhandler_tolerate_same should be a list of pairs of the form
# [(idx0, idx1), (idx0, idx2), ...]
# The first element of each pair is the input index of a destroyed
# variable.
# The second element of each pair is the index of a different input where
# we will permit exactly the same variable to appear.
# For example, add_inplace.tolerate_same might be [(0,1)] if the destroyed
# input is also allowed to appear as the second argument.
#
# destroyhandler_tolerate_aliased is the same sort of list of
# pairs.
# op.destroyhandler_tolerate_aliased = [(idx0, idx1)] tells the
# destroyhandler to IGNORE an aliasing between a destroyed
# input idx0 and another input idx1.
# This is generally a bad idea, but it is safe in some
# cases, such as
# - the op reads from the aliased idx1 before modifying idx0
# - the idx0 and idx1 are guaranteed not to overlap (e.g.
# they are pointed at different rows of a matrix).
#
# CHECK FOR INPUT ALIASING
# OPT: pre-compute this on import
tolerate_same = getattr(app.op,
"destroyhandler_tolerate_same", [])
assert isinstance(tolerate_same, list)
tolerated = {
idx1
for idx0, idx1 in tolerate_same
if idx0 == destroyed_idx
}
tolerated.add(destroyed_idx)
tolerate_aliased = getattr(
app.op, "destroyhandler_tolerate_aliased", [])
assert isinstance(tolerate_aliased, list)
ignored = {
idx1
for idx0, idx1 in tolerate_aliased
if idx0 == destroyed_idx
}
for i, input in enumerate(app.inputs):
if i in ignored:
continue
if input in root_impact and (
i not in tolerated
or input is not destroyed_variable):
raise InconsistencyError(
"Input aliasing: %s (%i, %i)" %
(app, destroyed_idx, i))
# add the rule: app must be preceded by all other Apply instances that
# depend on destroyed_input
for r in root_impact:
assert not [
a for a, c in self.clients[r].items() if not c
]
root_clients.update(
[a for a, c in self.clients[r].items() if c])
# app itself is a client of the destroyed inputs,
# but should not run before itself
root_clients.remove(app)
if root_clients:
rval[app] = root_clients
return rval
class BackgroundLogger(BaseLogger):
"""A logger which can be used from multiple threads.
The messages from main thread will go to robot logging api (or Python
logging if Robot is not running). Messages from other threads are saved
to memory and can be later logged with ``log_background_messages()``.
This will also remove the messages from memory.
Example::
from robotbackgroundlogger import BackgroundLogger
logger = BackgroundLogger()
After that logger can be used mostly like ``robot.api.logger`::
logger.debug('Hello, world!')
logger.info('<b>HTML</b> example', html=True)
"""
LOGGING_THREADS = logger.librarylogger.LOGGING_THREADS
def __init__(self):
self.lock = threading.RLock()
self._messages = OrderedDict()
def write(self, msg, level, html=False):
with self.lock:
thread = threading.currentThread().getName()
if thread in self.LOGGING_THREADS:
logger.write(msg, level, html)
else:
message = BackgroundMessage(msg, level, html)
self._messages.setdefault(thread, []).append(message)
def log_background_messages(self, name=None):
"""Forwards messages logged on background to Robot Framework log.
By default forwards all messages logged by all threads, but can be
limited to a certain thread by passing thread's name as an argument.
Logged messages are removed from the message storage.
This method must be called from the main thread.
"""
thread = threading.currentThread().getName()
if thread not in self.LOGGING_THREADS:
raise RuntimeError(
"Logging background messages is only allowed from the main "
"thread. Current thread name: %s" % thread)
with self.lock:
if name:
self._log_messages_by_thread(name)
else:
self._log_all_messages()
def _log_messages_by_thread(self, name):
for message in self._messages.pop(name, []):
print(message.format())
def _log_all_messages(self):
for thread in list(self._messages):
# Only way to get custom timestamps currently is with print
print("*HTML* <b>Messages by '%s'</b>" % thread)
for message in self._messages.pop(thread):
print(message.format())
def reset_background_messages(self, name=None):
with self.lock:
if name:
self._messages.pop(name)
else:
self._messages.clear()
def push_comic_book(self, username, user, book, opts=None):
if not opts:
opts = getOpts(user.device, "comic")
oeb = CreateOeb(main.log, None, opts)
pubtype = 'book:book:KindleEar'
language = 'zh-cn'
setMetaData(
oeb,
book.title,
language,
local_time("%Y-%m-%d", user.timezone),
pubtype=pubtype,
)
oeb.container = ServerContainer(main.log)
#guide
id_, href = oeb.manifest.generate('masthead',
DEFAULT_MASTHEAD) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_MASTHEAD))
oeb.guide.add('masthead', 'Masthead Image', href)
id_, href = oeb.manifest.generate('cover', DEFAULT_COVER)
item = oeb.manifest.add(id_, href, MimeFromFilename(DEFAULT_COVER))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt, imgindex = 0, 0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
chapters = book.ParseFeedUrls()
if not chapters:
self.deliverlog(
username,
str(user.kindle_email),
book.title,
0,
status="nonews",
tz=user.timezone,
)
return
for (
bookname,
chapter_title,
img_list,
chapter_url,
next_chapter_index,
) in chapters:
try:
image_count = 0
for (
mime_or_section,
url,
filename,
content,
brief,
thumbnail,
) in book.gen_image_items(img_list, chapter_url):
if not mime_or_section or not filename or not content:
continue
if mime_or_section.startswith(r"image/"):
id_, href = oeb.manifest.generate(id="img",
href=filename)
item = oeb.manifest.add(id_,
href,
mime_or_section,
data=content)
if thumbnail:
toc_thumbnails[url] = href
else:
sections.setdefault(mime_or_section, [])
sections[mime_or_section].append(
(filename, brief, thumbnail, content))
image_count += 1
title = book.title + " " + chapter_title
if not image_count:
self.deliverlog(
username,
str(user.kindle_email),
title,
0,
status="can't download image",
tz=user.timezone,
)
rs = "No new feeds.2"
main.log.info(rs)
continue
insertHtmlToc = False
insertThumbnail = False
oeb.metadata.clear("title")
oeb.metadata.add("title", title)
InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc,
insertThumbnail)
oIO = byteStringIO()
o = EPUBOutput() if user.book_type == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
try:
ultima_log = DeliverLog.all().order("-time").get()
except:
ultima_log = sorted(DeliverLog.all(),
key=attrgetter("time"),
reverse=True)
ultima_log = ultima_log[0] if ultima_log else None
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 10:
time.sleep(8)
self.SendToKindle(
username,
user.kindle_email,
title,
user.book_type,
str(oIO.getvalue()),
user.timezone,
)
book.UpdateLastDelivered(bookname, chapter_title,
next_chapter_index)
rs = "%s(%s).%s Sent!" % (
title,
local_time(tz=user.timezone),
user.book_type,
)
main.log.info(rs)
except:
main.log.exception(u"Failed to push {} {}".format(
bookname, chapter_title))
def GET(self):
username = web.input().get("u")
bookid = web.input().get("id")
feedsId = web.input().get("feedsId")
user = KeUser.all().filter("name = ", username).get()
if not user:
return "User not exist!<br />"
to = user.kindle_email
if (';' in to) or (',' in to):
to = to.replace(',', ';').replace(' ', '').split(';')
to = list(filter(lambda x: x.find('@', 1,
len(x) - 1) > 0,
to)) #最简单的判断是否是EMAIL
booktype = user.book_type #mobi,epub
bookmode = user.book_mode or 'periodical' #periodical,comic
titlefmt = user.titlefmt
tz = user.timezone
bookid = bookid.split(',') if ',' in bookid else [bookid]
bks = []
for id_ in bookid:
try:
bks.append(Book.get_by_id(int(id_)))
except:
main.log.warn("Can't find book {}".format(id_))
continue
#return "id of book is invalid or book not exist!<br />"
# Deliver only some feeds in custom rss
if feedsId:
feedsId = [
int(item) for item in feedsId.split('|') if item.isdigit()
]
feedsId = [
Feed.get_by_id(item) for item in feedsId
if Feed.get_by_id(item)
]
book4meta = None
if len(bks) == 0:
return "No have book to push!"
elif len(bks) == 1:
if bks[0].builtin:
book4meta = BookClass(bks[0].title)
mhfile = book4meta.mastheadfile
coverfile = book4meta.coverfile
if issubclass(book4meta, BaseComicBook
): #如果单独推送一个继承自BaseComicBook的书籍，则自动设置为漫画模式
bookmode = 'comic'
else: #单独的推送自定义RSS
book4meta = bks[0]
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER
else: #多本书合并推送时使用“自定义RSS”的元属性
book4meta = user.ownfeeds
mhfile = DEFAULT_MASTHEAD
coverfile = DEFAULT_COVER_BV if user.merge_books else DEFAULT_COVER
if not book4meta:
return "No have book to push.<br />"
# 创建 OEB
#global log
opts = getOpts(user.device, bookmode)
oeb = CreateOeb(main.log, None, opts)
bookTitle = "%s %s" % (book4meta.title, local_time(
titlefmt, tz)) if titlefmt else book4meta.title
if bookmode == 'comic':
pubtype = 'book:book:KindleEar'
else:
pubtype = 'periodical:magazine:KindleEar'
setMetaData(oeb,
bookTitle,
book4meta.language,
local_time("%Y-%m-%d", tz),
pubtype=pubtype)
oeb.container = ServerContainer(main.log)
#guide
if mhfile:
id_, href = oeb.manifest.generate('masthead',
mhfile) # size:600*60
oeb.manifest.add(id_, href, MimeFromFilename(mhfile))
oeb.guide.add('masthead', 'Masthead Image', href)
if coverfile:
imgData = None
imgMime = ''
#使用保存在数据库的用户上传的封面
if coverfile == DEFAULT_COVER and user.cover:
imgData = user.cover
imgMime = 'image/jpeg' #保存在数据库中的只可能是jpeg格式
elif callable(coverfile): #如果封面需要回调的话
try:
imgData = book4meta().coverfile()
if imgData:
imgType = imghdr.what(None, imgData)
if imgType: #如果是合法图片
imgMime = r"image/" + imgType
else:
main.log.warn(
u'content of cover is invalid : [%s].' %
bookTitle)
imgData = None
except:
main.log.exception(u"Failed to fetch cover for book [%s]" %
bookTitle)
coverfile = DEFAULT_COVER
imgData = None
imgMime = ''
if imgData and imgMime:
id_, href = oeb.manifest.generate('cover', 'cover.jpg')
item = oeb.manifest.add(id_, href, imgMime, data=imgData)
else:
id_, href = oeb.manifest.generate('cover', coverfile)
item = oeb.manifest.add(id_, href, MimeFromFilename(coverfile))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
elif len(bks) > 1 and DEFAULT_COVER:
#将所有书籍的封面拼贴成一个
#如果DEFAULT_COVER=None说明用户不需要封面
id_, href = oeb.manifest.generate('cover', 'cover.jpg')
item = oeb.manifest.add(id_,
href,
'image/jpeg',
data=self.MergeCovers(bks, opts, user))
oeb.guide.add('cover', 'Cover', href)
oeb.metadata.add('cover', id_)
itemcnt, imgindex = 0, 0
sections = OrderedDict()
toc_thumbnails = {} #map img-url -> manifest-href
for bk in bks:
if bk.builtin:
cbook = BookClass(bk.title)
if not cbook:
main.log.warn('not exist book <%s>' % bk.title)
continue
book = cbook(imgindex=imgindex, opts=opts, user=user)
book.url_filters = [flt.url for flt in user.urlfilter]
if bk.needs_subscription: #需要登录
subs_info = user.subscription_info(bk.title)
if subs_info:
book.account = subs_info.account
book.password = subs_info.password
if issubclass(cbook, BaseComicBook):
self.push_comic_book(username, user, book)
continue
else: # 自定义RSS
if bk.feedscount == 0:
continue #return "the book has no feed!<br />"
book = BaseFeedBook(imgindex=imgindex, opts=opts, user=user)
book.title = bk.title
book.description = bk.description
book.language = bk.language
book.keep_image = bk.keep_image
book.oldest_article = bk.oldest_article
book.fulltext_by_readability = True
feeds = feedsId if feedsId else bk.feeds
book.feeds = []
for feed in feeds:
if feed.url.startswith(comic_domains):
self.ProcessComicRSS(username, user, feed)
else:
book.feeds.append(
(feed.title, feed.url, feed.isfulltext))
book.url_filters = [flt.url for flt in user.urlfilter]
# 对于html文件，变量名字自文档,thumbnail为文章第一个img的url
# 对于图片文件，section为图片mime,url为原始链接,title为文件名,content为二进制内容,
# img的thumbail仅当其为article的第一个img为True
# 对于CSS文件，sec_or_media 为 'text/css'，url 和 title 都为文件名
try: #书的质量可能不一，一本书的异常不能影响其他书籍的推送
for sec_or_media, url, title, content, brief, thumbnail in book.Items(
):
if not sec_or_media or not title or not content:
continue
if sec_or_media.startswith(r'image/'):
id_, href = oeb.manifest.generate(id='img', href=title)
item = oeb.manifest.add(id_,
href,
sec_or_media,
data=content)
if thumbnail:
toc_thumbnails[url] = href
imgindex += 1
elif sec_or_media == 'text/css':
if url not in oeb.manifest.hrefs: #Only one css needed
oeb.manifest.add('css',
url,
sec_or_media,
data=content)
else:
#id, href = oeb.manifest.generate(id='feed', href='feed%d.html'%itemcnt)
#item = oeb.manifest.add(id, href, 'application/xhtml+xml', data=content)
#oeb.spine.add(item, True)
sections.setdefault(sec_or_media, [])
sections[sec_or_media].append(
(title, brief, thumbnail, content))
itemcnt += 1
except Exception as e:
main.log.exception(u"Failed to push <%s>, Err:%s" %
(book.title, str(e)))
continue
volumeTitle = ''
if itemcnt > 0:
insertHtmlToc = GENERATE_HTML_TOC
insertThumbnail = GENERATE_TOC_THUMBNAIL
InsertToc(oeb, sections, toc_thumbnails, insertHtmlToc,
insertThumbnail)
oIO = byteStringIO()
o = EPUBOutput() if booktype == "epub" else MOBIOutput()
o.convert(oeb, oIO, opts, main.log)
try:
ultima_log = DeliverLog.all().order('-time').get()
except:
ultima_log = sorted(DeliverLog.all(),
key=attrgetter('time'),
reverse=True)
ultima_log = ultima_log[0] if ultima_log else None
if ultima_log:
diff = datetime.datetime.utcnow() - ultima_log.datetime
if diff.days * 86400 + diff.seconds < 10:
time.sleep(8)
self.SendToKindle(username, to, book4meta.title + volumeTitle,
booktype, str(oIO.getvalue()), tz)
rs = "%s(%s).%s Sent!" % (book4meta.title, local_time(tz=tz),
booktype)
main.log.info(rs)
return rs
else:
self.deliverlog(username,
str(to),
book4meta.title + volumeTitle,
0,
status='nonews',
tz=tz)
rs = "No new feeds.1"
main.log.info(rs)
return rs
# Enter your code here. Read input from STDIN. Print output to STDOUT
from collections import OrderedDict
if __name__ == '__main__':
# Takes less time to execute
n = int(input())
dic = OrderedDict()
for _ in range(n):
s = str(input())
dic.setdefault(s, 0)
dic[s] += 1
print(len(dic))
print(*dic.values(), end=' ')
# Takes more time to execute
arr = []
for _ in range(n):
arr.append(str(input()))
arr_wo_dup = []
for x in arr:
if x not in arr_wo_dup:
arr_wo_dup.append(x)
print(len(arr_wo_dup))
occurance = []
def _op_to_template(op: BaseOp):
"""Generate template given an operator inherited from BaseOp."""
# NOTE in-place update to BaseOp
# replace all PipelineParams with template var strings
processed_op = _process_base_ops(op)
if isinstance(op, dsl.ContainerOp):
# default output artifacts
output_artifact_paths = OrderedDict(op.output_artifact_paths)
output_artifact_paths.setdefault('mlpipeline-ui-metadata', '/mlpipeline-ui-metadata.json')
output_artifact_paths.setdefault('mlpipeline-metrics', '/mlpipeline-metrics.json')
output_artifacts = [
_build_conventional_artifact(name, path)
for name, path in output_artifact_paths.items()
]
# workflow template
template = {
'name': processed_op.name,
'container': K8sHelper.convert_k8s_obj_to_json(
processed_op.container
)
}
elif isinstance(op, dsl.ResourceOp):
# no output artifacts
output_artifacts = []
# workflow template
processed_op.resource["manifest"] = yaml.dump(
K8sHelper.convert_k8s_obj_to_json(processed_op.k8s_resource),
default_flow_style=False
)
template = {
'name': processed_op.name,
'resource': K8sHelper.convert_k8s_obj_to_json(
processed_op.resource
)
}
# inputs
inputs = _inputs_to_json(processed_op.inputs)
if inputs:
template['inputs'] = inputs
# outputs
if isinstance(op, dsl.ContainerOp):
param_outputs = processed_op.file_outputs
elif isinstance(op, dsl.ResourceOp):
param_outputs = processed_op.attribute_outputs
template['outputs'] = _outputs_to_json(op, processed_op.outputs,
param_outputs, output_artifacts)
# node selector
if processed_op.node_selector:
template['nodeSelector'] = processed_op.node_selector
# metadata
if processed_op.pod_annotations or processed_op.pod_labels:
template['metadata'] = {}
if processed_op.pod_annotations:
template['metadata']['annotations'] = processed_op.pod_annotations
if processed_op.pod_labels:
template['metadata']['labels'] = processed_op.pod_labels
# retries
if processed_op.num_retries:
template['retryStrategy'] = {'limit': processed_op.num_retries}
# sidecars
if processed_op.sidecars:
template['sidecars'] = processed_op.sidecars
return template
def _intermediate_bubbles_2(cls, brackets):
"""Operates in-place.
This implements the second intermediate bubbles method, where all debates
in the same intermediate bubble have the same number of wins, but there
might be multiple intermediate bubbles between brackets.
"""
new = OrderedDict()
unfilled = OrderedDict()
intermediates = OrderedDict(
) # Values are lists of {"aff", "neg"} dicts
for points, pool in brackets.items():
to_delete_from_unfilled = []
# First, check for unfilled intermediate brackets
for unfilled_points, unfilled_pool in unfilled.items():
intermediates.setdefault(unfilled_points, list())
if unfilled_pool["aff"] and unfilled_pool["neg"]:
raise DrawFatalError(
"An unfilled pool unexpectedly had both affirmative and negative teams."
)
elif unfilled_pool["aff"]:
# In a new bracket, take the lesser of how many excess affirmative
# teams there are, and how many negative teams in the pool we have.
num_teams = min(len(unfilled_pool["aff"]),
len(pool["neg"]))
intermediates[unfilled_points].append({
"aff":
unfilled_pool["aff"][:num_teams],
"neg":
pool["neg"][:num_teams]
})
del unfilled_pool["aff"][:num_teams]
del pool["neg"][:num_teams]
elif unfilled_pool["neg"]:
# Take the top teams from affirmative pool as appropriate.
num_teams = min(len(unfilled_pool["neg"]),
len(pool["aff"]))
intermediates[unfilled_points].append({
"aff":
pool["aff"][:num_teams],
"neg":
unfilled_pool["neg"][:num_teams]
})
del pool["aff"][:num_teams]
del unfilled_pool["neg"][:num_teams]
# If we've exhausted the unfilled pool, add all these
# intermediate brackets to the main list of brackets and mark
# them for deletion from the unfilled buffer.
if not unfilled_pool["aff"] and not unfilled_pool["neg"]:
num_brackets = len(intermediates[unfilled_points])
for i, intermediate_pool in enumerate(
intermediates[unfilled_points], start=1):
intermediate_points = unfilled_points - i / (
num_brackets + 1)
new[intermediate_points] = intermediate_pool
to_delete_from_unfilled.append(unfilled_points)
# Delete the unfilled brackets tht were marked for deletion
for unfilled_points in to_delete_from_unfilled:
del unfilled[unfilled_points]
# Find lesser and greater of number of aff and neg teams.
nums_teams = list(map(len, list(pool.values())))
n = min(nums_teams)
m = max(nums_teams)
# Assign the main bracket
new[points] = {"aff": pool["aff"][:n], "neg": pool["neg"][:n]}
# Take note of the excess teams, if any
if m > n:
unfilled[points] = {
"aff": pool["aff"][n:],
"neg": pool["neg"][n:]
}
if unfilled:
raise DrawFatalError(
"There are still unfilled intermediate brackets!\n" +
repr(unfilled))
# Currently, the brackets are out of order, since e.g. 3.5 would have been
# inserted after 3 (or maybe even after 2). Let's change that:
new_sorted = sorted(list(new.items()),
key=lambda x: x[0],
reverse=True)
brackets.clear()
brackets.update(new_sorted)
def _descriptor_components(raw_contents,
validate,
extra_keywords=(),
non_ascii_fields=()):
"""
Initial breakup of the server descriptor contents to make parsing easier.
A descriptor contains a series of 'keyword lines' which are simply a keyword
followed by an optional value. Lines can also be followed by a signature
block.
To get a sub-listing with just certain keywords use extra_keywords. This can
be useful if we care about their relative ordering with respect to each
other. For instance, we care about the ordering of 'accept' and 'reject'
entries because this influences the resulting exit policy, but for everything
else in server descriptors the order does not matter.
:param str raw_contents: descriptor content provided by the relay
:param bool validate: checks the validity of the descriptor's content if
True, skips these checks otherwise
:param list extra_keywords: entity keywords to put into a separate listing
with ordering intact
:param list non_ascii_fields: fields containing non-ascii content
:returns:
**collections.OrderedDict** with the 'keyword => (value, pgp key) entries'
mappings. If a extra_keywords was provided then this instead provides a two
value tuple, the second being a list of those entries.
"""
if isinstance(raw_contents, bytes):
raw_contents = stem.util.str_tools._to_unicode(raw_contents)
entries = OrderedDict()
extra_entries = [] # entries with a keyword in extra_keywords
remaining_lines = raw_contents.split('\n')
while remaining_lines:
line = remaining_lines.pop(0)
# V2 network status documents explicitly can contain blank lines...
#
# "Implementations MAY insert blank lines for clarity between sections;
# these blank lines are ignored."
#
# ... and server descriptors end with an extra newline. But other documents
# don't say how blank lines should be handled so globally ignoring them.
if not line:
continue
# Some lines have an 'opt ' for backward compatibility. They should be
# ignored. This prefix is being removed in...
# https://trac.torproject.org/projects/tor/ticket/5124
if line.startswith('opt '):
line = line[4:]
line_match = KEYWORD_LINE.match(line)
if not line_match:
if not validate:
continue
raise ValueError('Line contains invalid characters: %s' % line)
keyword, value = line_match.groups()
if value is None:
value = ''
try:
block_attr = _get_pseudo_pgp_block(remaining_lines)
if block_attr:
block_type, block_contents = block_attr
else:
block_type, block_contents = None, None
except ValueError:
if not validate:
continue
raise
if validate and keyword not in non_ascii_fields:
try:
value.encode('ascii')
except UnicodeError:
replaced = ''.join([(char if char in string.printable else '?')
for char in value])
raise ValueError("'%s' line had non-ascii content: %s" %
(keyword, replaced))
if keyword in extra_keywords:
extra_entries.append('%s %s' % (keyword, value))
else:
entries.setdefault(keyword, []).append(
(value, block_type, block_contents))
if extra_keywords:
return entries, extra_entries
else:
return entries
import random
import sys
from collections import OrderedDict
txt = "the theremin is theirs, ok? yes, it is. this is a theremin." #Your sample text goes here.
order = 3 #You can change the order of the ngram here
ngrams = []
count = OrderedDict()
for i in range(0, len(txt) - order + 1):
gram = txt[i:i + order]
ngrams.append(gram)
if gram not in count:
count.setdefault(gram, [])
if gram in ngrams:
#count[gram][0]+=1
if i is not (len(txt) - order):
count.setdefault(gram, []).append(txt[i + order])
currentGram = txt[0:order]
for i in range(20):
if (len(count[currentGram])) is 0:
sys.exit(0)
possibilities = random.choice(count[currentGram])
newGram = currentGram + possibilities
print(newGram, end='')
currentGram = newGram[-order:]
class Transient(Module):
"""Structure to store transient data."""
_REFERENCES = [{SOURCE.BIBCODE: '2017arXiv171002145G'}]
_EX_REPS = {'rad': 'radio', 'xray': 'x-ray'}
_OBS_KEYS = [
'times', 'telescopes', 'systems', 'modes', 'instruments', 'bandsets',
'bands', 'frequencies', 'u_frequencies', 'zeropoints', 'measures'
]
def __init__(self, **kwargs):
"""Initialize module."""
super(Transient, self).__init__(**kwargs)
self._keys = kwargs.get('keys', '')
self._data_determined_parameters = []
def process(self, **kwargs):
"""Process module."""
return self._data
def set_data(self,
all_data,
req_key_values={},
subtract_minimum_keys=[],
smooth_times=-1,
extrapolate_time=0.0,
limit_fitting_mjds=False,
exclude_bands=[],
exclude_instruments=[],
exclude_systems=[],
exclude_sources=[],
exclude_kinds=[],
time_unit=None,
time_list=[],
band_list=[],
band_telescopes=[],
band_systems=[],
band_instruments=[],
band_modes=[],
band_bandsets=[]):
"""Set transient data."""
prt = self._printer
self._all_data = all_data
self._data = OrderedDict()
if not self._all_data:
return
name = list(self._all_data.keys())[0]
self._data['name'] = name
numeric_keys = set()
ex_kinds = [
self._EX_REPS.get(x.lower(), x.lower()) for x in exclude_kinds
]
# Construct some source dictionaries for exclusion rules
src_dict = OrderedDict()
sources = self._all_data[name].get('sources', [])
for src in sources:
if SOURCE.BIBCODE in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.BIBCODE]
if SOURCE.ARXIVID in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.ARXIVID]
if SOURCE.NAME in src:
src_dict[src[SOURCE.ALIAS]] = src[SOURCE.NAME]
for key in self._keys:
subkeys = self._keys[key]
req_subkeys = [
x for x in subkeys if not isinstance(subkeys, dict)
or 'required' in listify(subkeys[x])
]
num_subkeys = [
x for x in subkeys if 'numeric' in listify(subkeys[x])
]
boo_subkeys = [
x for x in subkeys if 'boolean' in listify(subkeys[x])
]
exc_subkeys = [
x for x in subkeys if 'exclude' in listify(subkeys[x])
]
if (key not in self._all_data[name]
and not self._model.is_parameter_fixed_by_user(key)):
if subkeys.get('value', None) == 'recommended':
self._unset_recommended_keys.add(key)
continue
subdata = self._all_data[name].get(key)
if subdata is None:
continue
# Only include data that contains all subkeys
for entry in subdata:
if any([x not in entry for x in req_subkeys]):
continue
if any([x in entry for x in exc_subkeys]):
continue
if any([
x in entry and
((isinstance(entry[x], list) and any([
not is_number(y) or np.isnan(float(y))
for y in entry[x]
])) or not isinstance(entry[x], list) and
(not is_number(entry[x]) or np.isnan(float(entry[x]))))
for x in num_subkeys
]):
continue
skip_key = False
if 'frequency' not in entry:
for qkey in req_key_values:
if qkey in entry and entry[qkey] != '':
if entry[qkey] not in req_key_values[qkey]:
skip_key = True
break
if key == 'photometry':
if ('fluxdensity' in entry and 'magnitude' not in entry
and 'countrate' not in entry):
self._kinds_needed.add('radio')
if ('radio' in ex_kinds or
(not len(ex_kinds) or 'none' not in ex_kinds) and
'radio' not in self._model._kinds_supported):
continue
if (('countrate' in entry or 'unabsorbedflux' in entry
or 'flux' in entry) and 'magnitude' not in entry
and 'fluxdensity' not in entry):
self._kinds_needed.add('x-ray')
if ('x-ray' in ex_kinds or
(not len(ex_kinds) or 'none' not in ex_kinds) and
'x-ray' not in self._model._kinds_supported):
continue
if 'magnitude' in entry:
# For now, magnitudes are not excludable.
self._kinds_needed |= set(
['infrared', 'optical', 'ultraviolet'])
skip_entry = False
for x in subkeys:
if limit_fitting_mjds is not False and x == 'time':
val = np.mean([
float(x) for x in listify(entry.get(x, None))
])
if (val < limit_fitting_mjds[0]
or val > limit_fitting_mjds[1]):
skip_entry = True
break
if exclude_bands is not False and x == 'band':
if (entry.get(x, '') in exclude_bands and
(not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)
and (not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_instruments is not False
and x == 'instrument'):
if (entry.get(x, '') in exclude_instruments and
(not exclude_bands
or entry.get('band', '') in exclude_bands)
and (not exclude_systems or entry.get(
'system', '') in exclude_systems)):
skip_entry = True
break
if (exclude_systems is not False and x == 'system'):
if (entry.get(x, '') in exclude_systems and
(not exclude_bands
or entry.get('band', '') in exclude_bands) and
(not exclude_instruments or entry.get(
'instrument', '') in exclude_instruments)):
skip_entry = True
break
if (exclude_sources is not False and x == 'source'):
val = entry.get(x, '')
if (any(
[x in exclude_sources for x in val.split(',')])
or any([
src_dict.get(x, '') in exclude_sources
for x in val.split(',')
])):
skip_entry = True
break
if skip_entry:
continue
if ((('magnitude' in entry) !=
('band' in entry)) or ((('fluxdensity' in entry) !=
('frequency' in entry)) and
('magnitude' not in entry))
or (('countrate' in entry) and
('magnitude' not in entry) and
('instrument' not in entry))):
continue
for x in subkeys:
falseval = (False if x in boo_subkeys else
None if x in num_subkeys else '')
if x == 'value':
if not skip_key:
self._data[key] = entry.get(x, falseval)
else:
plural = self._model.plural(x)
val = entry.get(x, falseval)
if x in num_subkeys:
val = None if val is None else np.mean(
[float(x) for x in listify(val)])
if not skip_key:
self._data.setdefault(plural, []).append(val)
if x in num_subkeys:
numeric_keys.add(plural)
else:
self._data.setdefault('unmatched_' + plural,
[]).append(val)
if 'times' not in self._data or not any([
x in self._data
for x in ['magnitudes', 'frequencies', 'countrates']
]):
prt.message('no_fittable_data', [name])
return False
for key in list(self._data.keys()):
if isinstance(self._data[key], list):
self._data[key] = np.array(self._data[key])
if key not in numeric_keys:
continue
num_values = [
x for x in self._data[key] if isinstance(x, float)
]
if len(num_values):
self._data['min_' + key] = min(num_values)
self._data['max_' + key] = max(num_values)
else:
if is_number(self._data[key]):
self._data[key] = float(self._data[key])
self._data_determined_parameters.append(key)
if any(x in self._data
for x in ['magnitudes', 'countrates', 'fluxdensities']):
# Add a list of tags for each observation to indicate what unit
# observation is provided in.
self._data['measures'] = [
((['magnitude'] if x else []) + (['countrate'] if y else []) +
(['fluxdensity'] if x else []))
for x, y, z in zip(*(self._data['magnitudes'],
self._data['countrates'],
self._data['fluxdensities']))
]
if 'times' in self._data and (smooth_times >= 0 or time_list):
# Build an observation array out of the real data first.
obs = list(
zip(*(self._data[x] for x in self._OBS_KEYS if x != 'times')))
# Append extra observations if requested.
if len(band_list):
b_teles = band_telescopes if len(band_telescopes) == len(
band_list) else (
[band_telescopes[0] for x in band_list]
if len(band_telescopes) else ['' for x in band_list])
b_systs = band_systems if len(band_systems) == len(
band_list) else ([band_systems[0]
for x in band_list] if len(band_systems)
else ['' for x in band_list])
b_modes = band_modes if len(band_modes) == len(
band_list) else ([band_modes[0]
for x in band_list] if len(band_modes)
else ['' for x in band_list])
b_insts = band_instruments if len(band_instruments) == len(
band_list) else (
[band_instruments[0] for x in band_list]
if len(band_instruments) else ['' for x in band_list])
b_bsets = band_bandsets if len(band_bandsets) == len(
band_list) else ([band_bandsets[0]
for x in band_list] if len(band_bandsets)
else ['' for x in band_list])
b_freqs = [None for x in band_list]
b_u_freqs = ['' for x in band_list]
b_zerops = [None for x in band_list]
b_measures = [[] for x in band_list]
obs.extend(
list(
zip(*(b_teles, b_systs, b_modes, b_insts, b_bsets,
band_list, b_freqs, b_u_freqs, b_zerops,
b_measures))))
# Prune extra observations if they are duplicitous to existing.
uniqueobs = []
for o in obs:
to = tuple(o)
if to not in uniqueobs:
uniqueobs.append(to)
# Preprend times to real observations list.
minet, maxet = (extrapolate_time, extrapolate_time) if isinstance(
extrapolate_time, (float, int)) else (
(tuple(extrapolate_time) if len(extrapolate_time) == 2 else
(extrapolate_time[0], extrapolate_time[0])))
mint, maxt = (min(self._data['times']) - minet,
max(self._data['times']) + maxet)
if time_unit is None:
alltimes = time_list + [x for x in self._data['times']]
elif time_unit == 'mjd':
alltimes = [x - min(self._data['times']) for x in time_list
] + [x for x in self._data['times']]
elif time_unit == 'phase':
if 'maxdate' not in self._data:
raise (prt.message('no_maxdate', name))
max_mjd = astrotime(self._data['maxdate'].replace('/',
'-')).mjd
alltimes = [x + max_mjd for x in time_list
] + [x for x in self._data['times']]
else:
raise ('Unknown `time_unit`.')
if smooth_times >= 0:
alltimes += list(np.linspace(mint, maxt, max(smooth_times, 2)))
alltimes = list(sorted(set(alltimes)))
# Create additional fake observations.
currobslist = list(zip(*(self._data[x] for x in self._OBS_KEYS)))
obslist = []
for ti, t in enumerate(alltimes):
new_per = np.round(100.0 * float(ti) / len(alltimes), 1)
prt.message('construct_obs_array', [new_per], inline=True)
for o in uniqueobs:
newobs = tuple([t] + list(o))
if newobs not in currobslist:
obslist.append(newobs)
obslist.sort()
# Save these fake observations under keys with `extra_` prefix.
if len(obslist):
for x, y in zip(self._OBS_KEYS, zip(*obslist)):
self._data['extra_' + x] = y
for qkey in subtract_minimum_keys:
if 'upperlimits' in self._data:
new_vals = np.array(self._data[qkey])[np.array(
self._data['upperlimits']) != True] # noqa E712
if len(new_vals):
self._data['min_' + qkey] = min(new_vals)
self._data['max_' + qkey] = max(new_vals)
minv = self._data['min_' + qkey]
self._data[qkey] = [x - minv for x in self._data[qkey]]
if 'extra_' + qkey in self._data:
self._data['extra_' + qkey] = [
x - minv for x in self._data['extra_' + qkey]
]
return True
def get_data_determined_parameters(self):
"""Return list of parameters determined by data."""
return self._data_determined_parameters
def send_request(self, request):
"""Send requests to other modules."""
if request == 'min_times':
return self._data['min_times']
return []
def _draw_text_as_text(self, gc, x, y, s, prop, angle, ismath, mtext=None):
writer = self.writer
color = rgb2hex(gc.get_rgb())
style = {}
if color != '#000000':
style['fill'] = color
if gc.get_alpha() != 1.0:
style['opacity'] = short_float_fmt(gc.get_alpha())
if not ismath:
font = self._get_font(prop)
font.set_text(s, 0.0, flags=LOAD_NO_HINTING)
fontsize = prop.get_size_in_points()
fontfamily = font.family_name
fontstyle = prop.get_style()
attrib = {}
# Must add "px" to workaround a Firefox bug
style['font-size'] = short_float_fmt(fontsize) + 'px'
style['font-family'] = six.text_type(fontfamily)
style['font-style'] = prop.get_style().lower()
style['font-weight'] = six.text_type(prop.get_weight()).lower()
attrib['style'] = generate_css(style)
if mtext and (angle == 0 or mtext.get_rotation_mode() == "anchor"):
# If text anchoring can be supported, get the original
# coordinates and add alignment information.
# Get anchor coordinates.
transform = mtext.get_transform()
ax, ay = transform.transform_point(mtext.get_position())
ay = self.height - ay
# Don't do vertical anchor alignment. Most applications do not
# support 'alignment-baseline' yet. Apply the vertical layout
# to the anchor point manually for now.
angle_rad = np.deg2rad(angle)
dir_vert = np.array([np.sin(angle_rad), np.cos(angle_rad)])
v_offset = np.dot(dir_vert, [(x - ax), (y - ay)])
ax = ax + v_offset * dir_vert[0]
ay = ay + v_offset * dir_vert[1]
ha_mpl_to_svg = {
'left': 'start',
'right': 'end',
'center': 'middle'
}
style['text-anchor'] = ha_mpl_to_svg[mtext.get_ha()]
attrib['x'] = short_float_fmt(ax)
attrib['y'] = short_float_fmt(ay)
attrib['style'] = generate_css(style)
attrib['transform'] = "rotate(%s, %s, %s)" % (short_float_fmt(
-angle), short_float_fmt(ax), short_float_fmt(ay))
writer.element('text', s, attrib=attrib)
else:
attrib['transform'] = generate_transform([
('translate', (x, y)), ('rotate', (-angle, ))
])
writer.element('text', s, attrib=attrib)
if rcParams['svg.fonttype'] == 'svgfont':
fontset = self._fonts.setdefault(font.fname, set())
for c in s:
fontset.add(ord(c))
else:
writer.comment(s)
width, height, descent, svg_elements, used_characters = \
self.mathtext_parser.parse(s, 72, prop)
svg_glyphs = svg_elements.svg_glyphs
svg_rects = svg_elements.svg_rects
attrib = {}
attrib['style'] = generate_css(style)
attrib['transform'] = generate_transform([('translate', (x, y)),
('rotate', (-angle, ))])
# Apply attributes to 'g', not 'text', because we likely
# have some rectangles as well with the same style and
# transformation
writer.start('g', attrib=attrib)
writer.start('text')
# Sort the characters by font, and output one tspan for
# each
spans = OrderedDict()
for font, fontsize, thetext, new_x, new_y, metrics in svg_glyphs:
style = generate_css({
'font-size': short_float_fmt(fontsize) + 'px',
'font-family': font.family_name,
'font-style': font.style_name.lower(),
'font-weight': font.style_name.lower()
})
if thetext == 32:
thetext = 0xa0 # non-breaking space
spans.setdefault(style, []).append((new_x, -new_y, thetext))
if rcParams['svg.fonttype'] == 'svgfont':
for font, fontsize, thetext, new_x, new_y, metrics in svg_glyphs:
fontset = self._fonts.setdefault(font.fname, set())
fontset.add(thetext)
for style, chars in six.iteritems(spans):
chars.sort()
same_y = True
if len(chars) > 1:
last_y = chars[0][1]
for i in xrange(1, len(chars)):
if chars[i][1] != last_y:
same_y = False
break
if same_y:
ys = six.text_type(chars[0][1])
else:
ys = ' '.join(six.text_type(c[1]) for c in chars)
attrib = {
'style': style,
'x': ' '.join(short_float_fmt(c[0]) for c in chars),
'y': ys
}
writer.element('tspan',
''.join(unichr(c[2]) for c in chars),
attrib=attrib)
writer.end('text')
if len(svg_rects):
for x, y, width, height in svg_rects:
writer.element('rect',
x=short_float_fmt(x),
y=short_float_fmt(-y + height),
width=short_float_fmt(width),
height=short_float_fmt(height))
writer.end('g')
class GeneData:
def __init__(self,
genes_hdf5_file,
worker_index=None,
workers=None,
read_1hot=True,
read_targets=True):
# open HDF5
self.genes_hdf5_in = h5py.File(genes_hdf5_file)
# simple stats
self.num_seqs, self.seq_length, self.seq_depth = self.genes_hdf5_in[
'seqs_1hot'].shape
self.pool_width = int(np.array(self.genes_hdf5_in['pool_width']))
#########################################
# gene sequences
self.gene_seqs = []
for si in range(len(self.genes_hdf5_in['seq_chrom'])):
gene_seq = GeneSeq(
self.genes_hdf5_in['seq_chrom'][si].decode('UTF-8'),
self.genes_hdf5_in['seq_start'][si],
self.genes_hdf5_in['seq_end'][si])
self.gene_seqs.append(gene_seq)
if read_1hot:
self.seqs_1hot = np.array(self.genes_hdf5_in['seqs_1hot'])
else:
self.seqs_1hot = None
#########################################
# TSS information
self.tss = []
for tss_i in range(len(self.genes_hdf5_in['tss_id'])):
# map to gene seq
seq_i = self.genes_hdf5_in['tss_seq'][tss_i]
tss_seq = self.gene_seqs[seq_i]
# read in TSS
tss = TSS(self.genes_hdf5_in['tss_id'][tss_i].decode('UTF-8'),
self.genes_hdf5_in['tss_gene'][tss_i].decode('UTF-8'),
self.genes_hdf5_in['tss_chrom'][tss_i].decode('UTF-8'),
self.genes_hdf5_in['tss_pos'][tss_i], tss_seq)
# tss_seq,
# self.genes_hdf5_in['tss_strand'][tss_i].decode('UTF-8'))
self.tss.append(tss)
# append to GeneSeq
tss_seq.append_tss(tss)
self.num_tss = len(self.tss)
#########################################
# gene information
self.gene_tss = OrderedDict()
self.gene_index = OrderedDict()
for tss_i in range(len(self.tss)):
gene_id = self.tss[tss_i].gene_id
self.gene_tss.setdefault(gene_id, []).append(tss_i)
if gene_id not in self.gene_index:
self.gene_index[gene_id] = len(self.gene_index)
self.num_genes = len(self.gene_tss)
#########################################
# determine genes split across sequences
gene_seqs = {}
for seq_i in range(self.num_seqs):
for tss in self.gene_seqs[seq_i].tss_list:
gene_seqs.setdefault(tss.gene_id, set()).add(seq_i)
self.multi_seq_genes = set()
for gene_id in gene_seqs:
if len(gene_seqs[gene_id]) > 1:
self.multi_seq_genes.add(gene_id)
#########################################
# target information
if 'tss_targets' in self.genes_hdf5_in and read_targets:
self.tss_targets = self.genes_hdf5_in['tss_targets']
self.target_labels = [
tl.decode('UTF-8')
for tl in self.genes_hdf5_in['target_labels']
]
if 'target_ids' in self.genes_hdf5_in: # TEMP
self.target_ids = [
tl.decode('UTF-8')
for tl in self.genes_hdf5_in['target_ids']
]
else:
self.target_ids = [''] * len(self.target_labels)
self.num_targets = len(self.target_labels)
else:
self.tss_targets = None
self.target_ids = None
self.target_labels = None
self.num_targets = None
def gene_ids(self):
""" Return a list of gene identifiers """
return list(self.gene_tss.keys())
def subset_genes(self, gene_ids):
''' Limit the sequences to a subset containing the given transcripts. '''
if type(gene_ids) != set:
gene_ids = set(gene_ids)
seq_mask = np.zeros(self.num_seqs, dtype='bool')
tss_mask = []
for si in range(self.num_seqs):
# determine TSSs matching given genes.
seq_tss_list = [
tss for tss in self.gene_seqs[si].tss_list
if tss.gene_id in gene_ids
]
seq_tss_mask = [
tss.gene_id in gene_ids for tss in self.gene_seqs[si].tss_list
]
tss_mask += seq_tss_mask
# filter TSSs to those matching given genes.
if len(seq_tss_list) > 0:
seq_mask[si] = True
self.gene_seqs[si].tss_list = seq_tss_list
# filter sequences for those with a match
self.gene_seqs = [
self.gene_seqs[si] for si in range(self.num_seqs) if seq_mask[si]
]
if self.seqs_1hot is not None:
self.seqs_1hot = self.seqs_1hot[seq_mask]
self.num_seqs = len(self.gene_seqs)
if self.tss_targets is not None:
tss_mask = np.array(tss_mask, dtype='bool')
self.tss_targets = self.tss_targets[tss_mask, :]
def tss_ids(self):
""" Return a list of TSS identifiers """
return [tss.identifier for tss in self.tss]
def worker(self, wi, worker_num):
""" Limit the sequences to one worker's share. """
worker_mask = np.array(
[si % worker_num == wi for si in range(self.num_seqs)])
# filter gene sequences
if self.seqs_1hot is not None:
self.seqs_1hot = self.seqs_1hot[worker_mask, :, :]
self.gene_seqs = [
self.gene_seqs[si] for si in range(self.num_seqs)
if worker_mask[si]
]
self.num_seqs = len(self.gene_seqs)
# hash remaining gene sequences
seq_hash = set()
for gene_seq in self.gene_seqs:
seq_hash.add((gene_seq.chrom, gene_seq.start))
# filter TSSs
worker_tss = []
for tss in self.tss:
tss_seq_key = (tss.gene_seq.chrom, tss.gene_seq.start)
if tss_seq_key in seq_hash:
worker_tss.append(tss)
self.tss = worker_tss
# tss_targets isn't relevant for SED
def __exit__(self):
# close HDF5
self.genes_hdf5_in.close()
def handle(self, *app_labels, **options):
format = options.get('format')
indent = options.get('indent')
using = options.get('database')
excludes = options.get('exclude')
output = options.get('output')
show_traceback = options.get('traceback')
use_natural_foreign_keys = options.get('use_natural_foreign_keys')
use_natural_primary_keys = options.get('use_natural_primary_keys')
use_base_manager = options.get('use_base_manager')
pks = options.get('primary_keys')
if pks:
primary_keys = pks.split(',')
else:
primary_keys = []
excluded_apps = set()
excluded_models = set()
for exclude in excludes:
if '.' in exclude:
try:
model = apps.get_model(exclude)
except LookupError:
raise CommandError('Unknown model in excludes: %s' %
exclude)
excluded_models.add(model)
else:
try:
app_config = apps.get_app_config(exclude)
except LookupError:
raise CommandError('Unknown app in excludes: %s' % exclude)
excluded_apps.add(app_config)
if len(app_labels) == 0:
if primary_keys:
raise CommandError(
"You can only use --pks option with one model")
app_list = OrderedDict((app_config, None)
for app_config in apps.get_app_configs()
if app_config.models_module is not None
and app_config not in excluded_apps)
else:
if len(app_labels) > 1 and primary_keys:
raise CommandError(
"You can only use --pks option with one model")
app_list = OrderedDict()
for label in app_labels:
try:
app_label, model_label = label.split('.')
try:
app_config = apps.get_app_config(app_label)
except LookupError:
raise CommandError("Unknown application: %s" %
app_label)
if app_config.models_module is None or app_config in excluded_apps:
continue
try:
model = app_config.get_model(model_label)
except LookupError:
raise CommandError("Unknown model: %s.%s" %
(app_label, model_label))
app_list_value = app_list.setdefault(app_config, [])
# We may have previously seen a "all-models" request for
# this app (no model qualifier was given). In this case
# there is no need adding specific models to the list.
if app_list_value is not None:
if model not in app_list_value:
app_list_value.append(model)
except ValueError:
if primary_keys:
raise CommandError(
"You can only use --pks option with one model")
# This is just an app - no model qualifier
app_label = label
try:
app_config = apps.get_app_config(app_label)
except LookupError:
raise CommandError("Unknown application: %s" %
app_label)
if app_config.models_module is None or app_config in excluded_apps:
continue
app_list[app_config] = None
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
try:
serializers.get_serializer(format)
except serializers.SerializerDoesNotExist:
pass
raise CommandError("Unknown serialization format: %s" % format)
def get_objects():
# Collate the objects to be serialized.
for model in serializers.sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate(
using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(
model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
for obj in queryset.iterator():
yield obj
try:
self.stdout.ending = None
stream = open(output, 'w') if output else None
try:
serializers.serialize(
format,
get_objects(),
indent=indent,
use_natural_foreign_keys=use_natural_foreign_keys,
use_natural_primary_keys=use_natural_primary_keys,
stream=stream or self.stdout)
finally:
if stream:
stream.close()
except Exception as e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
class RendererSVG(RendererBase):
FONT_SCALE = 100.0
fontd = maxdict(50)
def __init__(self, width, height, svgwriter, basename=None, image_dpi=72):
self.width = width
self.height = height
self.writer = XMLWriter(svgwriter)
self.image_dpi = image_dpi # the actual dpi we want to rasterize stuff with
self._groupd = {}
if not rcParams['svg.image_inline']:
assert basename is not None
self.basename = basename
self._imaged = {}
self._clipd = OrderedDict()
self._char_defs = {}
self._markers = {}
self._path_collection_id = 0
self._imaged = {}
self._hatchd = OrderedDict()
self._has_gouraud = False
self._n_gradients = 0
self._fonts = OrderedDict()
self.mathtext_parser = MathTextParser('SVG')
RendererBase.__init__(self)
self._glyph_map = dict()
str_height = short_float_fmt(height)
str_width = short_float_fmt(width)
svgwriter.write(svgProlog)
self._start_id = self.writer.start(
'svg',
width='%spt' % str_width,
height='%spt' % str_height,
viewBox='0 0 %s %s' % (str_width, str_height),
xmlns="http://www.w3.org/2000/svg",
version="1.1",
attrib={'xmlns:xlink': "http://www.w3.org/1999/xlink"})
self._write_default_style()
def finalize(self):
self._write_clips()
self._write_hatches()
self._write_svgfonts()
self.writer.close(self._start_id)
self.writer.flush()
def _write_default_style(self):
writer = self.writer
default_style = generate_css({
'stroke-linejoin': 'round',
'stroke-linecap': 'butt'
})
writer.start('defs')
writer.start('style', type='text/css')
writer.data('*{%s}\n' % default_style)
writer.end('style')
writer.end('defs')
def _make_id(self, type, content):
content = str(content)
if rcParams['svg.hashsalt'] is None:
salt = str(uuid.uuid4())
else:
salt = rcParams['svg.hashsalt']
if six.PY3:
content = content.encode('utf8')
salt = salt.encode('utf8')
m = hashlib.md5()
m.update(salt)
m.update(content)
return '%s%s' % (type, m.hexdigest()[:10])
def _make_flip_transform(self, transform):
return (transform +
Affine2D().scale(1.0, -1.0).translate(0.0, self.height))
def _get_font(self, prop):
fname = findfont(prop)
font = get_font(fname)
font.clear()
size = prop.get_size_in_points()
font.set_size(size, 72.0)
return font
def _get_hatch(self, gc, rgbFace):
"""
Create a new hatch pattern
"""
if rgbFace is not None:
rgbFace = tuple(rgbFace)
edge = gc.get_hatch_color()
if edge is not None:
edge = tuple(edge)
dictkey = (gc.get_hatch(), rgbFace, edge)
oid = self._hatchd.get(dictkey)
if oid is None:
oid = self._make_id('h', dictkey)
self._hatchd[dictkey] = ((gc.get_hatch_path(), rgbFace, edge), oid)
else:
_, oid = oid
return oid
def _write_hatches(self):
if not len(self._hatchd):
return
HATCH_SIZE = 72
writer = self.writer
writer.start('defs')
for ((path, face, stroke), oid) in six.itervalues(self._hatchd):
writer.start('pattern',
id=oid,
patternUnits="userSpaceOnUse",
x="0",
y="0",
width=six.text_type(HATCH_SIZE),
height=six.text_type(HATCH_SIZE))
path_data = self._convert_path(path,
Affine2D().scale(HATCH_SIZE).scale(
1.0,
-1.0).translate(0, HATCH_SIZE),
simplify=False)
if face is None:
fill = 'none'
else:
fill = rgb2hex(face)
writer.element('rect',
x="0",
y="0",
width=six.text_type(HATCH_SIZE + 1),
height=six.text_type(HATCH_SIZE + 1),
fill=fill)
writer.element('path',
d=path_data,
style=generate_css({
'fill':
rgb2hex(stroke),
'stroke':
rgb2hex(stroke),
'stroke-width':
six.text_type(rcParams['hatch.linewidth']),
'stroke-linecap':
'butt',
'stroke-linejoin':
'miter'
}))
writer.end('pattern')
writer.end('defs')
def _get_style_dict(self, gc, rgbFace):
"""
return the style string. style is generated from the
GraphicsContext and rgbFace
"""
attrib = {}
forced_alpha = gc.get_forced_alpha()
if gc.get_hatch() is not None:
attrib['fill'] = "url(#%s)" % self._get_hatch(gc, rgbFace)
if rgbFace is not None and len(
rgbFace) == 4 and rgbFace[3] != 1.0 and not forced_alpha:
attrib['fill-opacity'] = short_float_fmt(rgbFace[3])
else:
if rgbFace is None:
attrib['fill'] = 'none'
else:
if tuple(rgbFace[:3]) != (0, 0, 0):
attrib['fill'] = rgb2hex(rgbFace)
if len(rgbFace
) == 4 and rgbFace[3] != 1.0 and not forced_alpha:
attrib['fill-opacity'] = short_float_fmt(rgbFace[3])
if forced_alpha and gc.get_alpha() != 1.0:
attrib['opacity'] = short_float_fmt(gc.get_alpha())
offset, seq = gc.get_dashes()
if seq is not None:
attrib['stroke-dasharray'] = ','.join(
[short_float_fmt(val) for val in seq])
attrib['stroke-dashoffset'] = short_float_fmt(float(offset))
linewidth = gc.get_linewidth()
if linewidth:
rgb = gc.get_rgb()
attrib['stroke'] = rgb2hex(rgb)
if not forced_alpha and rgb[3] != 1.0:
attrib['stroke-opacity'] = short_float_fmt(rgb[3])
if linewidth != 1.0:
attrib['stroke-width'] = short_float_fmt(linewidth)
if gc.get_joinstyle() != 'round':
attrib['stroke-linejoin'] = gc.get_joinstyle()
if gc.get_capstyle() != 'butt':
attrib['stroke-linecap'] = _capstyle_d[gc.get_capstyle()]
return attrib
def _get_style(self, gc, rgbFace):
return generate_css(self._get_style_dict(gc, rgbFace))
def _get_clip(self, gc):
cliprect = gc.get_clip_rectangle()
clippath, clippath_trans = gc.get_clip_path()
if clippath is not None:
clippath_trans = self._make_flip_transform(clippath_trans)
dictkey = (id(clippath), str(clippath_trans))
elif cliprect is not None:
x, y, w, h = cliprect.bounds
y = self.height - (y + h)
dictkey = (x, y, w, h)
else:
return None
clip = self._clipd.get(dictkey)
if clip is None:
oid = self._make_id('p', dictkey)
if clippath is not None:
self._clipd[dictkey] = ((clippath, clippath_trans), oid)
else:
self._clipd[dictkey] = (dictkey, oid)
else:
clip, oid = clip
return oid
def _write_clips(self):
if not len(self._clipd):
return
writer = self.writer
writer.start('defs')
for clip, oid in six.itervalues(self._clipd):
writer.start('clipPath', id=oid)
if len(clip) == 2:
clippath, clippath_trans = clip
path_data = self._convert_path(clippath,
clippath_trans,
simplify=False)
writer.element('path', d=path_data)
else:
x, y, w, h = clip
writer.element('rect',
x=short_float_fmt(x),
y=short_float_fmt(y),
width=short_float_fmt(w),
height=short_float_fmt(h))
writer.end('clipPath')
writer.end('defs')
def _write_svgfonts(self):
if not rcParams['svg.fonttype'] == 'svgfont':
return
writer = self.writer
writer.start('defs')
for font_fname, chars in six.iteritems(self._fonts):
font = get_font(font_fname)
font.set_size(72, 72)
sfnt = font.get_sfnt()
writer.start('font', id=sfnt[1, 0, 0, 4].decode("mac_roman"))
writer.element(
'font-face',
attrib={
'font-family': font.family_name,
'font-style': font.style_name.lower(),
'units-per-em': '72',
'bbox':
' '.join(short_float_fmt(x / 64.0) for x in font.bbox)
})
for char in chars:
glyph = font.load_char(char, flags=LOAD_NO_HINTING)
verts, codes = font.get_path()
path = Path(verts, codes)
path_data = self._convert_path(path)
# name = font.get_glyph_name(char)
writer.element(
'glyph',
d=path_data,
attrib={
# 'glyph-name': name,
'unicode':
unichr(char),
'horiz-adv-x':
short_float_fmt(glyph.linearHoriAdvance / 65536.0)
})
writer.end('font')
writer.end('defs')
def open_group(self, s, gid=None):
"""
Open a grouping element with label *s*. If *gid* is given, use
*gid* as the id of the group.
"""
if gid:
self.writer.start('g', id=gid)
else:
self._groupd[s] = self._groupd.get(s, 0) + 1
self.writer.start('g', id="%s_%d" % (s, self._groupd[s]))
def close_group(self, s):
self.writer.end('g')
def option_image_nocomposite(self):
"""
return whether to generate a composite image from multiple images on
a set of axes
"""
return not rcParams['image.composite_image']
def _convert_path(self,
path,
transform=None,
clip=None,
simplify=None,
sketch=None):
if clip:
clip = (0.0, 0.0, self.width, self.height)
else:
clip = None
return _path.convert_to_string(path, transform, clip, simplify, sketch,
6, [b'M', b'L', b'Q', b'C', b'z'],
False).decode('ascii')
def draw_path(self, gc, path, transform, rgbFace=None):
trans_and_flip = self._make_flip_transform(transform)
clip = (rgbFace is None and gc.get_hatch_path() is None)
simplify = path.should_simplify and clip
path_data = self._convert_path(path,
trans_and_flip,
clip=clip,
simplify=simplify,
sketch=gc.get_sketch_params())
attrib = {}
attrib['style'] = self._get_style(gc, rgbFace)
clipid = self._get_clip(gc)
if clipid is not None:
attrib['clip-path'] = 'url(#%s)' % clipid
if gc.get_url() is not None:
self.writer.start('a', {'xlink:href': gc.get_url()})
self.writer.element('path', d=path_data, attrib=attrib)
if gc.get_url() is not None:
self.writer.end('a')
def draw_markers(self,
gc,
marker_path,
marker_trans,
path,
trans,
rgbFace=None):
if not len(path.vertices):
return
writer = self.writer
path_data = self._convert_path(marker_path,
marker_trans +
Affine2D().scale(1.0, -1.0),
simplify=False)
style = self._get_style_dict(gc, rgbFace)
dictkey = (path_data, generate_css(style))
oid = self._markers.get(dictkey)
style = generate_css(
{k: v
for k, v in six.iteritems(style) if k.startswith('stroke')})
if oid is None:
oid = self._make_id('m', dictkey)
writer.start('defs')
writer.element('path', id=oid, d=path_data, style=style)
writer.end('defs')
self._markers[dictkey] = oid
attrib = {}
clipid = self._get_clip(gc)
if clipid is not None:
attrib['clip-path'] = 'url(#%s)' % clipid
writer.start('g', attrib=attrib)
trans_and_flip = self._make_flip_transform(trans)
attrib = {'xlink:href': '#%s' % oid}
clip = (0, 0, self.width * 72, self.height * 72)
for vertices, code in path.iter_segments(trans_and_flip,
clip=clip,
simplify=False):
if len(vertices):
x, y = vertices[-2:]
attrib['x'] = short_float_fmt(x)
attrib['y'] = short_float_fmt(y)
attrib['style'] = self._get_style(gc, rgbFace)
writer.element('use', attrib=attrib)
writer.end('g')
def draw_path_collection(self, gc, master_transform, paths, all_transforms,
offsets, offsetTrans, facecolors, edgecolors,
linewidths, linestyles, antialiaseds, urls,
offset_position):
# Is the optimization worth it? Rough calculation:
# cost of emitting a path in-line is
# (len_path + 5) * uses_per_path
# cost of definition+use is
# (len_path + 3) + 9 * uses_per_path
len_path = len(paths[0].vertices) if len(paths) > 0 else 0
uses_per_path = self._iter_collection_uses_per_path(
paths, all_transforms, offsets, facecolors, edgecolors)
should_do_optimization = \
len_path + 9 * uses_per_path + 3 < (len_path + 5) * uses_per_path
if not should_do_optimization:
return RendererBase.draw_path_collection(
self, gc, master_transform, paths, all_transforms, offsets,
offsetTrans, facecolors, edgecolors, linewidths, linestyles,
antialiaseds, urls, offset_position)
writer = self.writer
path_codes = []
writer.start('defs')
for i, (path, transform) in enumerate(
self._iter_collection_raw_paths(master_transform, paths,
all_transforms)):
transform = Affine2D(transform.get_matrix()).scale(1.0, -1.0)
d = self._convert_path(path, transform, simplify=False)
oid = 'C%x_%x_%s' % (self._path_collection_id, i,
self._make_id('', d))
writer.element('path', id=oid, d=d)
path_codes.append(oid)
writer.end('defs')
for xo, yo, path_id, gc0, rgbFace in self._iter_collection(
gc, master_transform, all_transforms, path_codes, offsets,
offsetTrans, facecolors, edgecolors, linewidths, linestyles,
antialiaseds, urls, offset_position):
clipid = self._get_clip(gc0)
url = gc0.get_url()
if url is not None:
writer.start('a', attrib={'xlink:href': url})
if clipid is not None:
writer.start('g', attrib={'clip-path': 'url(#%s)' % clipid})
attrib = {
'xlink:href': '#%s' % path_id,
'x': short_float_fmt(xo),
'y': short_float_fmt(self.height - yo),
'style': self._get_style(gc0, rgbFace)
}
writer.element('use', attrib=attrib)
if clipid is not None:
writer.end('g')
if url is not None:
writer.end('a')
self._path_collection_id += 1
def draw_gouraud_triangle(self, gc, points, colors, trans):
# This uses a method described here:
#
# http://www.svgopen.org/2005/papers/Converting3DFaceToSVG/index.html
#
# that uses three overlapping linear gradients to simulate a
# Gouraud triangle. Each gradient goes from fully opaque in
# one corner to fully transparent along the opposite edge.
# The line between the stop points is perpendicular to the
# opposite edge. Underlying these three gradients is a solid
# triangle whose color is the average of all three points.
writer = self.writer
if not self._has_gouraud:
self._has_gouraud = True
writer.start('filter', id='colorAdd')
writer.element('feComposite',
attrib={'in': 'SourceGraphic'},
in2='BackgroundImage',
operator='arithmetic',
k2="1",
k3="1")
writer.end('filter')
avg_color = np.sum(colors[:, :], axis=0) / 3.0
# Just skip fully-transparent triangles
if avg_color[-1] == 0.0:
return
trans_and_flip = self._make_flip_transform(trans)
tpoints = trans_and_flip.transform(points)
writer.start('defs')
for i in range(3):
x1, y1 = tpoints[i]
x2, y2 = tpoints[(i + 1) % 3]
x3, y3 = tpoints[(i + 2) % 3]
c = colors[i][:]
if x2 == x3:
xb = x2
yb = y1
elif y2 == y3:
xb = x1
yb = y2
else:
m1 = (y2 - y3) / (x2 - x3)
b1 = y2 - (m1 * x2)
m2 = -(1.0 / m1)
b2 = y1 - (m2 * x1)
xb = (-b1 + b2) / (m1 - m2)
yb = m2 * xb + b2
writer.start('linearGradient',
id="GR%x_%d" % (self._n_gradients, i),
x1=short_float_fmt(x1),
y1=short_float_fmt(y1),
x2=short_float_fmt(xb),
y2=short_float_fmt(yb))
writer.element('stop',
offset='0',
style=generate_css({
'stop-color':
rgb2hex(c),
'stop-opacity':
short_float_fmt(c[-1])
}))
writer.element('stop',
offset='1',
style=generate_css({
'stop-color': rgb2hex(c),
'stop-opacity': "0"
}))
writer.end('linearGradient')
writer.element('polygon',
id='GT%x' % self._n_gradients,
points=" ".join([
short_float_fmt(x) for x in (x1, y1, x2, y2, x3, y3)
]))
writer.end('defs')
avg_color = np.sum(colors[:, :], axis=0) / 3.0
href = '#GT%x' % self._n_gradients
writer.element('use',
attrib={
'xlink:href': href,
'fill': rgb2hex(avg_color),
'fill-opacity': short_float_fmt(avg_color[-1])
})
for i in range(3):
writer.element('use',
attrib={
'xlink:href': href,
'fill':
'url(#GR%x_%d)' % (self._n_gradients, i),
'fill-opacity': '1',
'filter': 'url(#colorAdd)'
})
self._n_gradients += 1
def draw_gouraud_triangles(self, gc, triangles_array, colors_array,
transform):
attrib = {}
clipid = self._get_clip(gc)
if clipid is not None:
attrib['clip-path'] = 'url(#%s)' % clipid
self.writer.start('g', attrib=attrib)
transform = transform.frozen()
for tri, col in zip(triangles_array, colors_array):
self.draw_gouraud_triangle(gc, tri, col, transform)
self.writer.end('g')
def option_scale_image(self):
return True
def get_image_magnification(self):
return self.image_dpi / 72.0
def draw_image(self, gc, x, y, im, transform=None):
h, w = im.shape[:2]
if w == 0 or h == 0:
return
attrib = {}
clipid = self._get_clip(gc)
if clipid is not None:
# Can't apply clip-path directly to the image because the
# image has a transformation, which would also be applied
# to the clip-path
self.writer.start('g', attrib={'clip-path': 'url(#%s)' % clipid})
oid = gc.get_gid()
url = gc.get_url()
if url is not None:
self.writer.start('a', attrib={'xlink:href': url})
if rcParams['svg.image_inline']:
bytesio = io.BytesIO()
_png.write_png(im, bytesio)
oid = oid or self._make_id('image', bytesio.getvalue())
attrib['xlink:href'] = (
"data:image/png;base64,\n" +
base64.b64encode(bytesio.getvalue()).decode('ascii'))
else:
self._imaged[self.basename] = self._imaged.get(self.basename,
0) + 1
filename = '%s.image%d.png' % (self.basename,
self._imaged[self.basename])
_log.info('Writing image file for inclusion: %s', filename)
_png.write_png(im, filename)
oid = oid or 'Im_' + self._make_id('image', filename)
attrib['xlink:href'] = filename
attrib['id'] = oid
if transform is None:
w = 72.0 * w / self.image_dpi
h = 72.0 * h / self.image_dpi
self.writer.element('image',
transform=generate_transform([
('scale', (1, -1)), ('translate', (0, -h))
]),
x=short_float_fmt(x),
y=short_float_fmt(-(self.height - y - h)),
width=short_float_fmt(w),
height=short_float_fmt(h),
attrib=attrib)
else:
alpha = gc.get_alpha()
if alpha != 1.0:
attrib['opacity'] = short_float_fmt(alpha)
flipped = (Affine2D().scale(1.0 / w, 1.0 / h) + transform +
Affine2D().translate(x, y).scale(1.0, -1.0).translate(
0.0, self.height))
attrib['transform'] = generate_transform([('matrix',
flipped.frozen())])
self.writer.element('image',
width=short_float_fmt(w),
height=short_float_fmt(h),
attrib=attrib)
if url is not None:
self.writer.end('a')
if clipid is not None:
self.writer.end('g')
def _adjust_char_id(self, char_id):
return char_id.replace("%20", "_")
def _draw_text_as_path(self, gc, x, y, s, prop, angle, ismath, mtext=None):
"""
draw the text by converting them to paths using textpath module.
Parameters
----------
prop : `matplotlib.font_manager.FontProperties`
font property
s : str
text to be converted
usetex : bool
If True, use matplotlib usetex mode.
ismath : bool
If True, use mathtext parser. If "TeX", use *usetex* mode.
"""
writer = self.writer
writer.comment(s)
glyph_map = self._glyph_map
text2path = self._text2path
color = rgb2hex(gc.get_rgb())
fontsize = prop.get_size_in_points()
style = {}
if color != '#000000':
style['fill'] = color
if gc.get_alpha() != 1.0:
style['opacity'] = short_float_fmt(gc.get_alpha())
if not ismath:
font = text2path._get_font(prop)
_glyphs = text2path.get_glyphs_with_font(
font, s, glyph_map=glyph_map, return_new_glyphs_only=True)
glyph_info, glyph_map_new, rects = _glyphs
if glyph_map_new:
writer.start('defs')
for char_id, glyph_path in six.iteritems(glyph_map_new):
path = Path(*glyph_path)
path_data = self._convert_path(path, simplify=False)
writer.element('path', id=char_id, d=path_data)
writer.end('defs')
glyph_map.update(glyph_map_new)
attrib = {}
attrib['style'] = generate_css(style)
font_scale = fontsize / text2path.FONT_SCALE
attrib['transform'] = generate_transform([
('translate', (x, y)), ('rotate', (-angle, )),
('scale', (font_scale, -font_scale))
])
writer.start('g', attrib=attrib)
for glyph_id, xposition, yposition, scale in glyph_info:
attrib = {'xlink:href': '#%s' % glyph_id}
if xposition != 0.0:
attrib['x'] = short_float_fmt(xposition)
if yposition != 0.0:
attrib['y'] = short_float_fmt(yposition)
writer.element('use', attrib=attrib)
writer.end('g')
else:
if ismath == "TeX":
_glyphs = text2path.get_glyphs_tex(prop,
s,
glyph_map=glyph_map,
return_new_glyphs_only=True)
else:
_glyphs = text2path.get_glyphs_mathtext(
prop, s, glyph_map=glyph_map, return_new_glyphs_only=True)
glyph_info, glyph_map_new, rects = _glyphs
# we store the character glyphs w/o flipping. Instead, the
# coordinate will be flipped when this characters are
# used.
if glyph_map_new:
writer.start('defs')
for char_id, glyph_path in six.iteritems(glyph_map_new):
char_id = self._adjust_char_id(char_id)
# Some characters are blank
if not len(glyph_path[0]):
path_data = ""
else:
path = Path(*glyph_path)
path_data = self._convert_path(path, simplify=False)
writer.element('path', id=char_id, d=path_data)
writer.end('defs')
glyph_map.update(glyph_map_new)
attrib = {}
font_scale = fontsize / text2path.FONT_SCALE
attrib['style'] = generate_css(style)
attrib['transform'] = generate_transform([
('translate', (x, y)), ('rotate', (-angle, )),
('scale', (font_scale, -font_scale))
])
writer.start('g', attrib=attrib)
for char_id, xposition, yposition, scale in glyph_info:
char_id = self._adjust_char_id(char_id)
writer.element('use',
transform=generate_transform([
('translate', (xposition, yposition)),
('scale', (scale, )),
]),
attrib={'xlink:href': '#%s' % char_id})
for verts, codes in rects:
path = Path(verts, codes)
path_data = self._convert_path(path, simplify=False)
writer.element('path', d=path_data)
writer.end('g')
def _draw_text_as_text(self, gc, x, y, s, prop, angle, ismath, mtext=None):
writer = self.writer
color = rgb2hex(gc.get_rgb())
style = {}
if color != '#000000':
style['fill'] = color
if gc.get_alpha() != 1.0:
style['opacity'] = short_float_fmt(gc.get_alpha())
if not ismath:
font = self._get_font(prop)
font.set_text(s, 0.0, flags=LOAD_NO_HINTING)
fontsize = prop.get_size_in_points()
fontfamily = font.family_name
fontstyle = prop.get_style()
attrib = {}
# Must add "px" to workaround a Firefox bug
style['font-size'] = short_float_fmt(fontsize) + 'px'
style['font-family'] = six.text_type(fontfamily)
style['font-style'] = prop.get_style().lower()
style['font-weight'] = six.text_type(prop.get_weight()).lower()
attrib['style'] = generate_css(style)
if mtext and (angle == 0 or mtext.get_rotation_mode() == "anchor"):
# If text anchoring can be supported, get the original
# coordinates and add alignment information.
# Get anchor coordinates.
transform = mtext.get_transform()
ax, ay = transform.transform_point(mtext.get_position())
ay = self.height - ay
# Don't do vertical anchor alignment. Most applications do not
# support 'alignment-baseline' yet. Apply the vertical layout
# to the anchor point manually for now.
angle_rad = np.deg2rad(angle)
dir_vert = np.array([np.sin(angle_rad), np.cos(angle_rad)])
v_offset = np.dot(dir_vert, [(x - ax), (y - ay)])
ax = ax + v_offset * dir_vert[0]
ay = ay + v_offset * dir_vert[1]
ha_mpl_to_svg = {
'left': 'start',
'right': 'end',
'center': 'middle'
}
style['text-anchor'] = ha_mpl_to_svg[mtext.get_ha()]
attrib['x'] = short_float_fmt(ax)
attrib['y'] = short_float_fmt(ay)
attrib['style'] = generate_css(style)
attrib['transform'] = "rotate(%s, %s, %s)" % (short_float_fmt(
-angle), short_float_fmt(ax), short_float_fmt(ay))
writer.element('text', s, attrib=attrib)
else:
attrib['transform'] = generate_transform([
('translate', (x, y)), ('rotate', (-angle, ))
])
writer.element('text', s, attrib=attrib)
if rcParams['svg.fonttype'] == 'svgfont':
fontset = self._fonts.setdefault(font.fname, set())
for c in s:
fontset.add(ord(c))
else:
writer.comment(s)
width, height, descent, svg_elements, used_characters = \
self.mathtext_parser.parse(s, 72, prop)
svg_glyphs = svg_elements.svg_glyphs
svg_rects = svg_elements.svg_rects
attrib = {}
attrib['style'] = generate_css(style)
attrib['transform'] = generate_transform([('translate', (x, y)),
('rotate', (-angle, ))])
# Apply attributes to 'g', not 'text', because we likely
# have some rectangles as well with the same style and
# transformation
writer.start('g', attrib=attrib)
writer.start('text')
# Sort the characters by font, and output one tspan for
# each
spans = OrderedDict()
for font, fontsize, thetext, new_x, new_y, metrics in svg_glyphs:
style = generate_css({
'font-size': short_float_fmt(fontsize) + 'px',
'font-family': font.family_name,
'font-style': font.style_name.lower(),
'font-weight': font.style_name.lower()
})
if thetext == 32:
thetext = 0xa0 # non-breaking space
spans.setdefault(style, []).append((new_x, -new_y, thetext))
if rcParams['svg.fonttype'] == 'svgfont':
for font, fontsize, thetext, new_x, new_y, metrics in svg_glyphs:
fontset = self._fonts.setdefault(font.fname, set())
fontset.add(thetext)
for style, chars in six.iteritems(spans):
chars.sort()
same_y = True
if len(chars) > 1:
last_y = chars[0][1]
for i in xrange(1, len(chars)):
if chars[i][1] != last_y:
same_y = False
break
if same_y:
ys = six.text_type(chars[0][1])
else:
ys = ' '.join(six.text_type(c[1]) for c in chars)
attrib = {
'style': style,
'x': ' '.join(short_float_fmt(c[0]) for c in chars),
'y': ys
}
writer.element('tspan',
''.join(unichr(c[2]) for c in chars),
attrib=attrib)
writer.end('text')
if len(svg_rects):
for x, y, width, height in svg_rects:
writer.element('rect',
x=short_float_fmt(x),
y=short_float_fmt(-y + height),
width=short_float_fmt(width),
height=short_float_fmt(height))
writer.end('g')
def draw_tex(self, gc, x, y, s, prop, angle, ismath='TeX!', mtext=None):
self._draw_text_as_path(gc, x, y, s, prop, angle, ismath="TeX")
def draw_text(self, gc, x, y, s, prop, angle, ismath=False, mtext=None):
clipid = self._get_clip(gc)
if clipid is not None:
# Cannot apply clip-path directly to the text, because
# is has a transformation
self.writer.start('g', attrib={'clip-path': 'url(#%s)' % clipid})
if gc.get_url() is not None:
self.writer.start('a', {'xlink:href': gc.get_url()})
if rcParams['svg.fonttype'] == 'path':
self._draw_text_as_path(gc, x, y, s, prop, angle, ismath, mtext)
else:
self._draw_text_as_text(gc, x, y, s, prop, angle, ismath, mtext)
if gc.get_url() is not None:
self.writer.end('a')
if clipid is not None:
self.writer.end('g')
def flipy(self):
return True
def get_canvas_width_height(self):
return self.width, self.height
def get_text_width_height_descent(self, s, prop, ismath):
return self._text2path.get_text_width_height_descent(s, prop, ismath)
StdBlckLst.extend(get_standard_funtions(csv_file_to_table(open(StdFuncsCSV))))
# Dictionary to speedup block type fetching by name
StdBlckDct = OrderedDict()
for section in StdBlckLst:
for desc in section["list"]:
words = desc["comment"].split('"')
if len(words) > 1:
desc["comment"] = words[1]
desc["usage"] = ("\n (%s) => (%s)" % (", ".join([
"%s:%s" % (input[1], input[0]) for input in desc["inputs"]
]), ", ".join(
["%s:%s" % (output[1], output[0]) for output in desc["outputs"]])))
BlkLst = StdBlckDct.setdefault(desc["name"], [])
BlkLst.append((section["name"], desc))
# -------------------------------------------------------------------------------
# Languages Keywords
# -------------------------------------------------------------------------------
# Keywords for Pou Declaration
POU_BLOCK_START_KEYWORDS = ["FUNCTION", "FUNCTION_BLOCK", "PROGRAM"]
POU_BLOCK_END_KEYWORDS = ["END_FUNCTION", "END_FUNCTION_BLOCK", "END_PROGRAM"]
POU_KEYWORDS = ["EN", "ENO", "F_EDGE", "R_EDGE"
] + POU_BLOCK_START_KEYWORDS + POU_BLOCK_END_KEYWORDS
for category in StdBlckLst:
for block in category["list"]:
if block["name"] not in POU_KEYWORDS:
POU_KEYWORDS.append(block["name"])
parser = argparse.ArgumentParser()
parser.add_argument("--key", help="input key")
parser.add_argument("--val", help="input value")
args = parser.parse_args()
key = args.key
value = args.val
storage_path = os.path.join(tempfile.gettempdir(), 'storage.data')
data = None
if os.path.isfile(storage_path):
with open(storage_path, "r+") as f:
data = json.load(f)
elif value is None:
print(None)
exit()
else:
with open(storage_path, "w+") as f:
data = OrderedDict()
data[key] = value
json.dump(data, f)
#writing...
with open(storage_path, "w+") as f:
if value is not None:
data.setdefault(key, value)
if data[key] != value:
data[key] = data[key], value
else:
print(data.get(key, None))
json.dump(data, f)
def test_awxcollector(setup_environment):
'''
Efforts to improve the performance of cleanup_jobs involved
sub-classing the django Collector class. This unit test will
check for parity between the django Collector and the modified
AWXCollector class. AWXCollector is used in cleanup_jobs to
bulk-delete old jobs from the database.
Specifically, Collector has four dictionaries to check:
.dependencies, .data, .fast_deletes, and .field_updates
These tests will convert each dictionary from AWXCollector
(after running .collect on jobs), from querysets to sets of
objects. The final result should be a dictionary that is
equivalent to django's Collector.
'''
(old_jobs, new_jobs, days_str) = setup_environment
collector = Collector('default')
collector.collect(old_jobs)
awx_col = AWXCollector('default')
# awx_col accepts a queryset as input
awx_col.collect(Job.objects.filter(pk__in=[obj.pk for obj in old_jobs]))
# check that dependencies are the same
assert awx_col.dependencies == collector.dependencies
# check that objects to delete are the same
awx_del_dict = OrderedDict()
for model, instances in awx_col.data.items():
awx_del_dict.setdefault(model, set())
for inst in instances:
# .update() will put each object in a queryset into the set
awx_del_dict[model].update(inst)
assert awx_del_dict == collector.data
# check that field updates are the same
awx_del_dict = OrderedDict()
for model, instances_for_fieldvalues in awx_col.field_updates.items():
awx_del_dict.setdefault(model, {})
for (field, value), instances in instances_for_fieldvalues.items():
awx_del_dict[model].setdefault((field, value), set())
for inst in instances:
awx_del_dict[model][(field, value)].update(inst)
# collector field updates don't use the base (polymorphic parent) model, e.g.
# it will use JobTemplate instead of UnifiedJobTemplate. Therefore,
# we need to rebuild the dictionary and grab the model from the field
collector_del_dict = OrderedDict()
for model, instances_for_fieldvalues in collector.field_updates.items():
for (field, value), instances in instances_for_fieldvalues.items():
collector_del_dict.setdefault(field.model, {})
collector_del_dict[field.model][(
field, value)] = collector.field_updates[model][(field, value)]
assert awx_del_dict == collector_del_dict
# check that fast deletes are the same
collector_fast_deletes = set()
for q in collector.fast_deletes:
collector_fast_deletes.update(q)
awx_col_fast_deletes = set()
for q in awx_col.fast_deletes:
awx_col_fast_deletes.update(q)
assert collector_fast_deletes == awx_col_fast_deletes
def compress(self, log=None, **options):
"""
Searches templates containing 'compress' nodes and compresses them
"offline" -- outside of the request/response cycle.
The result is cached with a cache-key derived from the content of the
compress nodes (not the content of the possibly linked files!).
"""
extensions = options.get('extensions')
extensions = self.handle_extensions(extensions or ['html'])
verbosity = int(options.get("verbosity", 0))
if not log:
log = StringIO()
if not settings.TEMPLATE_LOADERS:
raise OfflineGenerationError("No template loaders defined. You "
"must set TEMPLATE_LOADERS in your "
"settings.")
paths = set()
for loader in self.get_loaders():
try:
module = import_module(loader.__module__)
get_template_sources = getattr(module,
'get_template_sources', None)
if get_template_sources is None:
get_template_sources = loader.get_template_sources
paths.update(list(get_template_sources('')))
except (ImportError, AttributeError, TypeError):
# Yeah, this didn't work out so well, let's move on
pass
if not paths:
raise OfflineGenerationError("No template paths found. None of "
"the configured template loaders "
"provided template paths. See "
"https://docs.djangoproject.com/en/1.8/topics/templates/ "
"for more information on template "
"loaders.")
if verbosity > 1:
log.write("Considering paths:\n\t" + "\n\t".join(paths) + "\n")
templates = set()
for path in paths:
for root, dirs, files in os.walk(path,
followlinks=options.get('followlinks', False)):
templates.update(os.path.join(root, name)
for name in files if not name.startswith('.') and
any(fnmatch(name, "*%s" % glob) for glob in extensions))
if not templates:
raise OfflineGenerationError("No templates found. Make sure your "
"TEMPLATE_LOADERS and TEMPLATE_DIRS "
"settings are correct.")
if verbosity > 1:
log.write("Found templates:\n\t" + "\n\t".join(templates) + "\n")
engine = options.get("engine", "django")
parser = self.__get_parser(engine)
compressor_nodes = OrderedDict()
for template_name in templates:
try:
template = parser.parse(template_name)
except IOError: # unreadable file -> ignore
if verbosity > 0:
log.write("Unreadable template at: %s\n" % template_name)
continue
except TemplateSyntaxError as e: # broken template -> ignore
if verbosity > 0:
log.write("Invalid template %s: %s\n" % (template_name, e))
continue
except TemplateDoesNotExist: # non existent template -> ignore
if verbosity > 0:
log.write("Non-existent template at: %s\n" % template_name)
continue
except UnicodeDecodeError:
if verbosity > 0:
log.write("UnicodeDecodeError while trying to read "
"template %s\n" % template_name)
try:
nodes = list(parser.walk_nodes(template))
except (TemplateDoesNotExist, TemplateSyntaxError) as e:
# Could be an error in some base template
if verbosity > 0:
log.write("Error parsing template %s: %s\n" % (template_name, e))
continue
if nodes:
template.template_name = template_name
compressor_nodes.setdefault(template, []).extend(nodes)
if not compressor_nodes:
raise OfflineGenerationError(
"No 'compress' template tags found in templates."
"Try running compress command with --follow-links and/or"
"--extension=EXTENSIONS")
if verbosity > 0:
log.write("Found 'compress' tags in:\n\t" +
"\n\t".join((t.template_name
for t in compressor_nodes.keys())) + "\n")
contexts = settings.COMPRESS_OFFLINE_CONTEXT
if isinstance(contexts, six.string_types):
try:
module, function = get_mod_func(contexts)
contexts = getattr(import_module(module), function)()
except (AttributeError, ImportError, TypeError) as e:
raise ImportError("Couldn't import offline context function %s: %s" %
(settings.COMPRESS_OFFLINE_CONTEXT, e))
elif not isinstance(contexts, (list, tuple)):
contexts = [contexts]
log.write("Compressing... ")
block_count = context_count = 0
results = []
offline_manifest = OrderedDict()
for context_dict in contexts:
context_count += 1
init_context = parser.get_init_context(context_dict)
for template, nodes in compressor_nodes.items():
context = Context(init_context)
template._log = log
template._log_verbosity = verbosity
if not parser.process_template(template, context):
continue
for node in nodes:
context.push()
parser.process_node(template, context, node)
rendered = parser.render_nodelist(template, context, node)
key = get_offline_hexdigest(rendered)
if key in offline_manifest:
continue
try:
result = parser.render_node(template, context, node)
except Exception as e:
raise CommandError("An error occurred during rendering %s: "
"%s" % (template.template_name, e))
offline_manifest[key] = result
context.pop()
results.append(result)
block_count += 1
write_offline_manifest(offline_manifest)
log.write("done\nCompressed %d block(s) from %d template(s) for %d context(s).\n" %
(block_count, len(compressor_nodes), context_count))
return block_count, results
class PoserPmdFile(object):
# Author of the class: Fredi Hartmann, Germany, aka ADP
# Email : [email protected]
def __init__(self, fh=None):
if isinstance(fh, str):
fh = open(fh, "rb")
if fh is not None:
self.filename = fh.name
self.fh = fh # Filehandle to read from.
self.read_header() # Check file-signature
else:
self.filename = self.fh = None
self.num_targets = None # No target read yet.
self.targets = OrderedDict() # storage for Morph-Structure
def read_header(self):
"""
Check file-signature.
Raise IOError if signature is not "PZMD".
:return: True
"""
self.fh.seek(0)
s = self.fh.read(4)
if s != b"PZMD":
raise IOError("Not a Poser PMD file")
return True
def _unpack(self, fmt, size):
result = struct.unpack("!" + fmt, self.fh.read(size))
return result[0]
def read_all(self):
"""
Calls read_target() and read_deltas()
:return: None
"""
self.read_targets()
self.read_deltas()
def read_targets(self):
"""
Read all morphtargets from open file (filehandle in self.fh)
:return: <OrderedDict>self.targets
"""
if self.fh.closed:
raise IOError("File '%s' is closed" % self.fh.name)
self.fh.seek(0x10)
self.num_targets = self._read_uint(4)
self.targets.clear()
for i in range(self.num_targets):
morphname = self._read_str()
actorname = self._read_str()
target = self.targets.setdefault(morphname, OrderedDict())
actor = target.setdefault(actorname, OrderedDict())
actor["numb_deltas"] = self._read_uint(4)
actor["morphpos"] = self._read_uint(4)
self.read_uuids(self.fh.tell())
return self.targets
def read_deltas(self):
"""
Read deltas from open file (filehandle in self.fh)
into self.targets.
:return: None
"""
if self.targets is None:
self.read_targets()
def read_one(pos):
result = list()
self.fh.seek(pos)
indexes = self._read_uint(4)
for i in range(indexes):
result.append(
DELTA(self._read_uint(4), VERTEX(*self._read_vector())))
return result
for morphname, v in self.targets.items():
for actor, data in v.items():
data["deltas"] = read_one(data["morphpos"])
def read_uuids(self, pos):
"""
Read all uuids starting at file-position <pos>.
:param pos: fileposition
:return: None
"""
self.fh.seek(pos)
for name, target in self.targets.items():
for actorname, actordata in target.items():
actordata["UUID"] = self._read_str()
def _read_str(self, length=1):
strlen = self._read_uint(length)
return self.fh.read(strlen).decode("latin1")
def _read_int(self, size):
if size == 1:
return self._unpack("b", size)
if size == 2:
return self._unpack("h", size)
if size == 4:
return self._unpack("i", size)
raise ValueError("invalid int size: " + size)
def _read_uint(self, size):
if size == 1:
return self._unpack("B", size)
if size == 2:
return self._unpack("H", size)
if size == 4:
return self._unpack("I", size)
raise ValueError("invalid int size: " + size)
def _read_vector(self):
return struct.unpack("!fff", self.fh.read(12))
def _read_float(self):
return self._unpack("f", 4)
def setdefault(self, key, default=None):
"""
Override ``.setdefault()`` to perform case-insensitively.
"""
return OrderedDict.setdefault(self, key.lower(), default)
class Blueprint(FlaskBlueprint, ArgumentsMixin, ResponseMixin, PaginationMixin,
EtagMixin):
"""Blueprint that registers info in API documentation"""
def __init__(self, *args, **kwargs):
self.description = kwargs.pop('description', '')
super().__init__(*args, **kwargs)
# _docs is an ordered dict storing endpoints documentation:
# {endpoint: {
# 'get': documentation,
# 'post': documentation,
# ...
# }
# }
self._docs = OrderedDict()
def route(self, rule, **options):
"""Decorator to register url rule in application
Also stores doc info for later registration
Use this to decorate a :class:`MethodView <flask.views.MethodView>` or
a resource function
"""
def decorator(func):
# By default, endpoint name is function name
endpoint = options.pop('endpoint', func.__name__)
# MethodView (class)
if isinstance(func, MethodViewType):
# This decorator may be called multiple times on the same
# MethodView, but Flask will complain if different views are
# mapped to the same endpoint, so we should call 'as_view' only
# once and keep the result in MethodView._view_func
if not getattr(func, '_view_func', None):
func._view_func = func.as_view(endpoint)
view_func = func._view_func
# Function
else:
view_func = func
# Add URL rule in Flask and store endpoint documentation
self.add_url_rule(rule, endpoint, view_func, **options)
self._store_endpoint_docs(endpoint, func, **options)
return func
return decorator
def _store_endpoint_docs(self, endpoint, obj, **kwargs):
"""Store view or function doc info"""
endpoint_doc = self._docs.setdefault(endpoint, OrderedDict())
def store_method_docs(method, function):
# Get summary/description from docstring
docstring = function.__doc__
doc = load_info_from_docstring(docstring) if docstring else {}
# Update doc with description from @doc decorator
doc.update(getattr(function, '_apidoc', {}))
# Add function doc to table for later registration
method_l = method.lower()
# Check another doc was not already registed for endpoint/method
if method_l in endpoint_doc and endpoint_doc[method_l] is not doc:
# If multiple routes point to the same endpoint, the doc may
# be already registered.
# Only trigger exception if a different doc is passed.
raise EndpointMethodDocAlreadyRegistedError(
"Another doc is already registered for endpoint '{}' "
"method {}".format(endpoint, method_l.upper()))
endpoint_doc[method_l] = doc
# MethodView (class)
if isinstance(obj, MethodViewType):
for method in HTTP_METHODS:
if method in obj.methods:
func = getattr(obj, method.lower())
store_method_docs(method, func)
# Function
else:
methods = kwargs.pop('methods', None) or ['GET']
for method in methods:
store_method_docs(method, obj)
def register_views_in_doc(self, app, spec):
"""Register views information in documentation
If a schema in a parameter or a response appears in the spec
`definitions` section, it is replaced by a reference to its definition
in the parameter or response documentation:
"schema":{"$ref": "#/definitions/MySchema"}
"""
# This method uses the documentation information associated with the
# endpoint (in self._docs) to provide documentation for the route to
# the spec object.
#
for endpoint, doc in self._docs.items():
# doc is a dict of documentation per method for the endpoint
# {'get': documentation, 'post': documentation,...}
# Prepend Blueprint name to endpoint
endpoint = '.'.join((self.name, endpoint))
# Tag all operations with Blueprint name
# Format operations documentation in OpenAPI structure
for operation in doc.values():
operation['tags'] = [self.name]
self._prepare_doc(operation, spec.openapi_version)
# TODO: Do we support multiple rules per endpoint?
# https://github.com/marshmallow-code/apispec/issues/181
for rule in app.url_map.iter_rules(endpoint):
# We need to deepcopy operations here
# because it can be modified in APISpec.path(), which causes
# issues if there are multiple rules for the same endpoint
if APISPEC_VERSION_MAJOR < 1:
spec.add_path(rule=rule, operations=deepcopy(doc))
else:
spec.path(rule=rule, operations=deepcopy(doc))
@staticmethod
def _prepare_doc(operation, openapi_version):
"""Format operation documentation in OpenAPI structure
The decorators store all documentation information in a dict structure
that is close to OpenAPI doc structure, so this information could
_almost_ be copied as is. Yet, some adjustemnts may have to be
performed, especially if the spec structure differs between OpenAPI
versions: the OpenAPI version is not known when the decorators are
applied but only at registration time when this method is called.
"""
if openapi_version.major >= 3:
if 'responses' in operation:
for resp in operation['responses'].values():
if 'schema' in resp:
resp['content'] = {
'application/json': {
'schema': resp.pop('schema')
}
}
if 'parameters' in operation:
for param in operation['parameters']:
if param['in'] == 'body':
request_body = {
**{
'content': {
'application/json': {
'schema': param['schema']
}
}
},
**{
x: param[x]
for x in ('description', 'required') if x in param
}
}
operation['requestBody'] = request_body
# There can be only one requestBody
continue
parameters = [
param for param in operation['parameters']
if not param['in'] == 'body'
]
if parameters:
operation['parameters'] = parameters
else:
del operation['parameters']
@staticmethod
def doc(**kwargs):
"""Decorator adding description attributes to a view function
Values passed as kwargs are copied verbatim in the docs
Example: ::
@blp.doc(description="Return pets based on ID",
summary="Find pets by ID"
)
def get(...):
...
"""
def decorator(func):
func._apidoc = deepupdate(getattr(func, '_apidoc', {}), kwargs)
return func
return decorator
def _compare_namelists(gold_namelists, comp_namelists, case):
###############################################################################
"""
Compare two namelists. Print diff information if any.
Returns comments
Note there will only be comments if the namelists were not an exact match
Expect args in form: {namelist -> {key -> value} }.
value can be an int, string, list, or dict
>>> teststr = '''&nml
... val = 'foo'
... aval = 'one','two', 'three'
... maval = 'one', 'two', 'three', 'four'
... dval = 'one -> two', 'three -> four'
... mdval = 'one -> two', 'three -> four', 'five -> six'
... nval = 1850
... /
... &nml2
... val2 = .false.
... /
... '''
>>> _compare_namelists(_parse_namelists(teststr.splitlines(), 'foo'), _parse_namelists(teststr.splitlines(), 'bar'), None)
''
>>> teststr1 = '''&nml1
... val11 = 'foo'
... /
... &nml2
... val21 = 'foo'
... val22 = 'foo', 'bar', 'baz'
... val23 = 'baz'
... val24 = '1 -> 2', '2 -> 3', '3 -> 4'
... /
... &nml3
... val3 = .false.
... /'''
>>> teststr2 = '''&nml01
... val11 = 'foo'
... /
... &nml2
... val21 = 'foo0'
... val22 = 'foo', 'bar0', 'baz'
... val230 = 'baz'
... val24 = '1 -> 20', '2 -> 3', '30 -> 4'
... /
... &nml3
... val3 = .false.
... /'''
>>> comments = _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'), _parse_namelists(teststr2.splitlines(), 'bar'), None)
>>> print(comments)
Missing namelist: nml1
Differences in namelist 'nml2':
BASE: val21 = 'foo'
COMP: val21 = 'foo0'
BASE: val22 list item 1 = 'bar'
COMP: val22 list item 1 = 'bar0'
missing variable: 'val23'
BASE: val24 dict item 1 = 2
COMP: val24 dict item 1 = 20
dict variable 'val24' missing key 3 with value 4
dict variable 'val24' has extra key 30 with value 4
found extra variable: 'val230'
Found extra namelist: nml01
<BLANKLINE>
>>> teststr1 = '''&rad_cnst_nl
... icecldoptics = 'mitchell'
... logfile = 'cpl.log.150514-001533'
... case_name = 'ERB.f19_g16.B1850C5.skybridge_intel.C.150513-230221'
... runid = 'FOO'
... model_version = 'cam5_3_36'
... username = '******'
... iceopticsfile = '/projects/ccsm/inputdata/atm/cam/physprops/iceoptics_c080917.nc'
... liqcldoptics = 'gammadist'
... liqopticsfile = '/projects/ccsm/inputdata/atm/cam/physprops/F_nwvl200_mu20_lam50_res64_t298_c080428.nc'
... mode_defs = 'mam3_mode1:accum:=', 'A:num_a1:N:num_c1:num_mr:+',
... 'A:so4_a1:N:so4_c1:sulfate:/projects/ccsm/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', 'A:pom_a1:N:pom_c1:p-organic:/projects/ccsm/inputdata/atm/cam/physprops/ocpho_rrtmg_c101112.nc:+',
... 'A:soa_a1:N:soa_c1:s-organic:/projects/ccsm/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', 'A:bc_a1:N:bc_c1:black-c:/projects/ccsm/inputdata/atm/cam/physprops/bcpho_rrtmg_c100508.nc:+',
... 'A:dst_a1:N:dst_c1:dust:/projects/ccsm/inputdata/atm/cam/physprops/dust4_rrtmg_c090521.nc:+', 'A:ncl_a1:N:ncl_c1:seasalt:/projects/ccsm/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc',
... 'mam3_mode2:aitken:=', 'A:num_a2:N:num_c2:num_mr:+',
... 'A:so4_a2:N:so4_c2:sulfate:/projects/ccsm/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', 'A:soa_a2:N:soa_c2:s-organic:/projects/ccsm/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+',
... 'A:ncl_a2:N:ncl_c2:seasalt:/projects/ccsm/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc', 'mam3_mode3:coarse:=',
... 'A:num_a3:N:num_c3:num_mr:+', 'A:dst_a3:N:dst_c3:dust:/projects/ccsm/inputdata/atm/cam/physprops/dust4_rrtmg_c090521.nc:+',
... 'A:ncl_a3:N:ncl_c3:seasalt:/projects/ccsm/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc:+', 'A:so4_a3:N:so4_c3:sulfate:/projects/ccsm/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc'
... rad_climate = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2',
... 'N:ozone:O3', 'N:N2O:N2O', 'N:CH4:CH4',
... 'N:CFC11:CFC11', 'N:CFC12:CFC12', 'M:mam3_mode1:/projects/ccsm/inputdata/atm/cam/physprops/mam3_mode1_rrtmg_c110318.nc',
... 'M:mam3_mode2:/projects/ccsm/inputdata/atm/cam/physprops/mam3_mode2_rrtmg_c110318.nc', 'M:mam3_mode3:/projects/ccsm/inputdata/atm/cam/physprops/mam3_mode3_rrtmg_c110318.nc'
... /'''
>>> teststr2 = '''&rad_cnst_nl
... icecldoptics = 'mitchell'
... logfile = 'cpl.log.150514-2398745'
... case_name = 'ERB.f19_g16.B1850C5.skybridge_intel.C.150513-1274213'
... runid = 'BAR'
... model_version = 'cam5_3_36'
... username = '******'
... iceopticsfile = '/something/else/inputdata/atm/cam/physprops/iceoptics_c080917.nc'
... liqcldoptics = 'gammadist'
... liqopticsfile = '/something/else/inputdata/atm/cam/physprops/F_nwvl200_mu20_lam50_res64_t298_c080428.nc'
... mode_defs = 'mam3_mode1:accum:=', 'A:num_a1:N:num_c1:num_mr:+',
... 'A:so4_a1:N:so4_c1:sulfate:/something/else/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', 'A:pom_a1:N:pom_c1:p-organic:/something/else/inputdata/atm/cam/physprops/ocpho_rrtmg_c101112.nc:+',
... 'A:soa_a1:N:soa_c1:s-organic:/something/else/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+', 'A:bc_a1:N:bc_c1:black-c:/something/else/inputdata/atm/cam/physprops/bcpho_rrtmg_c100508.nc:+',
... 'A:dst_a1:N:dst_c1:dust:/something/else/inputdata/atm/cam/physprops/dust4_rrtmg_c090521.nc:+', 'A:ncl_a1:N:ncl_c1:seasalt:/something/else/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc',
... 'mam3_mode2:aitken:=', 'A:num_a2:N:num_c2:num_mr:+',
... 'A:so4_a2:N:so4_c2:sulfate:/something/else/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc:+', 'A:soa_a2:N:soa_c2:s-organic:/something/else/inputdata/atm/cam/physprops/ocphi_rrtmg_c100508.nc:+',
... 'A:ncl_a2:N:ncl_c2:seasalt:/something/else/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc', 'mam3_mode3:coarse:=',
... 'A:num_a3:N:num_c3:num_mr:+', 'A:dst_a3:N:dst_c3:dust:/something/else/inputdata/atm/cam/physprops/dust4_rrtmg_c090521.nc:+',
... 'A:ncl_a3:N:ncl_c3:seasalt:/something/else/inputdata/atm/cam/physprops/ssam_rrtmg_c100508.nc:+', 'A:so4_a3:N:so4_c3:sulfate:/something/else/inputdata/atm/cam/physprops/sulfate_rrtmg_c080918.nc'
... rad_climate = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2',
... 'N:ozone:O3', 'N:N2O:N2O', 'N:CH4:CH4',
... 'N:CFC11:CFC11', 'N:CFC12:CFC12', 'M:mam3_mode1:/something/else/inputdata/atm/cam/physprops/mam3_mode1_rrtmg_c110318.nc',
... 'M:mam3_mode2:/something/else/inputdata/atm/cam/physprops/mam3_mode2_rrtmg_c110318.nc', 'M:mam3_mode3:/something/else/inputdata/atm/cam/physprops/mam3_mode3_rrtmg_c110318.nc'
... /'''
>>> _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'), _parse_namelists(teststr2.splitlines(), 'bar'), 'ERB.f19_g16.B1850C5.skybridge_intel')
''
"""
different_namelists = OrderedDict()
for namelist, gold_names in gold_namelists.items():
if (namelist not in comp_namelists):
different_namelists[namelist] = [
"Missing namelist: {}\n".format(namelist)
]
else:
comp_names = comp_namelists[namelist]
for name, gold_value in gold_names.items():
if (name not in comp_names):
different_namelists.setdefault(namelist, []).append(
" missing variable: '{}'\n".format(name))
else:
comp_value = comp_names[name]
comments = _compare_values(name, gold_value, comp_value,
case)
if comments != "":
different_namelists.setdefault(namelist,
[]).append(comments)
for name in comp_names:
if (name not in gold_names):
different_namelists.setdefault(namelist, []).append(
" found extra variable: '{}'\n".format(name))
for namelist in comp_namelists:
if (namelist not in gold_namelists):
different_namelists[namelist] = [
"Found extra namelist: {}\n".format(namelist)
]
comments = ""
for namelist, nlcomment in different_namelists.items():
if len(nlcomment) == 1:
comments += nlcomment[0]
else:
comments += "Differences in namelist '{}':\n".format(namelist)
comments += "".join(nlcomment)
return comments
def _traverse_profile_sections(profile_sections, request, start_section_num=0):
sections_by_title = OrderedDict()
section_topics_by_title = OrderedDict()
section_topic_indicators_by_title = OrderedDict()
for section_num, profile_section in enumerate(profile_sections,
start=start_section_num + 1):
(country, profile_title) = (str(profile_section.get_parent()), profile_section.title) \
if isinstance(profile_section, ProfileSectionPage) \
else (str(profile_section), 'Country Overview')
default_section = {
'id': 'section-{}'.format(section_num),
'title': profile_title,
'href': 'section-{}-topics-tab'.format(section_num),
'key_issues': [],
}
section = sections_by_title.setdefault(profile_title.lower(),
default_section)
topics_by_title = section_topics_by_title.setdefault(
profile_title.lower(), OrderedDict())
topic_indicators_by_title = section_topic_indicators_by_title.setdefault(
profile_title.lower(), OrderedDict())
start_topic_num = len(topics_by_title.keys())
for topic_num, section_topic in enumerate(profile_section.body,
start=start_topic_num + 1):
# Topics that have no indicators (key issues) should be
# displayed separately.
topic_title = section_topic.value['title']
if not section_topic.value['indicators']:
section['key_issues'].append({
'id':
'{}-key_issue-{}'.format(section['id'], topic_num),
'title':
topic_title,
'country':
country,
'href':
profile_section.get_url(request),
})
else:
default_topic = {
'id':
'{}-topic-{}'.format(section['id'], topic_num),
'title':
topic_title,
'href':
'{}-topic-{}-indicators-tab'.format(
section['id'], topic_num),
}
topic = topics_by_title.setdefault(topic_title.lower(),
default_topic)
indicators_by_title = topic_indicators_by_title.setdefault(
topic_title.lower(), OrderedDict())
start_indicator_num = len(indicators_by_title.keys())
for indicator_num, topic_indicator in enumerate(
section_topic.value['indicators'],
start=start_indicator_num + 1):
indicator_title = topic_indicator.value['title']
default_indicator = {
'id':
'{}-indicator-{}'.format(topic['id'], indicator_num),
'title':
indicator_title,
'href':
'{}-indicator-{}-country-selections-tab'.format(
topic['id'], indicator_num),
'countries': [],
}
indicator = indicators_by_title.setdefault(
indicator_title.lower(), default_indicator)
indicator['countries'].append({
'title':
country,
'href':
profile_section.get_url(request),
})
topic['indicators'] = indicators_by_title.values()
section['topics'] = topics_by_title.values()
return sections_by_title.values()
def showHitCoinList(request):
context = {}
items = StorageReport.objects.all()
clientItems = Clients.objects.all()
userMap = {}
for item in clientItems:
node_id = item.node_id
username = item.username
userMap[node_id] = username
rewardAddDict = OrderedDict()
onlineTime_all = 0
for item in items:
node_id = item.node_id
repo_size = item.repo_size
storage_size = item.storage_size
address = item.address
if address == "127.0.0.1" or address == "localhost":
continue
if node_id not in userMap:
continue
print(address)
hit_per_bit = 2 / 10000000000
reward_repo_size_divide_10G = 0.0
for i in range(len(repo_size) - 1, -1, -1):
b = int(repo_size[i])
reward_repo_size_divide_10G += b * hit_per_bit
hit_per_bit *= 10
rewardAddDict.setdefault(address, (0, 0.0))
rewardAddDict[address] = (rewardAddDict[address][0] + 1, rewardAddDict[address][1] + reward_repo_size_divide_10G) # hours
onlineTime_all += 1
addresses = list(rewardAddDict.keys())
locations, errorIps = getLocByIpList(addresses)
if locations == None:
context["Error"] = True
return render(request, "hitcoinList.html", locals())
print("location is: ")
print(locations)
for ip in errorIps:
rewardAddDict.pop(ip)
addresses.remove(ip)
longitudes = []
latitudes = []
hours = []
realHours = []
for i in range(len(addresses)):
add = addresses[i]
loc = locations[i]
value = rewardAddDict[add]
longitudes.append(loc[0])
latitudes.append(loc[1])
hours.append(math.ceil(value[0] / onlineTime_all * 30)) # only record the online time
realHours.append(value[0])
return render(request, "hitcoinList.html", locals())
def parse_optgeo_options(filename):
""" Parse an optgeo_options.txt file into specific OptGeoTarget Target Options"""
logger.info("Reading optgeo options from file: %s\n" % filename)
global_opts = OrderedDict()
sys_opts = OrderedDict()
section = None
section_opts = OrderedDict()
with open(filename) as f:
for ln, line in enumerate(f, 1):
# Anything after "#" is a comment
line = line.split("#", maxsplit=1)[0].strip()
if not line: continue
ls = line.split()
key = ls[0].lower()
if key[0] == "$":
# section sign $
if key == '$end':
if section is None:
warn_press_key(
"Line %i: Encountered $end before any section."
% ln)
elif section == 'global':
# global options read finish
global_opts = section_opts
elif section == 'system':
# check if system section contains name
if 'name' not in section_opts:
warn_press_key(
"Line %i: You need to specify a name for the system section ending."
% ln)
elif section_opts['name'] in sys_opts:
warn_press_key(
"Line %i: A system named %s already exists in Systems"
% (ln, section_opts['name']))
else:
sys_opts[section_opts['name']] = section_opts
section = None
section_opts = OrderedDict()
else:
if section is not None:
warn_press_key(
"Line %i: Encountered section start %s before previous section $end."
% (ln, key))
if key == '$global':
section = 'global'
elif key == '$system':
section = 'system'
else:
warn_press_key(
"Line %i: Encountered unsupported section name %s "
% (ln, key))
else:
# put normal key-value options into section_opts
if key in ['name', 'geometry', 'topology']:
if len(ls) != 2:
warn_press_key(
"Line %i: one value expected for key %s" %
(ln, key))
if section == 'global':
warn_press_key(
"Line %i: key %s should not appear in $global section"
% (ln, key))
section_opts[key] = ls[1]
elif key in [
'bond_denom', 'angle_denom', 'dihedral_denom',
'improper_denom'
]:
if len(ls) != 2:
warn_press_key(
"Line %i: one value expected for key %s" %
(ln, key))
section_opts[key] = float(ls[1])
elif key == 'mol2':
# special parsing for mol2 option for SMIRNOFF engine
# the value is a list of filenames
section_opts[key] = ls[1:]
# apply a few default global options
global_opts.setdefault('bond_denom', 0.02)
global_opts.setdefault('angle_denom', 3)
global_opts.setdefault('dihedral_denom', 10.0)
global_opts.setdefault('improper_denom', 10.0)
# copy global options into each system
for sys_name, sys_opt_dict in sys_opts.items():
for k, v in global_opts.items():
# do not overwrite system options
sys_opt_dict.setdefault(k, v)
for k in ['name', 'geometry', 'topology']:
if k not in sys_opt_dict:
warn_press_key(
"key %s missing in system section named %s" %
(k, sys_name))
return sys_opts
else:
size = np.zeros(len(x))
# n is the number of nodes
n = len(x)
# creates node numbers from the nodes file
node = np.zeros(n, dtype=np.int32)
# to check for duplicate nodes
# this piece of code uses OrderedDict to remove duplicate nodes
# source "http://stackoverflow.com/questions/12698987"
# ###################################################################
tmp = OrderedDict()
for point in zip(x, y, z, size):
tmp.setdefault(point[:2], point)
# in python 3 tmp.values() is a view object that needs to be
# converted to a list
mypoints = list(tmp.values())
# ###################################################################
n_rev = len(mypoints)
# replace x,y,z,size and n with their unique equivalents
if (duplicates_flag == 1):
for i in range(n_rev):
x[i] = mypoints[i][0]
y[i] = mypoints[i][1]
z[i] = mypoints[i][2]
size[i] = mypoints[i][3]
n = n_rev
