def make_sections(allruns, common=None):
# print allruns.keys()
if common is None:
common = {}
# print('Selecting %d with %s' % (len(allruns), common))
if len(allruns) == 1:
key = list(allruns.keys())[0]
value = allruns[key]
return dict(type='sample', common=common, key=key, value=value)
fields_size = [(field, len(list(allruns.groups_by_field_value(field))))
for field in allruns.field_names_in_all_keys()]
# Now choose the one with the least choices
fields_size.sort(key=lambda x: x[1])
if not fields_size:
# [frozendict({'i': 1, 'n': 3}), frozendict({'i': 2, 'n': 3}), frozendict({}), frozendict({'i': 0, 'n': 3})]
msg = 'Not all records of the same type have the same fields'
msg += pformat(list(allruns.keys()))
raise ValueError(msg)
field = fields_size[0][0]
division = {}
for value, samples in allruns.groups_by_field_value(field):
samples = samples.remove_field(field)
c = dict(common)
c[field] = value
try:
division[value] = make_sections(samples, common=c)
except Exception:
msg = 'Error occurred inside grouping by field %r = %r' % (field, value)
msg += '\nCommon: %r' % common
msg += '\nSamples: %s' % list(samples.keys())
logger.error(msg)
raise
return dict(type='division', field=field,
division=division, common=common)
def check_fails(f, *args, **kwargs):
try:
f(*args, **kwargs)
except BaseException as e:
logger.error('Known failure for %s ' % f)
logger.warn('Fails with error %s' % e)
#comptest_fails = kwargs.get('comptest_fails', f.__name__)
d = 'out/comptests-failures'
if not os.path.exists(d):
os.makedirs(d)
job_id = JobCompute.current_job_id
if job_id is None:
job_id = 'nojob-%s' % f.__name__
out = os.path.join(d, job_id + '.txt')
# for i in range(1000):
# outi = out % i
# if not os.path.exists(outi):
with open(out, 'w') as f:
f.write(traceback.format_exc(e))
else:
msg = 'Function was supposed to fail.'
raise_desc(Exception, msg, f=f, args=args, kwargs=kwargs)
def index_reports(reports, index, update=None): # @UnusedVariable
"""
Writes an index for the report to the file given.
The special key "report" gives the report type.
report[dict(report=...,param1=..., param2=...) ] => filename
"""
# print('Updating because of new report %s' % update)
dirname = os.path.dirname(index)
if not os.path.exists(dirname):
os.makedirs(dirname)
# logger.info('Writing on %s' % friendly_path(index))
f = open(index, 'w')
f.write("""
<html>
<head>
<style type="text/css">
span.when { float: right; }
li { clear: both; }
a.self { color: black; text-decoration: none; }
</style>
</head>
<body>
""")
mtime = lambda x: os.path.getmtime(x)
existing = list([x for x in list(reports.items()) if os.path.exists(x[1])])
# create order statistics
alltimes = np.array([mtime(b) for _, b in existing])
def order(filename):
""" returns between 0 and 1 the order statistics """
assert os.path.exists(filename)
histime = mtime(filename)
compare = (alltimes < histime)
return np.mean(compare * 1.0)
def style_order(order):
if order > 0.95:
return "color: green;"
if order > 0.9:
return "color: orange;"
if order < 0.5:
return "color: gray;"
return ""
@contract(k=dict, filename=str)
def write_li(k, filename, element='li'):
desc = ", ".join('%s = %s' % (a, b) for a, b in list(k.items()))
href = os.path.relpath(os.path.realpath(filename),
os.path.dirname(os.path.realpath(index)))
if os.path.exists(filename):
when = duration_compact(time.time() - mtime(filename))
span_when = '<span class="when">%s ago</span>' % when
style = style_order(order(filename))
a = '<a href="%s">%s</a>' % (href, desc)
else:
# print('File %s does not exist yet' % filename)
style = ""
span_when = '<span class="when">missing</span>'
a = '<a href="%s">%s</a>' % (href, desc)
f.write('<%s style="%s">%s %s</%s>' % (element, style, a, span_when,
element))
# write the first 10
existing.sort(key=lambda x: (-mtime(x[1])))
nlast = min(len(existing), 10)
last = existing[:nlast]
f.write('<h2 id="last">Last %d report</h2>\n' % (nlast))
f.write('<ul>')
for i in range(nlast):
write_li(*last[i])
f.write('</ul>')
if False:
for report_type, r in reports.groups_by_field_value('report'):
f.write('<h2 id="%s">%s</h2>\n' % (report_type, report_type))
f.write('<ul>')
r = reports.select(report=report_type)
items = list(r.items())
items.sort(key=lambda x: str(x[0])) # XXX use natsort
for k, filename in items:
write_li(k, filename)
f.write('</ul>')
f.write('<h2>All report</h2>\n')
try:
sections = make_sections(reports)
except:
logger.error(str(list(reports.keys())))
raise
if sections['type'] == 'sample':
# only one...
sections = dict(type='division', field='raw',
division=dict(raw1=sections), common=dict())
def write_sections(sections, parents):
assert 'type' in sections
assert sections['type'] == 'division', sections
field = sections['field']
division = sections['division']
f.write('<ul>')
sorted_values = natsorted(list(division.keys()))
for value in sorted_values:
parents.append(value)
html_id = "-".join(map(str, parents))
bottom = division[value]
if bottom['type'] == 'sample':
d = {field: value}
if not bottom['key']:
write_li(k=d, filename=bottom['value'], element='li')
else:
f.write('<li> <p id="%s"><a class="self" href="#%s">%s = %s</a></p>\n'
% (html_id, html_id, field, value))
f.write('<ul>')
write_li(k=bottom['key'], filename=bottom['value'], element='li')
f.write('</ul>')
f.write('</li>')
else:
f.write('<li> <p id="%s"><a class="self" href="#%s">%s = %s</a></p>\n'
% (html_id, html_id, field, value))
write_sections(bottom, parents)
f.write('</li>')
f.write('</ul>')
write_sections(sections, parents=[])
f.write('''
</body>
</html>
''')
f.close()