def jobs_tables_by_algo_rows_sample_groups(context, samples_groups, tables):
source_descs = comp_store(Stats.all_descriptions())
# Crate a new store, add the key "group"
allstats = StoreResults()
for id_group, samples in samples_groups.items():
for key, value in samples.items():
nkey = dict(id_group=id_group, **key)
allstats[nkey] = value
for id_statstable, stats in tables.items():
for id_algo, samples in allstats.groups_by_field_value('id_algo'):
job_id = 'byalgo-%s-%s' % (id_algo, id_statstable)
r = context.comp(table_by_rows,
"byalgo-rows-sample-groups-%s-%s" % (id_algo, id_statstable),
samples=samples,
rows_field='id_group', # rows = tc
cols_fields=stats, # which statistics for each col
source_descs=source_descs,
job_id=job_id)
report_attrs = dict(id_statstable=id_statstable)
report_attrs.update(samples.fields_with_unique_values())
assert report_attrs['id_algo'] == id_algo
context.add_report(r, 'byalgo-rows-sample-groups', **report_attrs)
def jobs_tables_by_algo_rows_sample_groups(context, samples_groups, tables):
source_descs = comp_store(Stats.all_descriptions())
# Crate a new store, add the key "group"
allstats = StoreResults()
for id_group, samples in samples_groups.items():
for key, value in samples.items():
nkey = dict(id_group=id_group, **key)
allstats[nkey] = value
for id_statstable, stats in tables.items():
for id_algo, samples in allstats.groups_by_field_value('id_algo'):
job_id = 'byalgo-%s-%s' % (id_algo, id_statstable)
r = context.comp(
table_by_rows,
"byalgo-rows-sample-groups-%s-%s" % (id_algo, id_statstable),
samples=samples,
rows_field='id_group', # rows = tc
cols_fields=stats, # which statistics for each col
source_descs=source_descs,
job_id=job_id)
report_attrs = dict(id_statstable=id_statstable)
report_attrs.update(samples.fields_with_unique_values())
assert report_attrs['id_algo'] == id_algo
context.add_report(r, 'byalgo-rows-sample-groups', **report_attrs)
class ReportManager(object):
# TODO: make it use a context
def __init__(self, outdir, index_filename=None):
self.outdir = outdir
if index_filename is None:
index_filename = os.path.join(self.outdir, 'report_index.html')
self.index_filename = index_filename
self.allreports = StoreResults()
self.allreports_filename = StoreResults()
# report_type -> set of keys necessary
self._report_types_format = {}
self.html_resources_prefix = ''
# check if we are called more than once; would be a bug
self.index_job_created = False
def set_html_resources_prefix(self, prefix):
"""
Sets the prefix for the resources filename.
example: set_resources_prefix('jbds')
"""
self.html_resources_prefix = prefix + '-'
def _check_report_format(self, report_type, **kwargs):
keys = sorted(list(kwargs.keys()))
# print('report %r %r' % (report_type, keys))
if not report_type in self._report_types_format:
self._report_types_format[report_type] = keys
else:
keys0 = self._report_types_format[report_type]
if not keys == keys0:
msg = 'Report %r %r' % (report_type, keys)
msg += '\ndoes not match previous format %r' % keys0
raise ValueError(msg)
def get(self, report_type, **kwargs):
key = frozendict2(report=report_type, **kwargs)
return self.allreports[key]
@contract(report_type='str')
def add(self, report, report_type, **kwargs):
"""
Adds a report to the collection.
:param report: Promise of a Report object
:param report_type: A string that describes the "type" of the report
:param kwargs: str->str,int,float parameters used for grouping
"""
if not isinstance(report_type, str):
msg = 'Need a string for report_type, got %r.' % describe_value(report_type)
raise ValueError(msg)
from compmake import Promise
if not isinstance(report, Promise):
msg = ('ReportManager is mean to be given Promise objects, '
'which are the output of comp(). Obtained: %s'
% describe_type(report))
raise ValueError(msg)
# check the format is ok
self._check_report_format(report_type, **kwargs)
key = frozendict2(report=report_type, **kwargs)
if key in self.allreports:
msg = 'Already added report for %s' % key
msg += '\n its values is %s' % self.allreports[key]
msg += '\n new value would be %s' % report
raise ValueError(msg)
self.allreports[key] = report
report_type_sane = report_type.replace('_', '')
key_no_report = dict(**key)
del key_no_report['report']
basename = self.html_resources_prefix + report_type_sane
if key_no_report:
basename += '-' + basename_from_key(key_no_report)
dirname = os.path.join(self.outdir, report_type_sane)
filename = os.path.join(dirname, basename)
self.allreports_filename[key] = filename + '.html'
def create_index_job(self):
if self.index_job_created:
msg = 'create_index_job() was already called once'
raise ValueError(msg)
self.index_job_created = True
if not self.allreports:
# no reports necessary
return
from compmake import comp
# Do not pass as argument, it will take lots of memory!
# XXX FIXME: there should be a way to make this update or not
# otherwise new reports do not appear
optimize_space = False
if optimize_space and len(self.allreports_filename) > 100:
allreports_filename = comp_store(self.allreports_filename, 'allfilenames')
else:
allreports_filename = self.allreports_filename
type2reports = {}
for report_type, xs in self.allreports_filename.groups_by_field_value('report'):
type2reports[report_type] = StoreResults(**xs.remove_field('report'))
for key in self.allreports:
job_report = self.allreports[key]
filename = self.allreports_filename[key]
write_job_id = job_report.job_id + '-write'
# Create the links to reports of the same type
report_type = key['report']
other_reports_same_type = type2reports[report_type]
key = dict(**key)
del key['report']
# find the closest report for different type
others = []
for other_type, other_type_reports in type2reports.items():
if other_type == report_type:
continue
best = get_most_similar(other_type_reports, key)
if best is not None:
# print('Best match:\n-%s %s\n- %s %s' % (report_type, key,
# other_type, best))
others.append((other_type, best, other_type_reports[best]))
report_type_sane = report_type.replace('_', '')
report_nid = self.html_resources_prefix + report_type_sane
if key:
report_nid += '-' + basename_from_key(key)
comp(write_report_and_update,
report=job_report, report_nid=report_nid,
report_html=filename, all_reports=allreports_filename,
index_filename=self.index_filename,
write_pickle=False,
this_report=key,
other_reports_same_type=other_reports_same_type,
most_similar_other_type=others,
job_id=write_job_id)
