def report_results_single(func, objspec_name, results):
def get_string_result(res):
if res is None:
s = 'ok'
elif isinstance(res, Skipped):
s = 'skipped'
elif isinstance(res, PartiallySkipped):
parts = res.get_skipped_parts()
s = 'no ' + ','.join(parts)
else:
print('how to interpret %s? ' % describe_value(res))
s = '?'
return s
r = Report()
if not results:
r.text('warning', 'no test objects defined')
return r
rows = []
data = []
for id_object, res in list(results.items()):
rows.append(id_object)
data.append([get_string_result(res)])
r.table('summary', rows=rows, data=data)
return r
def report_summary(results):
# results = tuple (name, res dict)
r = Report()
cols = ['scenario', 'number of bits', 'number of states']
data = []
for name, res in results:
agent = res['agent']
nstates = len(agent.get_all_states())
nbits = agent.get_num_states_components()
row = [name, nbits, nstates]
data.append(row)
r.table('summary', data=data, cols=cols)
cols = ['number of bits', 'number of states']
data = []
rows = []
for name, res in results:
agent = res['agent']
nstates = len(agent.get_all_states())
nbits = agent.get_num_states_components()
rows.append(name)
data.append([nbits, nstates])
r.table('summary2', data=data, cols=cols, rows=rows)
return r
def create_report_comb_stats(comb_id, tc_ids, alg_ids, deps):
r = Report('set-%s' % comb_id)
# has_ground_truth = 'cheat' in alg_ids or 'echeat' in alg_ids
#
# if 'cheat' in alg_ids: cheater = 'cheat'
# if 'echeat' in alg_ids: cheater = 'echeat'
#
# if has_ground_truth:
# for tc_id in tc_ids:
# max_spearman = deps[(tc_id, cheater)]['spearman']
# for alg_id in alg_ids:
# res = deps[(tc_id, alg_id)]
# res['spearman_score'] = res['spearman'] / max_spearman
def tablevar(var, format='%.2f', not_found=np.NaN):
def getter(tc_id, alg_id):
res = deps[(tc_id, alg_id)]
if var in res:
return format % res[var]
else:
return not_found
return getter
print entries
for var, format in entries:
caption = var
r.table(var, caption=caption,
**generic_table(tc_ids, alg_ids, tablevar(var, format)))
return r
def report_results_single(func, objspec_name, results):
def get_string_result(res):
if res is None:
s = 'ok'
elif isinstance(res, Skipped):
s = 'skipped'
elif isinstance(res, PartiallySkipped):
parts = res.get_skipped_parts()
s = 'no ' + ','.join(parts)
else:
print('how to interpret %s? ' % describe_value(res))
s = '?'
return s
r = Report()
if not results:
r.text('warning', 'no test objects defined')
return r
rows = []
data = []
for id_object, res in results.items():
rows.append(id_object)
data.append([get_string_result(res)])
r.table('summary', rows=rows, data=data)
return r
def sample_var_time_correlation(
sample, expdata, configuration, saccades, #@UnusedVariable
variables, delays, type='pearson'):
# all together
R, P, labels = get_correlation_matrix(saccades, variables, delays,
type)
# Significance
S = P < 0.01
nvars = len(variables)
Rhalf = R[:nvars, :]
Phalf = P[:nvars, :]
Shalf = S[:nvars, :]
ylabels = labels[:nvars]
r = Report()
attach_description(r, create_description(variables, delays, type))
with r.data_pylab('correlation') as pylab:
draw_correlation_figure(pylab, labels, ylabels, Rhalf)
rshow = lambda x: "%+.2f" % x
r.table('correlation_values', values_to_strings(Rhalf, rshow),
cols=labels, rows=ylabels, caption="%s coefficient" % type)
r.table('pvalues', values_to_strings(Phalf, pvalue_format),
cols=labels, rows=ylabels, caption="p-values")
with r.data_pylab('significance') as pylab:
draw_significance_figure(pylab, labels, ylabels, Shalf)
return r
def table_by_rows(id_report, samples, rows_field, cols_fields, source_descs):
samples2 = StoreResultsDict(samples)
class Missing(dict):
def __missing__(self, key):
logger.warning('Description for %r missing.' % key)
d = WithDescription(name=key, symbol='\\text{%s}' % key,
desc=None)
self[key] = d
return d
source_descs = Missing(source_descs)
r = Report(id_report)
data_views = [DataView.from_string(x, source_descs) for x in cols_fields]
# data: list of list of list
rows_field, data, reduced, display = summarize_data(samples2, rows_field, data_views)
rows = ['$%s$' % source_descs[x].get_symbol() for x in rows_field]
cols = ['$%s$' % x.get_symbol() for x in data_views]
r.table('table', data=display, cols=cols, rows=rows)
r.data('table_data', data=reduced,
caption="Data without presentation applied.")
r.data('table_data_source', data=data,
caption="Source data, before reduction.")
row_desc = "\n".join(['- $%s$: %s' % (x.get_symbol(), x.get_desc())
for x in map(source_descs.__getitem__, rows_field)])
col_desc = "\n".join(['- $%s$: %s' % (x.get_symbol(), x.get_desc())
for x in data_views])
r.text('row_desc', rst_escape_slash(row_desc), mime=MIME_RST)
r.text('col_desc', rst_escape_slash(col_desc), mime=MIME_RST)
return r
def fairness_table(all_results):
cols_desc = [' ID', 'Length (min)', 'Num. saccades', 'saccades/s',
'p_L', 'p value', 'rejected']
rows = []
for i, results in enumerate(all_results):
results = dict(results)
rejected = {True:'*', False:''}[results['fair_rejected']]
row = [i,
"%.1f" % (results['length'] / 60),
results['N'],
"%.2f" % results['density'],
"%.2f" % results['p_L'],
"%.3f" % results['fair_pvalue'],
rejected]
rows.append(row)
# caption = get_nice_dataset_description(dataset)
print rows
# sort by length
rows.sort(key=lambda x:-float(x[1]))
r = Report()
attach_description(r, description)
r.table('fairness', rows, cols=cols_desc)
return r
def create_report_comb_stats(comb_id, tc_ids, alg_ids, deps):
r = Report('set-%s' % comb_id)
# has_ground_truth = 'cheat' in alg_ids or 'echeat' in alg_ids
#
# if 'cheat' in alg_ids: cheater = 'cheat'
# if 'echeat' in alg_ids: cheater = 'echeat'
#
# if has_ground_truth:
# for tc_id in tc_ids:
# max_spearman = deps[(tc_id, cheater)]['spearman']
# for alg_id in alg_ids:
# res = deps[(tc_id, alg_id)]
# res['spearman_score'] = res['spearman'] / max_spearman
def tablevar(var, format='%.2f', not_found=np.NaN):
def getter(tc_id, alg_id):
res = deps[(tc_id, alg_id)]
if var in res:
return format % res[var]
else:
return not_found
return getter
print entries
for var, format in entries:
caption = var
r.table(var,
caption=caption,
**generic_table(tc_ids, alg_ids, tablevar(var, format)))
return r
def report_summary(results):
# results = tuple (name, res dict)
r = Report()
cols = ['scenario', 'number of bits', 'number of states']
data = []
for name, res in results:
agent = res['agent']
nstates = len(agent.get_all_states())
nbits = agent.get_num_states_components()
row = [name, nbits, nstates]
data.append(row)
r.table('summary', data=data, cols=cols)
cols = [ 'number of bits', 'number of states']
data = []
rows = []
for name, res in results:
agent = res['agent']
nstates = len(agent.get_all_states())
nbits = agent.get_num_states_components()
rows.append(name)
data.append([nbits, nstates])
r.table('summary2', data=data, cols=cols, rows=rows)
return r
def cov_plots(d):
Q = d.Q
Q_inv = d.Q_inv
r = Report('cov')
r.table('Q', Q)
r.table('Q_inv', Q_inv)
return r
def test_coords1():
vl = np.array([
0,
1,
0,
])
va = np.array([np.deg2rad(20), 0, 0])
vel = {
'F': vl,
'FL': vl + va,
'FR': vl - va,
'B': (-vl),
'BL': (-vl + va),
'BR': (-vl - va),
}
def make_motion(v):
A = se2.algebra_from_vector(v)
Q = SE2.group_from_algebra(A)
return Q
motions = dictmap(make_motion, vel)
print motions
for k, v in motions.items():
print(' - %s: %s -> %s' % (k, vel[k], SE2.friendly(v)))
names = sorted(vel.keys())
def commuting(a, b):
q1 = motions[a]
q2 = motions[b]
return SE2.distance(SE2.multiply(q1, q2), SE2.multiply(q2, q1))
def same(a, b):
q1 = motions[a]
q2 = motions[b]
return SE2.distance(q1, q2)
def anti(a, b):
q1 = motions[a]
q2 = motions[b]
return SE2.distance(q1, SE2.inverse(q2))
cD = construct_matrix_iterators((names, names), commuting)
aD = construct_matrix_iterators((names, names), anti)
D = construct_matrix_iterators((names, names), same)
r = Report('test_coords1')
r.table('D', data=D, cols=names, rows=names, fmt='%f')
r.table('aD', data=aD, cols=names, rows=names, fmt='%f')
r.table('cD', data=cD, cols=names, rows=names, fmt='%f')
r.to_html('out/test_coords1/test_coords1.html')
def test_coords1():
vl = np.array([0, 1, 0, ])
va = np.array([np.deg2rad(20), 0, 0])
vel = {
'F': vl,
'FL': vl + va,
'FR': vl - va,
'B': (-vl),
'BL': (-vl + va),
'BR': (-vl - va),
}
def make_motion(v):
A = se2.algebra_from_vector(v)
Q = SE2.group_from_algebra(A)
return Q
motions = dictmap(make_motion, vel)
print motions
for k, v in motions.items():
print(' - %s: %s -> %s' % (k, vel[k], SE2.friendly(v)))
names = sorted(vel.keys())
def commuting(a, b):
q1 = motions[a]
q2 = motions[b]
return SE2.distance(SE2.multiply(q1, q2),
SE2.multiply(q2, q1))
def same(a, b):
q1 = motions[a]
q2 = motions[b]
return SE2.distance(q1, q2)
def anti(a, b):
q1 = motions[a]
q2 = motions[b]
return SE2.distance(q1, SE2.inverse(q2))
cD = construct_matrix_iterators((names, names), commuting)
aD = construct_matrix_iterators((names, names), anti)
D = construct_matrix_iterators((names, names), same)
r = Report('test_coords1')
r.table('D', data=D, cols=names, rows=names, fmt='%f')
r.table('aD', data=aD, cols=names, rows=names, fmt='%f')
r.table('cD', data=cD, cols=names, rows=names, fmt='%f')
r.to_html('out/test_coords1/test_coords1.html')
def compute_general_statistics(id, db, samples, interval_function,
signal, signal_component):
r = Report(id)
x = get_all_data_for_signal(db, samples, interval_function,
signal, signal_component)
limit = 0.3
perc = [0.001, limit, 1, 10, 25, 50, 75, 90, 99, 100 - limit, 100 - 0.001]
xp = map(lambda p: "%.3f" % scipy.stats.scoreatpercentile(x, p), perc)
lower = scipy.stats.scoreatpercentile(x, limit)
upper = scipy.stats.scoreatpercentile(x, 100 - limit)
f = r.figure()
with r.data_pylab('histogram') as pylab:
bins = numpy. linspace(lower, upper, 100)
pylab.hist(x, bins=bins)
f.sub('histogram')
labels = map(lambda p: "%.3f%%" % p, perc)
r.table("percentiles", data=[xp], cols=labels, caption="Percentiles")
r.table("stats", data=[[x.mean(), x.std()]], cols=['mean', 'std.dev.'],
caption="Other statistics")
print "Computing correlation..."
corr, lags = xcorr(x, maxlag=20)
print "...done."
with r.data_pylab('cross_correlation') as pylab:
delta = (1.0 / 60) * lags * 1000;
pylab.plot(delta, corr, 'o-')
pylab.axis([min(delta), max(delta), -0.7, 1.1])
pylab.xlabel('delay (ms)')
f = r.figure()
f.sub('cross_correlation')
return r
def independence_table(all_results):
cols_desc = ['ID', 'Length (min)', 'N', 'n_L', 'n_R',
'n_RL', 'n_LL',
'n_RR', 'n_LR',
'p_L interval',
'best p_L',
'indep pvalue',
'why'
]
rows = []
for i, results in enumerate(all_results):
results = dict(results)
#rejected = {True:'*', False:''}[results['indep_rejected']]
def pformat(seq):
num = results['n_' + seq]
prob = results['p_' + seq]
return '%d (%.2f)' % (num, prob)
row = [i,
"%.1f" % (results['length'] / 60),
results['N'],
results['n_L'],
results['n_R'],
pformat('RL'),
pformat('LL'),
pformat('RR'),
pformat('LR'),
"[%.2f, %.2f]" % (results['p_L_lb'], results['p_L_ub']),
"%.2f" % results['best_p_L'],
pvalue_format(results['indep_pvalue']),
results['why']
]
rows.append(row)
r = Report()
attach_description(r, description)
# Sort by length
rows.sort(key=lambda x:-float(x[1]))
r.table('independence', rows, cols=cols_desc)
return r
def create_report(all_results):
cols_desc = [
"ID",
"Num. saccades",
"a",
"mu",
"Levy log.lik.",
"Levy Akaike w.",
"lambda",
"Exp. log.lik.",
"Exp. Akaike w",
"best model",
]
rows = []
for i, results in enumerate(all_results):
results = dict(results)
if results["lambda_w"] > results["mu_w"]:
best = "exponential"
else:
best = "levy"
row = [
i,
results["N"],
results["a"],
"%.2f" % results["mu"],
"%.2f" % results["mu_lik"],
"%.4f" % results["mu_w"],
"%.2f" % results["lambda"],
"%.2f" % results["lambda_lik"],
"%.4f" % results["lambda_w"],
best,
]
rows.append(row)
# sort by length
rows.sort(key=lambda x: -float(x[2]))
r = Report()
attach_description(r, description)
r.table("levy_vs_exp", rows, cols=cols_desc)
return r
def report_results_pairs(func, objspec1_name, objspec2_name, results):
reason2symbol = {}
def get_string_result(res):
if res is None:
s = 'ok'
elif isinstance(res, Skipped):
s = 'skipped'
reason = res.get_reason()
if not reason in reason2symbol:
reason2symbol[reason] = len(reason2symbol) + 1
s += '(%s)' % reason2symbol[reason]
elif isinstance(res, PartiallySkipped):
parts = res.get_skipped_parts()
s = 'no ' + ','.join(parts)
else:
print('how to interpret %s? ' % describe_value(res))
s = '?'
return s
r = Report()
if not results:
r.text('warning', 'no test objects defined')
return r
rows = sorted(set([a for a, _ in results]))
cols = sorted(set([b for _, b in results]))
data = [[None for a in range(len(cols))] for b in range(len(rows))]
# a nice bug: data = [[None * len(cols)] * len(rows)
for ((i, id_object1),
(j, id_object2)) in itertools.product(enumerate(rows),
enumerate(cols)):
res = results[(id_object1, id_object2)]
data[i][j] = get_string_result(res)
r.table('summary', rows=rows, data=data, cols=cols)
expl = ""
for reason, symbol in reason2symbol.items():
expl += '(%s): %s\n' % (symbol, reason)
r.text('notes', expl)
return r
def report_results_pairs(func, objspec1_name, objspec2_name, results):
reason2symbol = {}
def get_string_result(res):
if res is None:
s = 'ok'
elif isinstance(res, Skipped):
s = 'skipped'
reason = res.get_reason()
if not reason in reason2symbol:
reason2symbol[reason] = len(reason2symbol) + 1
s += '(%s)' % reason2symbol[reason]
elif isinstance(res, PartiallySkipped):
parts = res.get_skipped_parts()
s = 'no ' + ','.join(parts)
else:
print('how to interpret %s? ' % describe_value(res))
s = '?'
return s
r = Report()
if not results:
r.text('warning', 'no test objects defined')
return r
rows = sorted(set([a for a, _ in results]))
cols = sorted(set([b for _, b in results]))
data = [[None for a in range(len(cols))] for b in range(len(rows))]
# a nice bug: data = [[None * len(cols)] * len(rows)
for ((i, id_object1), (j, id_object2)) in itertools.product(enumerate(rows), enumerate(cols)):
res = results[(id_object1, id_object2)]
data[i][j] = get_string_result(res)
r.table('summary', rows=rows, data=data, cols=cols)
expl = ""
for reason, symbol in list(reason2symbol.items()):
expl += '(%s): %s\n' % (symbol, reason)
r.text('notes', expl)
return r
def run_report(stat):
# def write_report(result, metric):
report = Report('OnlinePlanningTest')
report.text('summary', 'Visualizing of online planning test')
labels = stat.labels
for key in labels.keys():
report.text(key, str(labels[key]))
images = {
'y_start': stat.y_start,
'y_goal': stat.y_goal,
'y_result': stat.y_result,
'y_pred': stat.y_goal_pred,
'y_found_pred': stat.y_found_pred
}
keys = ['y_start', 'y_goal', 'y_result', 'y_pred', 'y_found_pred']
# Plot images
f = report.figure(cols=len(images))
for key in keys:
with f.plot(key, caption=key) as pylab:
pylab.imshow(images[key].get_rgb(), interpolation='nearest')
data = []
for key in keys:
yr = images[key]
this_row = []
for key2 in keys:
yc = images[key2]
yr_yc = stat.metric_goal.distance(yr, yc)
this_row.append(yr_yc)
data.append(this_row)
report.table('table',
data=data,
cols=images.keys(),
rows=images.keys(),
caption='Distances')
return report
def run_report(stat):
# def write_report(result, metric):
report = Report('OnlinePlanningTest')
report.text('summary', 'Visualizing of online planning test')
labels = stat.labels
for key in labels.keys():
report.text(key, str(labels[key]))
images = {'y_start':stat.y_start,
'y_goal':stat.y_goal,
'y_result':stat.y_result,
'y_pred':stat.y_goal_pred,
'y_found_pred':stat.y_found_pred}
keys = ['y_start',
'y_goal',
'y_result',
'y_pred',
'y_found_pred']
# Plot images
f = report.figure(cols=len(images))
for key in keys:
with f.plot(key, caption=key) as pylab:
pylab.imshow(images[key].get_rgb(), interpolation='nearest')
data = []
for key in keys:
yr = images[key]
this_row = []
for key2 in keys:
yc = images[key2]
yr_yc = stat.metric_goal.distance(yr, yc)
this_row.append(yr_yc)
data.append(this_row)
report.table('table', data=data, cols=images.keys(), rows=images.keys(), caption='Distances')
return report
def table_by_rows(id_report, samples, rows_field, cols_fields, source_descs):
samples2 = StoreResultsDict(samples)
class Missing(dict):
def __missing__(self, key):
logger.warning("Description for %r missing." % key)
d = WithDescription(name=key, symbol="\\text{%s}" % key, desc=None)
self[key] = d
return d
source_descs = Missing(source_descs)
r = Report(id_report)
data_views = [DataView.from_string(x, source_descs) for x in cols_fields]
# data: list of list of list
rows_field, data, reduced, display = summarize_data(
samples2, rows_field, data_views)
rows = ["$%s$" % source_descs[x].get_symbol() for x in rows_field]
cols = ["$%s$" % x.get_symbol() for x in data_views]
r.table("table", data=display, cols=cols, rows=rows)
r.data("table_data",
data=reduced,
caption="Data without presentation applied.")
r.data("table_data_source",
data=data,
caption="Source data, before reduction.")
row_desc = "\n".join([
"- $%s$: %s" % (x.get_symbol(), x.get_desc())
for x in list(map(source_descs.__getitem__, rows_field))
])
col_desc = "\n".join(
["- $%s$: %s" % (x.get_symbol(), x.get_desc()) for x in data_views])
r.text("row_desc", rst_escape_slash(row_desc), mime=MIME_RST)
r.text("col_desc", rst_escape_slash(col_desc), mime=MIME_RST)
return r
def report_results_pairs_jobs(context, func, objspec1_name, objspec2_name,
jobs):
""" This version gets the jobs ID """
reason2symbol = {}
def get_string_result(res):
if res is None:
s = 'ok'
elif isinstance(res, Skipped):
s = 'skipped'
reason = res.get_reason()
if not reason in reason2symbol:
reason2symbol[reason] = len(reason2symbol) + 1
s += '(%s)' % reason2symbol[reason]
elif isinstance(res, PartiallySkipped):
parts = res.get_skipped_parts()
s = 'no ' + ','.join(parts)
else:
print('how to interpret %s? ' % describe_value(res))
s = '?'
return s
r = Report()
if not jobs:
r.text('warning', 'no test objects defined')
return r
rows = sorted(set([a for a, _ in jobs]))
cols = sorted(set([b for _, b in jobs]))
data = [[None for a in range(len(cols))] for b in range(len(rows))]
# a nice bug: data = [[None * len(cols)] * len(rows)
db = context.get_compmake_db()
comb = itertools.product(enumerate(rows), enumerate(cols))
for ((i, id_object1), (j, id_object2)) in comb:
job_id = jobs[(id_object1, id_object2)]
cache = get_job_cache(job_id, db)
if cache.state == Cache.DONE:
res = get_job_userobject(job_id, db)
s = get_string_result(res)
elif cache.state == Cache.FAILED:
s = 'FAIL'
elif cache.state == Cache.BLOCKED:
s = 'blocked'
# elif cache.state == Cache.IN_PROGRESS:
# s = '(in progress)'
elif cache.state == Cache.NOT_STARTED:
s = ' '
data[i][j] = s
r.table('summary', rows=rows, data=data, cols=cols)
expl = ""
for reason, symbol in reason2symbol.items():
expl += '(%s): %s\n' % (symbol, reason)
r.text('notes', expl)
return r
def report_results_pairs_jobs(context, func, objspec1_name, objspec2_name, jobs):
""" This version gets the jobs ID """
reason2symbol = {}
def get_string_result(res):
if res is None:
s = 'ok'
elif isinstance(res, Skipped):
s = 'skipped'
reason = res.get_reason()
if not reason in reason2symbol:
reason2symbol[reason] = len(reason2symbol) + 1
s += '(%s)' % reason2symbol[reason]
elif isinstance(res, PartiallySkipped):
parts = res.get_skipped_parts()
s = 'no ' + ','.join(parts)
else:
print('how to interpret %s? ' % describe_value(res))
s = '?'
return s
r = Report()
if not jobs:
r.text('warning', 'no test objects defined')
return r
rows = sorted(set([a for a, _ in jobs]))
cols = sorted(set([b for _, b in jobs]))
data = [[None for a in range(len(cols))] for b in range(len(rows))]
# a nice bug: data = [[None * len(cols)] * len(rows)
db = context.get_compmake_db()
comb = itertools.product(enumerate(rows), enumerate(cols))
for ((i, id_object1), (j, id_object2)) in comb:
job_id = jobs[(id_object1, id_object2)]
cache = get_job_cache(job_id, db)
if cache.state == Cache.DONE:
res = get_job_userobject(job_id, db)
s = get_string_result(res)
elif cache.state == Cache.FAILED:
s = 'FAIL'
elif cache.state == Cache.BLOCKED:
s = 'blocked'
# elif cache.state == Cache.IN_PROGRESS:
# s = '(in progress)'
elif cache.state == Cache.NOT_STARTED:
s = ' '
else:
s = '?'
data[i][j] = s
r.table('summary', rows=rows, data=data, cols=cols)
expl = ""
for reason, symbol in list(reason2symbol.items()):
expl += '(%s): %s\n' % (symbol, reason)
r.text('notes', expl)
return r