def do_baseline(args): # pragma: io
"""Alternate entry into the program without calling the actual main.
"""
# Load in the eval data and sanity check
perf_da, meta = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.EVAL_RESULTS)
perf_da = xru.only_dataarray(perf_da)
logger.info("Meta data from source file: %s" % str(meta["args"]))
baseline_ds = compute_baseline(perf_da)
# Now dump the results
XRSerializer.save_derived(baseline_ds, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.BASELINE)
def main():
"""See README for instructions on calling analysis.
"""
description = "Analyze results from aggregated studies"
args = parse_args(general_parser(description))
logger.setLevel(logging.INFO) # Note this is the module-wide logger
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
# Load in the eval data and sanity check
perf_da, meta = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.EVAL_RESULTS)
perf_da = xru.only_dataarray(perf_da)
logger.info("Meta data from source file: %s" % str(meta["args"]))
# Check if there is baselines file, other make one
if cc.BASELINE not in XRSerializer.get_derived_keys(args[CmdArgs.db_root], db=args[CmdArgs.db]):
warnings.warn("Baselines not found. Need to construct baseline.")
do_baseline(args)
# Load in baseline scores data and sanity check (including compatibility with eval data)
baseline_ds, meta_ref = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.BASELINE)
logger.info("baseline data from source ref file: %s" % str(meta_ref["args"]))
# Check test case signatures match between eval data and baseline data
sig_errs, signatures = analyze_signature_pair(meta["signature"], meta_ref["signature"])
logger.info("Signature errors:\n%s" % sig_errs.to_string())
# Do the actual computation
agg_result, summary = compute_aggregates(perf_da, baseline_ds)
final_score = summary[PERF_MED][{ITER: -1}]
logger.info("median score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
final_score = summary[PERF_MEAN][{ITER: -1}]
logger.info("mean score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
final_score = summary[NORMED_MEAN][{ITER: -1}]
logger.info("normed mean score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
# Now saving results
meta = {"args": serializable_dict(args), "signature": signatures}
XRSerializer.save_derived(agg_result, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.PERF_RESULTS)
XRSerializer.save_derived(summary, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.MEAN_SCORE)
logger.info("done")
def main():
"""See README for instructions on calling aggregate.
"""
description = "Aggregate study results across functions and optimizers"
args = parse_args(agg_parser(description))
logger.setLevel(logging.INFO) # Note this is the module-wide logger
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
# Always sort after listdir
uuid_list = XRSerializer.get_uuids(args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=cc.EVAL)
uuid_list_ = XRSerializer.get_uuids(args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=cc.TIME)
assert uuid_list == uuid_list_, "UUID list does not match between time and eval results"
# Get iterator of all experiment data dumps, load in and process, and concat
data_G = load_experiments(uuid_list, args[CmdArgs.db_root],
args[CmdArgs.db])
all_perf, all_time, all_sigs = concat_experiments(
data_G, ravel=args[CmdArgs.ravel])
# Check the concat signatures make are coherent
sig_errs, signatures_median = analyze_signatures(all_sigs)
logger.info("Signature errors:\n%s" % sig_errs.to_string())
# Dump and save it all out
logger.info("saving")
meta = {"args": serializable_dict(args), "signature": signatures_median}
all_perf = all_perf.to_dataset(name="results")
XRSerializer.save_derived(all_perf,
meta,
args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=EVAL_RESULTS)
XRSerializer.save_derived(all_time,
meta,
args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=TIME_RESULTS)
logger.info("done")
def do_baseline(args): # pragma: io
"""Alternate entry into the program without calling the actual main.
"""
# Load in the eval data and sanity check
perf_ds, meta = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.EVAL_RESULTS)
logger.info("Meta data from source file: %s" % str(meta["args"]))
D = OrderedDict()
for kk in perf_ds:
perf_da = perf_ds[kk]
D[(kk,)] = compute_baseline(perf_da)
baseline_ds = ds_concat(D, dims=(cc.OBJECTIVE,))
# Keep in same order for cleanliness
baseline_ds = baseline_ds.sel({cc.OBJECTIVE: list(perf_ds)})
assert list(perf_ds) == baseline_ds.coords[cc.OBJECTIVE].values.tolist()
# Could optionally remove this once we think things have enough tests
for kk in D:
assert baseline_ds.sel({cc.OBJECTIVE: kk[0]}, drop=True).identical(D[kk])
# Now dump the results
XRSerializer.save_derived(baseline_ds, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.BASELINE)