def _ravel_perf(perf_da):
validate_perf(perf_da)
function_evals = np.ravel(perf_da.values, order="C")
perf_ds = build_eval_ds(function_evals[:, None])
perf_da = xru.only_dataarray(perf_ds)
return perf_da
def do_baseline(args): # pragma: io
"""Alternate entry into the program without calling the actual main.
"""
# Load in the eval data and sanity check
perf_da, meta = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.EVAL_RESULTS)
perf_da = xru.only_dataarray(perf_da)
logger.info("Meta data from source file: %s" % str(meta["args"]))
baseline_ds = compute_baseline(perf_da)
# Now dump the results
XRSerializer.save_derived(baseline_ds, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.BASELINE)
def main():
"""See README for instructions on calling analysis.
"""
description = "Analyze results from aggregated studies"
args = parse_args(general_parser(description))
logger.setLevel(logging.INFO) # Note this is the module-wide logger
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
# Load in the eval data and sanity check
perf_da, meta = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.EVAL_RESULTS)
perf_da = xru.only_dataarray(perf_da)
logger.info("Meta data from source file: %s" % str(meta["args"]))
# Check if there is baselines file, other make one
if cc.BASELINE not in XRSerializer.get_derived_keys(args[CmdArgs.db_root], db=args[CmdArgs.db]):
warnings.warn("Baselines not found. Need to construct baseline.")
do_baseline(args)
# Load in baseline scores data and sanity check (including compatibility with eval data)
baseline_ds, meta_ref = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.BASELINE)
logger.info("baseline data from source ref file: %s" % str(meta_ref["args"]))
# Check test case signatures match between eval data and baseline data
sig_errs, signatures = analyze_signature_pair(meta["signature"], meta_ref["signature"])
logger.info("Signature errors:\n%s" % sig_errs.to_string())
# Do the actual computation
agg_result, summary = compute_aggregates(perf_da, baseline_ds)
final_score = summary[PERF_MED][{ITER: -1}]
logger.info("median score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
final_score = summary[PERF_MEAN][{ITER: -1}]
logger.info("mean score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
final_score = summary[NORMED_MEAN][{ITER: -1}]
logger.info("normed mean score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
# Now saving results
meta = {"args": serializable_dict(args), "signature": signatures}
XRSerializer.save_derived(agg_result, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.PERF_RESULTS)
XRSerializer.save_derived(summary, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.MEAN_SCORE)
logger.info("done")
def test_only_dataarray(var_, da):
assume(var_ not in da.dims)
ds = xr.Dataset({var_: da})
xru.only_dataarray(ds)
def compute_aggregates(perf_da, baseline_ds):
"""Aggregate function evaluations in the experiments to get performance summaries of each method.
Parameters
----------
perf_da : :class:`xarray:xarray.DataArray`
Aggregate experimental results with each function evaluation in the experiments. `all_perf` has dimensions
``(ITER, SUGGEST, TEST_CASE, METHOD, TRIAL)`` as is assumed to have no nan values.
baseline_ds : :class:`xarray:xarray.Dataset`
Dataset with baseline performance. It was variables ``(PERF_MED, PERF_MEAN, PERF_CLIP, PERF_BEST)`` with
dimensions ``(ITER, TEST_CASE)``, ``(ITER, TEST_CASE)``, ``(TEST_CASE,)``, and ``(TEST_CASE,)``, respectively.
`PERF_MED` is a baseline of performance based on random search when using medians to summarize performance.
Likewise, `PERF_MEAN` is for means. `PERF_CLIP` is an upperbound to clip poor performance when using the mean.
`PERF_BEST` is an estimate on the global minimum.
Returns
-------
agg_result : :class:`xarray:xarray.Dataset`
Dataset with summary of performance for each method and test case combination. Contains variables:
``(PERF_MED, LB_MED, UB_MED, NORMED_MED, PERF_MEAN, LB_MEAN, UB_MEAN, NORMED_MEAN)``
each with dimensions ``(ITER, METHOD, TEST_CASE)``. `PERF_MED` is a median summary of performance with `LB_MED`
and `UB_MED` as error bars. `NORMED_MED` is a rescaled `PERF_MED` so we expect the optimal performance is 0,
and random search gives 1 at all `ITER`. Likewise, `PERF_MEAN`, `LB_MEAN`, `UB_MEAN`, `NORMED_MEAN` are for
mean performance.
summary : :class:`xarray:xarray.Dataset`
Dataset with overall summary of performance of each method. Contains variables
``(PERF_MED, LB_MED, UB_MED, PERF_MEAN, LB_MEAN, UB_MEAN)``
each with dimensions ``(ITER, METHOD)``.
"""
validate_agg_perf(perf_da, min_trial=1)
assert isinstance(baseline_ds, xr.Dataset)
assert tuple(baseline_ds[PERF_BEST].dims) == (TEST_CASE,)
assert tuple(baseline_ds[PERF_CLIP].dims) == (TEST_CASE,)
assert tuple(baseline_ds[PERF_MED].dims) == (ITER, TEST_CASE)
assert tuple(baseline_ds[PERF_MEAN].dims) == (ITER, TEST_CASE)
assert xru.coord_compat((perf_da, baseline_ds), (ITER, TEST_CASE))
assert not any(np.any(np.isnan(baseline_ds[kk].values)) for kk in baseline_ds)
# Now actually get the aggregate performance numbers per test case
agg_result = xru.ds_like(
perf_da,
(PERF_MED, LB_MED, UB_MED, NORMED_MED, PERF_MEAN, LB_MEAN, UB_MEAN, NORMED_MEAN),
(ITER, METHOD, TEST_CASE),
)
baseline_mean_da = xru.only_dataarray(xru.ds_like(perf_da, ["ref"], (ITER, TEST_CASE)))
# Using values here since just clearer to get raw items than xr object for func_name
for func_name in perf_da.coords[TEST_CASE].values:
rand_perf_med = baseline_ds[PERF_MED].sel({TEST_CASE: func_name}, drop=True).values
rand_perf_mean = baseline_ds[PERF_MEAN].sel({TEST_CASE: func_name}, drop=True).values
best_opt = baseline_ds[PERF_BEST].sel({TEST_CASE: func_name}, drop=True).values
base_clip_val = baseline_ds[PERF_CLIP].sel({TEST_CASE: func_name}, drop=True).values
assert np.all(np.diff(rand_perf_med) <= 0), "Baseline should be decreasing with iteration"
assert np.all(np.diff(rand_perf_mean) <= 0), "Baseline should be decreasing with iteration"
assert np.all(rand_perf_med > best_opt)
assert np.all(rand_perf_mean > best_opt)
assert np.all(rand_perf_mean <= base_clip_val)
baseline_mean_da.loc[{TEST_CASE: func_name}] = linear_rescale(
rand_perf_mean, best_opt, base_clip_val, 0.0, 1.0, enforce_bounds=False
)
for method_name in perf_da.coords[METHOD].values:
# Take the minimum over all suggestion at given iter + sanity check perf_da
curr_da = perf_da.sel({METHOD: method_name, TEST_CASE: func_name}, drop=True).min(dim=SUGGEST)
assert curr_da.dims == (ITER, TRIAL)
# Want to evaluate minimum so far during optimization
perf_array = np.minimum.accumulate(curr_da.values, axis=0)
# Compute median perf and CI on it
med_perf, LB, UB = qt.quantile_and_CI(perf_array, EVAL_Q, alpha=ALPHA)
assert med_perf.shape == rand_perf_med.shape
agg_result[PERF_MED].loc[{TEST_CASE: func_name, METHOD: method_name}] = med_perf
agg_result[LB_MED].loc[{TEST_CASE: func_name, METHOD: method_name}] = LB
agg_result[UB_MED].loc[{TEST_CASE: func_name, METHOD: method_name}] = UB
# Now store normed version, which is better for aggregation
normed = linear_rescale(med_perf, best_opt, rand_perf_med, 0.0, 1.0, enforce_bounds=False)
agg_result[NORMED_MED].loc[{TEST_CASE: func_name, METHOD: method_name}] = normed
# Compute mean perf and CI on it
perf_array = np.minimum(base_clip_val, perf_array)
mean_perf = np.mean(perf_array, axis=1)
assert mean_perf.shape == rand_perf_mean.shape
EB = t_EB(perf_array, alpha=ALPHA, axis=1)
assert EB.shape == rand_perf_mean.shape
agg_result[PERF_MEAN].loc[{TEST_CASE: func_name, METHOD: method_name}] = mean_perf
agg_result[LB_MEAN].loc[{TEST_CASE: func_name, METHOD: method_name}] = mean_perf - EB
agg_result[UB_MEAN].loc[{TEST_CASE: func_name, METHOD: method_name}] = mean_perf + EB
# Now store normed version, which is better for aggregation
normed = linear_rescale(mean_perf, best_opt, base_clip_val, 0.0, 1.0, enforce_bounds=False)
agg_result[NORMED_MEAN].loc[{TEST_CASE: func_name, METHOD: method_name}] = normed
assert not any(np.any(np.isnan(agg_result[kk].values)) for kk in agg_result)
# Compute summary score over all test cases, summarize performance of each method
summary = xru.ds_like(
perf_da,
(PERF_MED, LB_MED, UB_MED, PERF_MEAN, LB_MEAN, UB_MEAN, NORMED_MEAN, LB_NORMED_MEAN, UB_NORMED_MEAN),
(ITER, METHOD),
)
summary[PERF_MED], summary[LB_MED], summary[UB_MED] = xr.apply_ufunc(
qt.quantile_and_CI,
agg_result[NORMED_MED],
input_core_dims=[[TEST_CASE]],
kwargs={"q": EVAL_Q, "alpha": ALPHA},
output_core_dims=[[], [], []],
)
summary[PERF_MEAN] = agg_result[NORMED_MEAN].mean(dim=TEST_CASE)
EB = xr.apply_ufunc(t_EB, agg_result[NORMED_MEAN], input_core_dims=[[TEST_CASE]])
summary[LB_MEAN] = summary[PERF_MEAN] - EB
summary[UB_MEAN] = summary[PERF_MEAN] + EB
normalizer = baseline_mean_da.mean(dim=TEST_CASE)
summary[NORMED_MEAN] = summary[PERF_MEAN] / normalizer
summary[LB_NORMED_MEAN] = summary[LB_MEAN] / normalizer
summary[UB_NORMED_MEAN] = summary[UB_MEAN] / normalizer
assert all(tuple(summary[kk].dims) == (ITER, METHOD) for kk in summary)
return agg_result, summary
def load_experiments(uuid_list, db_root, dbid): # pragma: io
"""Generator to load the results of the experiments.
Parameters
----------
uuid_list : list(uuid.UUID)
List of UUIDs corresponding to experiments to load.
db_root : str
Root location for data store as requested by the serializer used.
dbid : str
Name of the data store as requested by the serializer used.
Yields
------
meta_data : (str, str, str)
The `meta_data` contains a `tuple` of `str` with ``test_case, optimizer, uuid``.
data : (:class:`xarray:xarray.DataArray`, :class:`xarray:xarray.Dataset`, list(float))
The `data` contains a tuple of ``(perf_da, time_ds, sig)``. The `perf_da` is an
:class:`xarray:xarray.DataArray` containing the evaluation results with dimensions ``(ITER, SUGGEST)``. The
`time_ds` is an :class:`xarray:xarray.Dataset` containing the timing results of the form accepted by
`summarize_time`. The coordinates must be compatible with `perf_da`. Finally, `sig` contains the `test_case`
signature and must be `list(float)`.
"""
uuids_seen = set()
for uuid_ in uuid_list:
logger.info(uuid_.hex)
# Load perf and timing data
perf_da, meta = XRSerializer.load(db_root,
db=dbid,
key=cc.EVAL,
uuid_=uuid_)
perf_da = xru.only_dataarray(perf_da)
time_ds, meta_t = XRSerializer.load(db_root,
db=dbid,
key=cc.TIME,
uuid_=uuid_)
assert meta == meta_t, "meta data should between time and eval files"
# Get signature to pass out as well
_, sig = meta["signature"]
logger.info(meta)
logger.info(sig)
# Build the new indices for combined data, this could be put in function for easier testing
eval_args = unserializable_dict(meta["args"]) # Unpack meta-data
test_case = str_join_safe(
ARG_DELIM, (eval_args[CmdArgs.classifier], eval_args[CmdArgs.data],
eval_args[CmdArgs.metric]))
optimizer = str_join_safe(
ARG_DELIM, (eval_args[CmdArgs.optimizer],
eval_args[CmdArgs.opt_rev], eval_args[CmdArgs.rev]))
args_uuid = eval_args[CmdArgs.uuid]
# Check UUID sanity
assert isinstance(args_uuid, str)
assert args_uuid == uuid_.hex, "UUID meta-data does not match filename"
assert args_uuid not in uuids_seen, "uuids being reused between studies"
uuids_seen.add(args_uuid)
# Return key -> data so this generator can be iterated over in dict like manner
meta_data = (test_case, optimizer, args_uuid)
data = (perf_da, time_ds, sig)
yield meta_data, data