def test_coord_compat_false(ds):
all_dims = [ds[kk].dims for kk in ds]
common_dims = sorted(intersect_seq(all_dims))
da_seq = [ds[kk] for kk in ds]
assume(len(da_seq) > 0)
assume(len(da_seq[0].dims) > 0)
da = da_seq[0]
kk = da.dims[0]
da_seq[0] = da.assign_coords(**{kk: range(da.sizes[kk])})
xru.coord_compat(da_seq, common_dims)
def test_coord_compat(ds):
all_dims = [ds[kk].dims for kk in ds]
common_dims = sorted(intersect_seq(all_dims))
da_seq = [ds[kk] for kk in ds]
compat = xru.coord_compat(da_seq, common_dims)
assert compat
def compute_aggregates(perf_da, baseline_ds):
"""Aggregate function evaluations in the experiments to get performance summaries of each method.
Parameters
----------
perf_da : :class:`xarray:xarray.DataArray`
Aggregate experimental results with each function evaluation in the experiments. `all_perf` has dimensions
``(ITER, SUGGEST, TEST_CASE, METHOD, TRIAL)`` as is assumed to have no nan values.
baseline_ds : :class:`xarray:xarray.Dataset`
Dataset with baseline performance. It was variables ``(PERF_MED, PERF_MEAN, PERF_CLIP, PERF_BEST)`` with
dimensions ``(ITER, TEST_CASE)``, ``(ITER, TEST_CASE)``, ``(TEST_CASE,)``, and ``(TEST_CASE,)``, respectively.
`PERF_MED` is a baseline of performance based on random search when using medians to summarize performance.
Likewise, `PERF_MEAN` is for means. `PERF_CLIP` is an upperbound to clip poor performance when using the mean.
`PERF_BEST` is an estimate on the global minimum.
Returns
-------
agg_result : :class:`xarray:xarray.Dataset`
Dataset with summary of performance for each method and test case combination. Contains variables:
``(PERF_MED, LB_MED, UB_MED, NORMED_MED, PERF_MEAN, LB_MEAN, UB_MEAN, NORMED_MEAN)``
each with dimensions ``(ITER, METHOD, TEST_CASE)``. `PERF_MED` is a median summary of performance with `LB_MED`
and `UB_MED` as error bars. `NORMED_MED` is a rescaled `PERF_MED` so we expect the optimal performance is 0,
and random search gives 1 at all `ITER`. Likewise, `PERF_MEAN`, `LB_MEAN`, `UB_MEAN`, `NORMED_MEAN` are for
mean performance.
summary : :class:`xarray:xarray.Dataset`
Dataset with overall summary of performance of each method. Contains variables
``(PERF_MED, LB_MED, UB_MED, PERF_MEAN, LB_MEAN, UB_MEAN)``
each with dimensions ``(ITER, METHOD)``.
"""
validate_agg_perf(perf_da, min_trial=1)
assert isinstance(baseline_ds, xr.Dataset)
assert tuple(baseline_ds[PERF_BEST].dims) == (TEST_CASE,)
assert tuple(baseline_ds[PERF_CLIP].dims) == (TEST_CASE,)
assert tuple(baseline_ds[PERF_MED].dims) == (ITER, TEST_CASE)
assert tuple(baseline_ds[PERF_MEAN].dims) == (ITER, TEST_CASE)
assert xru.coord_compat((perf_da, baseline_ds), (ITER, TEST_CASE))
assert not any(np.any(np.isnan(baseline_ds[kk].values)) for kk in baseline_ds)
# Now actually get the aggregate performance numbers per test case
agg_result = xru.ds_like(
perf_da,
(PERF_MED, LB_MED, UB_MED, NORMED_MED, PERF_MEAN, LB_MEAN, UB_MEAN, NORMED_MEAN),
(ITER, METHOD, TEST_CASE),
)
baseline_mean_da = xru.only_dataarray(xru.ds_like(perf_da, ["ref"], (ITER, TEST_CASE)))
# Using values here since just clearer to get raw items than xr object for func_name
for func_name in perf_da.coords[TEST_CASE].values:
rand_perf_med = baseline_ds[PERF_MED].sel({TEST_CASE: func_name}, drop=True).values
rand_perf_mean = baseline_ds[PERF_MEAN].sel({TEST_CASE: func_name}, drop=True).values
best_opt = baseline_ds[PERF_BEST].sel({TEST_CASE: func_name}, drop=True).values
base_clip_val = baseline_ds[PERF_CLIP].sel({TEST_CASE: func_name}, drop=True).values
assert np.all(np.diff(rand_perf_med) <= 0), "Baseline should be decreasing with iteration"
assert np.all(np.diff(rand_perf_mean) <= 0), "Baseline should be decreasing with iteration"
assert np.all(rand_perf_med > best_opt)
assert np.all(rand_perf_mean > best_opt)
assert np.all(rand_perf_mean <= base_clip_val)
baseline_mean_da.loc[{TEST_CASE: func_name}] = linear_rescale(
rand_perf_mean, best_opt, base_clip_val, 0.0, 1.0, enforce_bounds=False
)
for method_name in perf_da.coords[METHOD].values:
# Take the minimum over all suggestion at given iter + sanity check perf_da
curr_da = perf_da.sel({METHOD: method_name, TEST_CASE: func_name}, drop=True).min(dim=SUGGEST)
assert curr_da.dims == (ITER, TRIAL)
# Want to evaluate minimum so far during optimization
perf_array = np.minimum.accumulate(curr_da.values, axis=0)
# Compute median perf and CI on it
med_perf, LB, UB = qt.quantile_and_CI(perf_array, EVAL_Q, alpha=ALPHA)
assert med_perf.shape == rand_perf_med.shape
agg_result[PERF_MED].loc[{TEST_CASE: func_name, METHOD: method_name}] = med_perf
agg_result[LB_MED].loc[{TEST_CASE: func_name, METHOD: method_name}] = LB
agg_result[UB_MED].loc[{TEST_CASE: func_name, METHOD: method_name}] = UB
# Now store normed version, which is better for aggregation
normed = linear_rescale(med_perf, best_opt, rand_perf_med, 0.0, 1.0, enforce_bounds=False)
agg_result[NORMED_MED].loc[{TEST_CASE: func_name, METHOD: method_name}] = normed
# Compute mean perf and CI on it
perf_array = np.minimum(base_clip_val, perf_array)
mean_perf = np.mean(perf_array, axis=1)
assert mean_perf.shape == rand_perf_mean.shape
EB = t_EB(perf_array, alpha=ALPHA, axis=1)
assert EB.shape == rand_perf_mean.shape
agg_result[PERF_MEAN].loc[{TEST_CASE: func_name, METHOD: method_name}] = mean_perf
agg_result[LB_MEAN].loc[{TEST_CASE: func_name, METHOD: method_name}] = mean_perf - EB
agg_result[UB_MEAN].loc[{TEST_CASE: func_name, METHOD: method_name}] = mean_perf + EB
# Now store normed version, which is better for aggregation
normed = linear_rescale(mean_perf, best_opt, base_clip_val, 0.0, 1.0, enforce_bounds=False)
agg_result[NORMED_MEAN].loc[{TEST_CASE: func_name, METHOD: method_name}] = normed
assert not any(np.any(np.isnan(agg_result[kk].values)) for kk in agg_result)
# Compute summary score over all test cases, summarize performance of each method
summary = xru.ds_like(
perf_da,
(PERF_MED, LB_MED, UB_MED, PERF_MEAN, LB_MEAN, UB_MEAN, NORMED_MEAN, LB_NORMED_MEAN, UB_NORMED_MEAN),
(ITER, METHOD),
)
summary[PERF_MED], summary[LB_MED], summary[UB_MED] = xr.apply_ufunc(
qt.quantile_and_CI,
agg_result[NORMED_MED],
input_core_dims=[[TEST_CASE]],
kwargs={"q": EVAL_Q, "alpha": ALPHA},
output_core_dims=[[], [], []],
)
summary[PERF_MEAN] = agg_result[NORMED_MEAN].mean(dim=TEST_CASE)
EB = xr.apply_ufunc(t_EB, agg_result[NORMED_MEAN], input_core_dims=[[TEST_CASE]])
summary[LB_MEAN] = summary[PERF_MEAN] - EB
summary[UB_MEAN] = summary[PERF_MEAN] + EB
normalizer = baseline_mean_da.mean(dim=TEST_CASE)
summary[NORMED_MEAN] = summary[PERF_MEAN] / normalizer
summary[LB_NORMED_MEAN] = summary[LB_MEAN] / normalizer
summary[UB_NORMED_MEAN] = summary[UB_MEAN] / normalizer
assert all(tuple(summary[kk].dims) == (ITER, METHOD) for kk in summary)
return agg_result, summary
def concat_experiments(all_experiments, ravel=False):
"""Aggregate the Datasets from a series of experiments into combined Dataset.
Parameters
----------
all_experiments : typing.Iterable
Iterable (possible from a generator) with the Datasets from each experiment. Each item in `all_experiments` is
a pair containing ``(meta_data, data)``. See `load_experiments` for details on these variables,
ravel : bool
If true, ravel all studies to store batch suggestions as if they were serial.
Returns
-------
all_perf : :class:`xarray:xarray.Dataset`
DataArray containing all of the `perf_da` from the experiments. The meta-data from the experiments are included
as extra dimensions. `all_perf` has dimensions ``(ITER, SUGGEST, TEST_CASE, METHOD, TRIAL)``. To convert the
`uuid` to a trial, there must be an equal number of repetition in the experiments for each `TEST_CASE`,
`METHOD` combination. Likewise, all of the experiments need an equal number of `ITER` and `SUGGEST`. If `ravel`
is true, then the `SUGGEST` is singleton.
all_time : :class:`xarray:xarray.Dataset`
Dataset containing all of the `time_ds` from the experiments. The new dimensions are
``(ITER, TEST_CASE, METHOD, TRIAL)``. It has the same variables as `time_ds`.
all_suggest : :class:`xarray:xarray.Dataset`
DataArray containing all of the `suggest_ds` from the experiments. It has dimensions
``(ITER, SUGGEST, TEST_CASE, METHOD, TRIAL)``.
all_sigs : dict(str, list(list(float)))
Aggregate of all experiment signatures.
"""
all_perf = {}
all_time = {}
all_suggest = {}
all_sigs = {}
trial_counter = Counter()
for (test_case, optimizer, uuid), (perf_ds, time_ds, suggest_ds, sig) in all_experiments:
if ravel:
raise NotImplementedError("ravel is deprecated. Just reshape in analysis steps instead.")
case_key = (test_case, optimizer, trial_counter[(test_case, optimizer)])
trial_counter[(test_case, optimizer)] += 1
# Process perf data
assert all(perf_ds[kk].dims == (ITER, SUGGEST) for kk in perf_ds)
all_perf[case_key] = perf_ds
# Process time data
all_time[case_key] = summarize_time(time_ds)
# Process suggestion data
all_suggest_curr = all_suggest.setdefault(test_case, {})
all_suggest_curr[case_key] = suggest_ds
# Handle the signatures
all_sigs.setdefault(test_case, []).append(sig)
assert min(trial_counter.values()) == max(trial_counter.values()), "Uneven number of trials per test case"
# Now need to concat dict of datasets into single dataset
all_perf = xru.ds_concat(all_perf, dims=(TEST_CASE, METHOD, TRIAL))
assert all(all_perf[kk].dims == (ITER, SUGGEST, TEST_CASE, METHOD, TRIAL) for kk in all_perf)
assert not any(
np.any(np.isnan(all_perf[kk].values)) for kk in all_perf
), "Missing combinations of method and test case"
all_time = xru.ds_concat(all_time, dims=(TEST_CASE, METHOD, TRIAL))
assert all(all_time[kk].dims == (ITER, TEST_CASE, METHOD, TRIAL) for kk in all_time)
assert not any(np.any(np.isnan(all_time[kk].values)) for kk in all_time)
assert xru.coord_compat((all_perf, all_time), (ITER, TEST_CASE, METHOD, TRIAL))
for test_case in all_suggest:
all_suggest[test_case] = xru.ds_concat(all_suggest[test_case], dims=(TEST_CASE, METHOD, TRIAL))
assert all(
all_suggest[test_case][kk].dims == (ITER, SUGGEST, TEST_CASE, METHOD, TRIAL)
for kk in all_suggest[test_case]
)
assert not any(np.any(np.isnan(all_suggest[test_case][kk].values)) for kk in all_suggest[test_case])
assert xru.coord_compat((all_perf, all_suggest[test_case]), (ITER, METHOD, TRIAL))
assert all_suggest[test_case].coords[TEST_CASE].shape == (1,), "test case should be singleton"
return all_perf, all_time, all_suggest, all_sigs
def concat_experiments(all_experiments, ravel=False):
"""Aggregate the Datasets from a series of experiments into combined Dataset.
Parameters
----------
all_experiments : typing.Iterable
Iterable (possible from a generator) with the Datasets from each experiment. Each item in `all_experiments` is
a pair containing ``(meta_data, data)``. The `meta_data` contains a `tuple` of `str` with
``test_case, optimizer, uuid``. The `data` contains a tuple of ``(perf_da, time_ds, sig)``. The `perf_da` is an
:class:`xarray:xarray.DataArray` containing the evaluation results with dimensions ``(ITER, SUGGEST)``. The
`time_ds` is an :class:`xarray:xarray.Dataset` containing the timing results of the form accepted by
`summarize_time`. The coordinates must be compatible with `perf_da`. Finally, `sig` contains the `test_case`
signature and must be `list(float)`.
ravel : bool
If true, ravel all studies to store batch suggestions as if they were serial.
Returns
-------
all_perf : :class:`xarray:xarray.DataArray`
DataArray containing all of the `perf_da` from the experiments. The meta-data from the experiments are included
as extra dimensions. `all_perf` has dimensions ``(ITER, SUGGEST, TEST_CASE, METHOD, TRIAL)``. To convert the
`uuid` to a trial, there must be an equal number of repetition in the experiments for each `TEST_CASE`,
`METHOD` combination. Likewise, all of the experiments need an equal number of `ITER` and `SUGGEST`. If `ravel`
is true, then the `SUGGEST` is singleton.
all_time : :class:`xarray:xarray.Dataset`
Dataset containing all of the `time_ds` from the experiments. The new dimensions are
``(ITER, TEST_CASE, METHOD, TRIAL)``. It has the same variables as `time_ds`.
all_sigs : dict(str, list(list(float)))
Aggregate of all experiment signatures.
"""
all_perf = {}
all_time = {}
all_sigs = {}
trial_counter = Counter()
for (test_case, optimizer, uuid), (perf_da, time_ds,
sig) in all_experiments:
if ravel:
n_suggest = perf_da.sizes[SUGGEST]
perf_da = _ravel_perf(perf_da)
time_ds = _ravel_time(time_ds)
optimizer = str_join_safe(ARG_DELIM,
(optimizer, "p%d" % n_suggest),
append=True)
case_key = (test_case, optimizer, trial_counter[(test_case,
optimizer)])
trial_counter[(test_case, optimizer)] += 1
# Process perf data
assert perf_da.dims == (ITER, SUGGEST)
all_perf[case_key] = perf_da
# Process time data
all_time[case_key] = summarize_time(time_ds)
# Handle the signatures
all_sigs.setdefault(test_case, []).append(sig)
assert min(trial_counter.values()) == max(
trial_counter.values()), "Uneven number of trials per test case"
# Now need to concat dict of datasets into single dataset
all_perf = xru.da_concat(all_perf, dims=(TEST_CASE, METHOD, TRIAL))
assert all_perf.dims == (ITER, SUGGEST, TEST_CASE, METHOD, TRIAL)
assert not np.any(np.isnan(
all_perf.values)), "Missing combinations of method and test case"
all_time = xru.ds_concat(all_time, dims=(TEST_CASE, METHOD, TRIAL))
assert all(all_time[kk].dims == (ITER, TEST_CASE, METHOD, TRIAL)
for kk in all_time)
assert not any(np.any(np.isnan(all_time[kk].values)) for kk in all_time)
assert xru.coord_compat((all_perf, all_time),
(ITER, TEST_CASE, METHOD, TRIAL))
return all_perf, all_time, all_sigs