def __call__(self, curves):
"""Computes CVaR for a list of curves.
Args:
curves: A list of learning curves, each a 2D numpy array where curve[0, :]
is the timepoint variable and curve[1, :] is the dependent variable.
Returns:
for self.target in ['diffs', 'raw', 'drawdown']:
A 1-D numpy array of CVaR values, one per curve in the input
(length = the number of curves in the input).
for self.target == 'across':
A 1-D numpy array of CVaR values, one per eval point
(length = number of eval points)
"""
utils.assert_non_empty(curves)
if self.baseline == 'curve_range':
curve_ranges = utils.curve_range(curves)
curves = utils.divide_by_baseline(curves, curve_ranges)
elif self.baseline:
curves = utils.divide_by_baseline(curves, self.baseline)
cvar_list = []
if self.target == 'across':
# Compute CVaR across curves (at each eval point)
eval_point_vals = utils.across_runs_preprocess(
curves, self.eval_points, self.window_size,
self.lowpass_thresh)
n_eval_points = eval_point_vals.shape[1]
for i_point in range(n_eval_points):
cvar = utils.compute_cvar(eval_point_vals[:, i_point],
self.tail, self.alpha)
cvar_list.append(cvar)
else:
# Compute CVaR within curves (one per curve).
for curve in curves:
dependent_var = curve[1, :]
if self.target == 'raw':
pass
elif self.target == 'diffs':
normalized_diffs = utils.differences([curve])[0]
dependent_var = normalized_diffs[1, :]
elif self.target == 'drawdown':
dependent_var = utils.compute_drawdown(dependent_var)
cvar = utils.compute_cvar(dependent_var, self.tail, self.alpha)
cvar_list.append(cvar)
return np.array(cvar_list)
def __call__(self, rollout_sets):
"""Computes CVaR across rollouts of a fixed policy.
Args:
rollout_sets: A list of rollout sets, with length n_rollout_sets.
Each element of the list corresponds to the performance values of one
set of rollouts that we will measure dispersion across (e.g. for a
single model checkpoint). It is a 2D numpy array where rollouts[0, :] is
just an index variable (e.g. range(0, n_rollouts)) and rollouts[1, :]
are the performances per rollout.
Returns:
CVaR across rollouts, computed for each rollout set.
(1-D Numpy array with length = n_rollout_sets)
"""
utils.assert_non_empty(rollout_sets)
if self.baseline is not None:
if self.baseline == 'median_perf':
divisor = utils.median_rollout_performance(rollout_sets)
else:
divisor = self.baseline
rollout_sets = utils.divide_by_baseline(rollout_sets, divisor)
cvar_list = []
# Compute CVaR within each rollout set.
for rollout_set in rollout_sets:
dependent_var = rollout_set[1, :]
cvar = utils.compute_cvar(dependent_var, self.tail, self.alpha)
cvar_list.append(cvar)
return np.array(cvar_list)
def testDivideByBaselineSingle(self):
curves = [
np.array([[0, 1, 2], [3, 4., 5.]]),
np.array([[-5, -7, -10], [2.4, 3.6, 0.12]])
]
result = utils.divide_by_baseline(curves, baselines=2)
expected = [
np.array([[0, 1, 2], [1.5, 2, 2.5]]),
np.array([[-5, -7, -10], [1.2, 1.8, 0.06]])
]
np.testing.assert_allclose(result, expected)