def __call__(self, curves):
"""Computes normalized dispersion across runs.
Args:
curves: A list of learning curves, each a 2D numpy array where curve[0, :]
is the timepoint variable and curve[1, :] is the dependent variable.
Returns:
Dispersion across runs, computed at each of the eval_points.
(Numpy array with length n_eval_points).
"""
utils.assert_non_empty(curves)
# perform preprocessing for across-runs metrics
eval_point_values = utils.across_runs_preprocess(
curves, self.eval_points, self.window_size, self.lowpass_thresh)
# compute dispersion across curves
result = self._dispersion_fn(eval_point_values)
if self.baseline == 'curve_range':
curve_ranges = utils.curve_range(curves)
result /= np.median(curve_ranges)
elif self.baseline:
result = result / self.baseline
return result
def __call__(self, curves):
"""Computes energy of the signal above a given frequency threshold.
Normalized by the total energy of the signal.
Args:
curves: A list of learning curves, each a 2D numpy array where curve[0, :]
is the timepoint variable and curve[1, :] is the dependent variable.
Returns:
Amount of energy above the given frequency threshold, normalized by the
total energy of the signal.
"""
utils.assert_non_empty(curves)
energies = []
for curve in curves:
data = curve[1, :]
power_spectrum = np.abs(np.fft.fft(data))**2
time_step = curve[0, 1] - curve[0, 0]
# TODO(scychan) above assumes equal spacing
freqs = np.fft.fftfreq(data.size, time_step)
energy_above_thresh = (
np.sum(power_spectrum[freqs > self.thresh]) /
np.sum(power_spectrum[freqs > 0]))
energies.append(energy_above_thresh)
return energies
def __call__(self, curves):
"""Computes median performance.
Args:
curves: A list of learning curves, each a 2D numpy array where curve[0, :]
is the timepoint variable and curve[1, :] is the dependent variable.
Returns:
Median performance, computed in a window at each eval_point for each run.
(Numpy array with size n_run x n_eval_points.)
"""
utils.assert_non_empty(curves)
# Determine eval_points and window_size, if needed.
eval_points = copy.deepcopy(self.eval_points)
window_size = copy.deepcopy(self.window_size)
if eval_points is None or window_size is None:
if window_size is None:
valid_eval_points = utils.get_all_valid_eval_points(curves, 1)
window_size = valid_eval_points.max() - valid_eval_points.min(
) + 1
if eval_points is None:
eval_points = utils.get_all_valid_eval_points(
curves, window_size)
curves = self._normalize(curves)
perf = utils.apply_window_fn(curves, eval_points, np.median,
window_size)
return perf
def __call__(self, rollout_sets):
"""Computes dispersion across rollouts.
Args:
rollout_sets: A list of rollout sets, with length n_rollout_sets.
Each element of the list corresponds to the performance values of one
set of rollouts that we will measure dispersion across (e.g. for a
single model checkpoint). It is a 2D numpy array where rollouts[0, :] is
just an index variable (e.g. range(0, n_rollouts)) and rollouts[1, :]
are the performances per rollout.
Returns:
Dispersion across rollouts, computed for each rollout set.
(1-D Numpy array with length = n_rollout_sets)
"""
utils.assert_non_empty(rollout_sets)
dispersions = []
for rollout_set in rollout_sets:
dispersion = self._dispersion_fn(rollout_set[1, :])
dispersions.append(dispersion)
dispersions = np.array(dispersions)
if self.baseline:
if self.baseline == 'median_perf':
divisor = utils.median_rollout_performance(rollout_sets)
else:
divisor = self.baseline
dispersions /= divisor
return dispersions
def __call__(self, rollout_sets):
"""Computes CVaR across rollouts of a fixed policy.
Args:
rollout_sets: A list of rollout sets, with length n_rollout_sets.
Each element of the list corresponds to the performance values of one
set of rollouts that we will measure dispersion across (e.g. for a
single model checkpoint). It is a 2D numpy array where rollouts[0, :] is
just an index variable (e.g. range(0, n_rollouts)) and rollouts[1, :]
are the performances per rollout.
Returns:
CVaR across rollouts, computed for each rollout set.
(1-D Numpy array with length = n_rollout_sets)
"""
utils.assert_non_empty(rollout_sets)
if self.baseline is not None:
if self.baseline == 'median_perf':
divisor = utils.median_rollout_performance(rollout_sets)
else:
divisor = self.baseline
rollout_sets = utils.divide_by_baseline(rollout_sets, divisor)
cvar_list = []
# Compute CVaR within each rollout set.
for rollout_set in rollout_sets:
dependent_var = rollout_set[1, :]
cvar = utils.compute_cvar(dependent_var, self.tail, self.alpha)
cvar_list.append(cvar)
return np.array(cvar_list)
def __call__(self, curves):
"""Computes CVaR for a list of curves.
Args:
curves: A list of learning curves, each a 2D numpy array where curve[0, :]
is the timepoint variable and curve[1, :] is the dependent variable.
Returns:
for self.target in ['diffs', 'raw', 'drawdown']:
A 1-D numpy array of CVaR values, one per curve in the input
(length = the number of curves in the input).
for self.target == 'across':
A 1-D numpy array of CVaR values, one per eval point
(length = number of eval points)
"""
utils.assert_non_empty(curves)
if self.baseline == 'curve_range':
curve_ranges = utils.curve_range(curves)
curves = utils.divide_by_baseline(curves, curve_ranges)
elif self.baseline:
curves = utils.divide_by_baseline(curves, self.baseline)
cvar_list = []
if self.target == 'across':
# Compute CVaR across curves (at each eval point)
eval_point_vals = utils.across_runs_preprocess(
curves, self.eval_points, self.window_size,
self.lowpass_thresh)
n_eval_points = eval_point_vals.shape[1]
for i_point in range(n_eval_points):
cvar = utils.compute_cvar(eval_point_vals[:, i_point],
self.tail, self.alpha)
cvar_list.append(cvar)
else:
# Compute CVaR within curves (one per curve).
for curve in curves:
dependent_var = curve[1, :]
if self.target == 'raw':
pass
elif self.target == 'diffs':
normalized_diffs = utils.differences([curve])[0]
dependent_var = normalized_diffs[1, :]
elif self.target == 'drawdown':
dependent_var = utils.compute_drawdown(dependent_var)
cvar = utils.compute_cvar(dependent_var, self.tail, self.alpha)
cvar_list.append(cvar)
return np.array(cvar_list)
def __call__(self, curves):
"""Computes dispersion within runs.
Args:
curves: A list of learning curves, each a 2D numpy array where curve[0, :]
is the timepoint variable and curve[1, :] is the dependent variable.
Returns:
Dispersion within runs, computed at each eval_point for each run.
(Numpy array with size n_run x n_eval_points.)
"""
utils.assert_non_empty(curves)
# Detrend by differencing.
if self.detrend:
diff_curves = utils.differences(curves)
else:
diff_curves = curves
dispersions = []
# Process each curve separately, because length of run may differ for each.
for curve, diff_curve in zip(curves, diff_curves):
eval_points = copy.deepcopy(self.eval_points)
window_size = copy.deepcopy(self.window_size)
# Determine eval_points and window_size, if needed (based on diff_curve).
if self.eval_points is None or self.window_size is None:
if self.window_size is None:
valid_eval_points = utils.get_all_valid_eval_points(
[diff_curve], 1)
window_size = valid_eval_points.max(
) - valid_eval_points.min() + 1
if self.eval_points is None:
eval_points = utils.get_all_valid_eval_points([diff_curve],
window_size)
# Compute dispersion for the curve.
diffcurve_dispers = utils.apply_window_fn([diff_curve],
eval_points,
self._dispersion_fn,
window_size)
if self.baseline == 'curve_range':
curve_range = utils.curve_range([curve])[0]
diffcurve_dispers = diffcurve_dispers / curve_range
elif self.baseline:
diffcurve_dispers /= self.baseline
dispersions.extend(diffcurve_dispers)
return np.array(dispersions)
def __call__(self, curves):
"""Compute maximum drawdown."""
utils.assert_non_empty(curves)
if self.baseline is not None:
curves = utils.subtract_baseline(curves, self.baseline)
if self.mean_normalize:
curves = utils.mean_normalization(curves)
mdd = np.empty(len(curves))
for i, curve in enumerate(curves):
dependent_vals = curve[1, :]
drawdown = utils.compute_drawdown(dependent_vals)
mdd[i] = np.max(drawdown)
return mdd