def test_get_metric_params(self):
metric_instances = [
metrics_online.StddevAcrossRuns(eval_points=[3, 2, 1],
baseline=-2),
metrics_online.LowerCVaROnDiffs(alpha=0.77)
]
metric_params = eval_metrics.get_metric_params(metric_instances)
self.assertCountEqual(metric_params.keys(),
['StddevAcrossRuns', 'LowerCVaROnDiffs'])
self.assertEqual(
metric_params['StddevAcrossRuns'], {
'eval_points': [3, 2, 1],
'baseline': -2,
'lowpass_thresh': None,
'window_size': None,
})
self.assertEqual(
metric_params['LowerCVaROnDiffs'], {
'target': 'diffs',
'tail': 'lower',
'alpha': 0.77,
'baseline': None,
'eval_points': None,
'window_size': None,
'lowpass_thresh': None,
})
def test_compute_metrics(self):
curves = [
np.array([[-1, 0, 1], [1., 1., 1.]]),
np.array([[-1, 0, 1, 2], [2., 3., 4., 5.]])
]
evaluator = eval_metrics.Evaluator(
[metrics_online.StddevAcrossRuns(eval_points=[0, 1], baseline=1)])
results = evaluator.compute_metrics(curves)
np.testing.assert_allclose(results['StddevAcrossRuns'],
[1.41421356237, 2.12132034356])
def testCorrectStddevAcrossRuns(self, timepoints, lowpass_thresh, baseline,
expected):
curves = [
np.array([[-1, 0, 1], [1., 1., 1.]]),
np.array([[-1, 0, 1, 2], [2., 3., 4., 5.]])
]
metric = metrics_online.StddevAcrossRuns(lowpass_thresh=lowpass_thresh,
eval_points=timepoints,
baseline=baseline)
result = metric(curves)
np.testing.assert_allclose(result, expected)
def test_write_results(self):
# Generate some results.
curves = [
np.array([[-1, 0, 1], [1., 1., 1.]]),
np.array([[-1, 0, 1, 2], [2., 3., 4., 5.]])
]
metric = metrics_online.StddevAcrossRuns(eval_points=[0, 1],
baseline=1)
evaluator = eval_metrics.Evaluator([metric])
results = evaluator.compute_metrics(curves)
outfile_prefix = os.path.join(flags.FLAGS.test_tmpdir, 'results_')
params_path = evaluator.write_metric_params(outfile_prefix)
results_path = evaluator.write_results(results, outfile_prefix)
# Test write_results.
with open(results_path, 'r') as outfile:
results_loaded = outfile.readline()
results_dict = json.loads(results_loaded)
expected = {'StddevAcrossRuns': [1.41421356237, 2.12132034356]}
self.assertEqual(results_dict.keys(), expected.keys())
np.testing.assert_allclose(expected['StddevAcrossRuns'],
results_dict['StddevAcrossRuns'])
# Test write_metric_params.
with open(params_path, 'r') as outfile:
params_loaded = outfile.readline()
expected = json.dumps({
'StddevAcrossRuns': {
'eval_points': [0, 1],
'lowpass_thresh': None,
'baseline': 1,
'window_size': None,
}
})
self.assertJsonEqual(expected, params_loaded)
def test_window_empty(self):
curves = [np.array([[0, 2], [2, 3]])]
evaluator = eval_metrics.Evaluator([metrics_online.StddevAcrossRuns()])
self.assertRaises(ValueError, evaluator.compute_metrics, curves)
def test_window_out_of_range(self):
curves = [np.array([[0, 1], [1, 1]])]
evaluator = eval_metrics.Evaluator([metrics_online.StddevAcrossRuns()])
self.assertRaises(ValueError, evaluator.compute_metrics, curves)
def testErrorOnEvalPointsOutOfBounds(self, curves, eval_points):
metric = metrics_online.StddevAcrossRuns(lowpass_thresh=None,
eval_points=eval_points)
self.assertRaises(ValueError, metric, curves)