def test_confusion_metric_correct_for_sequence_prediction_rule(self): dim = 10 def _ground_truth_fn(history_item): state, _ = history_item return state.x env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=dim)) env.set_scalar_reward(rewards.NullReward()) # Always predict a sequence of 1s. metric = error_metrics.ConfusionMetric( env=env, prediction_fn=lambda x: [1 for _ in range(dim)], ground_truth_fn=_ground_truth_fn, stratify_fn=lambda x: [1 for _ in range(dim)], ) measurement = test_util.run_test_simulation(env=env, agent=None, metric=metric) logging.info("Measurement: %s.", measurement) self.assertEqual(measurement[1].fp, 50) self.assertEqual(measurement[1].tp, 50) self.assertNotIn(0, measurement)
def test_confusion_metric_correct_for_atomic_prediction_rule(self): def _ground_truth_fn(history_item): state, _ = history_item return state.x[0] env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=1)) env.set_scalar_reward(rewards.NullReward()) # Always predict 1. metric = error_metrics.ConfusionMetric( env=env, prediction_fn=lambda x: 1, ground_truth_fn=_ground_truth_fn, stratify_fn=lambda x: 1, ) measurement = test_util.run_test_simulation(env=env, agent=None, metric=metric) logging.info("Measurement: %s.", measurement) # The keys in measurement are given by group membership, which in this case # is defined to always be 1. self.assertEqual(measurement[1].fp, 5) self.assertEqual(measurement[1].tp, 5) self.assertNotIn(0, measurement)
def _setup_test_simulation(dim=1, calc_mean=False, modifier_fn=_modifier_fn): env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=dim)) env.set_scalar_reward(rewards.NullReward()) metric = value_tracking_metrics.AggregatorMetric( env=env, modifier_fn=modifier_fn, selection_fn=_selection_fn, stratify_fn=_stratify_fn, calc_mean=calc_mean) return env, metric
def test_summing_metric_give_correct_sum_dummy_env(self): env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=1)) env.set_scalar_reward(rewards.NullReward()) metric = value_tracking_metrics.SummingMetric( env=env, selection_fn=_selection_fn) measurement = test_util.run_test_simulation(env, agent=None, metric=metric, seed=0) self.assertTrue(np.all(np.equal(measurement, [5])))
def test_recall_with_zero_denominator(self): env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=1)) env.set_scalar_reward(rewards.NullReward()) # Ground truth is always 0, recall will have a zero denominator. metric = error_metrics.RecallMetric( env=env, prediction_fn=lambda x: 0, ground_truth_fn=lambda x: 0, stratify_fn=lambda x: 1) measurement = test_util.run_test_simulation( env=env, agent=None, metric=metric, num_steps=50) self.assertEqual({1: 0}, measurement)
def test_precision_with_zero_denominator(self): def _ground_truth_fn(history_item): state, _ = history_item return state.x[0] env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=1)) env.set_scalar_reward(rewards.NullReward()) # Always predict 0, precision will have a zero denominator. metric = error_metrics.PrecisionMetric( env=env, prediction_fn=lambda x: 0, ground_truth_fn=_ground_truth_fn, stratify_fn=lambda x: 1) measurement = test_util.run_test_simulation( env=env, agent=None, metric=metric, num_steps=50) self.assertEqual({1: 0}, measurement)
def test_recall_metric_correct_for_atomic_prediction_rule(self): def _ground_truth_fn(history_item): state, _ = history_item return state.x[0] env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=1)) env.set_scalar_reward(rewards.NullReward()) # Always predict 1. metric = error_metrics.RecallMetric( env=env, prediction_fn=lambda x: 1, ground_truth_fn=_ground_truth_fn, stratify_fn=lambda x: 1) measurement = test_util.run_test_simulation( env=env, agent=None, metric=metric, num_steps=50) logging.info('Measurement: %s.', measurement) self.assertEqual({1: 1}, measurement)
def test_stratified_accuracy_metric_correct_sequence_prediction(self): """Check correctness when stratifying into (wrong, right) bins.""" def _x_select(history_item): return [i == 1 for i in history_item.state.x] def _x_stratify(history_item): return history_item.state.x env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=10)) env.set_scalar_reward(rewards.NullReward()) metric = error_metrics.AccuracyMetric( env=env, numerator_fn=_x_select, stratify_fn=_x_stratify) measurement = test_util.run_test_simulation( env=env, agent=None, metric=metric) logging.info('Measurement: %s.', measurement) self.assertEqual(measurement[0], 0) self.assertEqual(measurement[1], 1)
def test_cost_metric_correct_for_atomic_prediction_rule(self): def _ground_truth_fn(history_item): state, _ = history_item return state.x[0] env = test_util.DeterministicDummyEnv(test_util.DummyParams(dim=1)) env.set_scalar_reward(rewards.NullReward()) cost_metric = error_metrics.CostedConfusionMetric( env=env, prediction_fn=lambda x: 1, ground_truth_fn=_ground_truth_fn, stratify_fn=lambda x: 1, cost_matrix=params.CostMatrix(tp=1, fp=-2, tn=-1, fn=-1)) measurement = test_util.run_test_simulation( env=env, agent=None, metric=cost_metric) logging.info('Cost measurement: %s.', measurement) self.assertEqual(measurement[1], -5) self.assertNotIn(0, measurement)