def test_value_change_measures_correctly_normalized(self): env = IncreasingEnv() metric = value_tracking_metrics.ValueChange(env, 'x', normalize_by_steps=True) for _ in range(17): env.step(action=env.action_space.sample()) self.assertAlmostEqual(metric.measure(env), 1.0)
def run(self): """Run a lending experiment. Returns: A json encoding of the experiment result. """ env, agent = self.scenario_builder() metrics = { 'initial_credit_distribution': lending_metrics.CreditDistribution(env, step=0), 'final_credit_distributions': lending_metrics.CreditDistribution(env, step=-1), 'recall': error_metrics.RecallMetric( env, prediction_fn=lambda x: x.action, ground_truth_fn=lambda x: not x.state.will_default, stratify_fn=lambda x: str(x.state.group_id)), 'precision': error_metrics.PrecisionMetric( env, prediction_fn=lambda x: x.action, ground_truth_fn=lambda x: not x.state.will_default, stratify_fn=lambda x: str(x.state.group_id)), 'profit rate': value_tracking_metrics.ValueChange(env, state_var='bank_cash'), } if self.include_cumulative_loans: metrics['cumulative_loans'] = lending_metrics.CumulativeLoans(env) metrics['cumulative_recall'] = lending_metrics.CumulativeRecall( env) metric_results = run_util.run_simulation(env, agent, metrics, self.num_steps, self.seed) report = { 'environment': { 'name': env.__class__.__name__, 'params': env.initial_params, 'history': env.history, 'env': env }, 'agent': { 'name': agent.__class__.__name__, 'params': agent.params, 'debug_string': agent.debug_string(), 'threshold_history': agent.group_specific_threshold_history, 'tpr_targets': agent.target_recall_history, }, 'experiment_params': self, 'metric_results': metric_results, } if self.return_json: return core.to_json(report, indent=4) return report
def test_value_change_measures_correctly_unnormalized(self): env = IncreasingEnv() metric = value_tracking_metrics.ValueChange(env, 'x', normalize_by_steps=False) # Running step 11 times records 10 steps in history because the 11th is # stored in current state. for _ in range(11): env.step(env.action_space.sample()) self.assertEqual(metric.measure(env), 10)
def test_metric_can_interact_with_lending(self): env = lending.DelayedImpactEnv() metric = value_tracking_metrics.ValueChange(env, 'bank_cash') test_util.run_test_simulation(env=env, metric=metric)