def test_value_change_measures_correctly_normalized(self):
env = IncreasingEnv()
metric = value_tracking_metrics.ValueChange(env,
'x',
normalize_by_steps=True)
for _ in range(17):
env.step(action=env.action_space.sample())
self.assertAlmostEqual(metric.measure(env), 1.0)
def run(self):
"""Run a lending experiment.
Returns:
A json encoding of the experiment result.
"""
env, agent = self.scenario_builder()
metrics = {
'initial_credit_distribution':
lending_metrics.CreditDistribution(env, step=0),
'final_credit_distributions':
lending_metrics.CreditDistribution(env, step=-1),
'recall':
error_metrics.RecallMetric(
env,
prediction_fn=lambda x: x.action,
ground_truth_fn=lambda x: not x.state.will_default,
stratify_fn=lambda x: str(x.state.group_id)),
'precision':
error_metrics.PrecisionMetric(
env,
prediction_fn=lambda x: x.action,
ground_truth_fn=lambda x: not x.state.will_default,
stratify_fn=lambda x: str(x.state.group_id)),
'profit rate':
value_tracking_metrics.ValueChange(env, state_var='bank_cash'),
}
if self.include_cumulative_loans:
metrics['cumulative_loans'] = lending_metrics.CumulativeLoans(env)
metrics['cumulative_recall'] = lending_metrics.CumulativeRecall(
env)
metric_results = run_util.run_simulation(env, agent, metrics,
self.num_steps, self.seed)
report = {
'environment': {
'name': env.__class__.__name__,
'params': env.initial_params,
'history': env.history,
'env': env
},
'agent': {
'name': agent.__class__.__name__,
'params': agent.params,
'debug_string': agent.debug_string(),
'threshold_history': agent.group_specific_threshold_history,
'tpr_targets': agent.target_recall_history,
},
'experiment_params': self,
'metric_results': metric_results,
}
if self.return_json:
return core.to_json(report, indent=4)
return report
def test_value_change_measures_correctly_unnormalized(self):
env = IncreasingEnv()
metric = value_tracking_metrics.ValueChange(env,
'x',
normalize_by_steps=False)
# Running step 11 times records 10 steps in history because the 11th is
# stored in current state.
for _ in range(11):
env.step(env.action_space.sample())
self.assertEqual(metric.measure(env), 10)
def test_metric_can_interact_with_lending(self):
env = lending.DelayedImpactEnv()
metric = value_tracking_metrics.ValueChange(env, 'bank_cash')
test_util.run_test_simulation(env=env, metric=metric)