def run(self):
"""Run a lending experiment.
Returns:
A json encoding of the experiment result.
"""
env, agent = self.scenario_builder()
metrics = {
'initial_credit_distribution':
lending_metrics.CreditDistribution(env, step=0),
'final_credit_distributions':
lending_metrics.CreditDistribution(env, step=-1),
'recall':
error_metrics.RecallMetric(
env,
prediction_fn=lambda x: x.action,
ground_truth_fn=lambda x: not x.state.will_default,
stratify_fn=lambda x: str(x.state.group_id)),
'precision':
error_metrics.PrecisionMetric(
env,
prediction_fn=lambda x: x.action,
ground_truth_fn=lambda x: not x.state.will_default,
stratify_fn=lambda x: str(x.state.group_id)),
'profit rate':
value_tracking_metrics.ValueChange(env, state_var='bank_cash'),
}
if self.include_cumulative_loans:
metrics['cumulative_loans'] = lending_metrics.CumulativeLoans(env)
metrics['cumulative_recall'] = lending_metrics.CumulativeRecall(
env)
metric_results = run_util.run_simulation(env, agent, metrics,
self.num_steps, self.seed)
report = {
'environment': {
'name': env.__class__.__name__,
'params': env.initial_params,
'history': env.history,
'env': env
},
'agent': {
'name': agent.__class__.__name__,
'params': agent.params,
'debug_string': agent.debug_string(),
'threshold_history': agent.group_specific_threshold_history,
'tpr_targets': agent.target_recall_history,
},
'experiment_params': self,
'metric_results': metric_results,
}
if self.return_json:
return core.to_json(report, indent=4)
return report
def test_cumulative_count(self):
env = lending.DelayedImpactEnv()
metric = lending_metrics.CumulativeLoans(env)
env.seed(100)
_ = env.reset()
for _ in range(10):
env.step(np.asarray(1))
result = metric.measure(env)
self.assertEqual(result.shape, (2, 10))
# On the first step, the combined number of loans given out should be 1.
self.assertEqual(result[:, 0].sum(), 1)
# On the last step, the combined number of loans given out should be 10.
self.assertEqual(result[:, -1].sum(), 10)
def test_no_loans_to_group_zero(self):
env = lending.DelayedImpactEnv()
metric = lending_metrics.CumulativeLoans(env)
env.seed(100)
obs = env.reset()
for _ in range(10):
# action is 0 for group 0 and 1 for group 1.
action = np.argmax(obs['group'])
obs, _, _, _ = env.step(action)
result = metric.measure(env)
self.assertEqual(result.shape, (2, 10))
# Group 0 gets no loans.
self.assertEqual(result[0, -1], 0)
# Group 1 gets at least 1 loan.
self.assertGreater(result[1, -1], 0)