def test_final_credit_distribution_metric_can_interact_with_lending(self):
env = lending.DelayedImpactEnv()
env.set_scalar_reward(rewards.NullReward())
# Use step=-1 to get the final credit distribution.
final_distribution = lending_metrics.CreditDistribution(env, step=-1)
initial_distribution = lending_metrics.CreditDistribution(env, step=0)
test_util.run_test_simulation(
env=env, metric=[final_distribution, initial_distribution])
def run(self):
"""Run a lending experiment.
Returns:
A json encoding of the experiment result.
"""
env, agent = self.scenario_builder()
metrics = {
'initial_credit_distribution':
lending_metrics.CreditDistribution(env, step=0),
'final_credit_distributions':
lending_metrics.CreditDistribution(env, step=-1),
'recall':
error_metrics.RecallMetric(
env,
prediction_fn=lambda x: x.action,
ground_truth_fn=lambda x: not x.state.will_default,
stratify_fn=lambda x: str(x.state.group_id)),
'precision':
error_metrics.PrecisionMetric(
env,
prediction_fn=lambda x: x.action,
ground_truth_fn=lambda x: not x.state.will_default,
stratify_fn=lambda x: str(x.state.group_id)),
'profit rate':
value_tracking_metrics.ValueChange(env, state_var='bank_cash'),
}
if self.include_cumulative_loans:
metrics['cumulative_loans'] = lending_metrics.CumulativeLoans(env)
metrics['cumulative_recall'] = lending_metrics.CumulativeRecall(
env)
metric_results = run_util.run_simulation(env, agent, metrics,
self.num_steps, self.seed)
report = {
'environment': {
'name': env.__class__.__name__,
'params': env.initial_params,
'history': env.history,
'env': env
},
'agent': {
'name': agent.__class__.__name__,
'params': agent.params,
'debug_string': agent.debug_string(),
'threshold_history': agent.group_specific_threshold_history,
'tpr_targets': agent.target_recall_history,
},
'experiment_params': self,
'metric_results': metric_results,
}
if self.return_json:
return core.to_json(report, indent=4)
return report
def plot_distribution_distance(envs, histories, path, distance_metric):
"""Plots the difference between credit distributions."""
plt.figure(figsize=(8, 3))
plt.title(distance_metric.name.lower() +
' distance between group 1 and group 2',
fontsize=16)
colors = ['b', 'g']
for title, history in histories.items():
plt.plot([
_distribution_difference(
lending_metrics.CreditDistribution(
envs[title], step=step).measure(envs[title]),
distance_metric) for step in range(len(history))
],
label='%s' % title)
plt.xticks(fontsize=12)
plt.yticks(fontsize=12)
plt.ylabel(distance_metric.name.lower() + ' distance', fontsize=16)
plt.xlabel('# Steps', fontsize=16)
plt.legend(loc='upper left', fontsize=12)
plt.grid(color='k', linewidth=0.5, axis='y')
plt.tight_layout()
_write(path.split(sep='.')[0] \
+ '_' + distance_metric.name.lower() + '.' + path.split(sep='.')[1])
def test_measure_distribution_change_measurement(self):
# The lower cluster has a 100% success rate and the upper cluster has a 0%
# success rate. This causes applicants to move constantly between clusters.
clusters = distributions.Mixture(
components=[
lending_params._credit_cluster_builder(
group_membership=[1, 0],
cluster_probs=[0.1, 0.9],
success_probs=[1., 0.])(),
lending_params._credit_cluster_builder(
group_membership=[0, 1],
cluster_probs=[0.8, 0.2],
success_probs=[1., 0.])(),
],
weights=(0.5, 0.5))
env = lending.DelayedImpactEnv(
lending_params.DelayedImpactParams(applicant_distribution=clusters))
initial_distribution = lending_metrics.CreditDistribution(env, 0)
final_distribution = lending_metrics.CreditDistribution(env, -1)
# Giving a loan should change the distribution.
env.step(np.asarray(1))
# Take another step to move current state into history. This step does not
# change the distribution because the loan is rejected.
env.step(np.asarray(0))
self.assertEqual({
'0': [0.1, 0.9],
'1': [0.8, 0.2]
}, initial_distribution.measure(env))
self.assertNotEqual({
'0': [0.1, 0.9],
'1': [0.8, 0.2]
}, final_distribution.measure(env))