def eval_calibration(sessions, agent, population_fit, use_MAP=True, n_bins=10, fixed_widths=False, to_plot=False):
"""Caluculate real choice probabilities as function of model choice probabilities."""
session_fits = population_fit["MAP_fits"]
assert len(session_fits[0]["params_T"]) == agent.n_params, "agent n_params does not match population_fit."
assert len(sessions) == len(session_fits), "Number of fits does not match number of sessions."
assert population_fit["agent_name"] == agent.name, "Agent name different from that used for fits."
# Create arrays containing model choice probabilites and true choices for each trial.
session_choices, session_choice_probs = ([], [])
for fit, session in zip(session_fits, sessions):
if use_MAP:
params_T = fit["params_T"]
else:
params_T = ru.sample_params_T_from_pop_params(population_fit["pop_params"], agent)
session_choices.append(session.CTSO["choices"].tolist())
session_choice_probs.append(agent.session_likelihood(session, params_T, return_trial_data=True)["choice_probs"])
choices = np.hstack(session_choices)
choice_probs = np.vstack(session_choice_probs)[:, 1]
# Calculate true vs model choice probs.
true_probs = np.zeros(n_bins)
model_probs = np.zeros(n_bins)
if fixed_widths: # Bins of equal width in model choice probability.
bin_edges = np.linspace(0, 1, n_bins + 1)
bin_width = bin_edges[1] - bin_edges[0]
else: # Bins of equal trial number.
choices = choices[np.argsort(choice_probs)]
choice_probs.sort()
bin_edges = choice_probs[np.round(np.linspace(0, len(choice_probs) - 1, n_bins + 1)).astype(int)]
bin_edges[0] = 0.0
for b in range(n_bins):
true_probs[b] = np.mean(choices[np.logical_and(bin_edges[b] < choice_probs, choice_probs <= bin_edges[b + 1])])
model_probs[b] = np.mean(
choice_probs[np.logical_and(bin_edges[b] < choice_probs, choice_probs <= bin_edges[b + 1])]
)
calibration = {"true_probs": true_probs, "model_probs": model_probs}
if to_plot:
rp.calibration_plot(calibration)
print ("Fraction correct: {}".format(sum((choice_probs > 0.5) == choices.astype(bool)) / float(len(choices))))
chosen_probs = np.hstack([choice_probs[choices == 1], 1.0 - choice_probs[choices == 0]])
print ("Geometric mean choice prob: {}".format(np.exp(np.mean(np.log(chosen_probs)))))
return calibration
