def eval_calibration(sessions, agent, population_fit, use_MAP=True, n_bins=10, fixed_widths=False, to_plot=False): """Caluculate real choice probabilities as function of model choice probabilities.""" session_fits = population_fit["MAP_fits"] assert len(session_fits[0]["params_T"]) == agent.n_params, "agent n_params does not match population_fit." assert len(sessions) == len(session_fits), "Number of fits does not match number of sessions." assert population_fit["agent_name"] == agent.name, "Agent name different from that used for fits." # Create arrays containing model choice probabilites and true choices for each trial. session_choices, session_choice_probs = ([], []) for fit, session in zip(session_fits, sessions): if use_MAP: params_T = fit["params_T"] else: params_T = ru.sample_params_T_from_pop_params(population_fit["pop_params"], agent) session_choices.append(session.CTSO["choices"].tolist()) session_choice_probs.append(agent.session_likelihood(session, params_T, return_trial_data=True)["choice_probs"]) choices = np.hstack(session_choices) choice_probs = np.vstack(session_choice_probs)[:, 1] # Calculate true vs model choice probs. true_probs = np.zeros(n_bins) model_probs = np.zeros(n_bins) if fixed_widths: # Bins of equal width in model choice probability. bin_edges = np.linspace(0, 1, n_bins + 1) bin_width = bin_edges[1] - bin_edges[0] else: # Bins of equal trial number. choices = choices[np.argsort(choice_probs)] choice_probs.sort() bin_edges = choice_probs[np.round(np.linspace(0, len(choice_probs) - 1, n_bins + 1)).astype(int)] bin_edges[0] = 0.0 for b in range(n_bins): true_probs[b] = np.mean(choices[np.logical_and(bin_edges[b] < choice_probs, choice_probs <= bin_edges[b + 1])]) model_probs[b] = np.mean( choice_probs[np.logical_and(bin_edges[b] < choice_probs, choice_probs <= bin_edges[b + 1])] ) calibration = {"true_probs": true_probs, "model_probs": model_probs} if to_plot: rp.calibration_plot(calibration) print ("Fraction correct: {}".format(sum((choice_probs > 0.5) == choices.astype(bool)) / float(len(choices)))) chosen_probs = np.hstack([choice_probs[choices == 1], 1.0 - choice_probs[choices == 0]]) print ("Geometric mean choice prob: {}".format(np.exp(np.mean(np.log(chosen_probs))))) return calibration