# store the information in the dict excluding the image left out
combined_dict_without_one_image.update({composite_key:list_of_values})
# else if the image chosen to be left out above is equal to the image for this composite key
elif leave_one_out == image_ck:
# store the information in the dict including the image left out
combined_dict_with_one_image.update({composite_key:list_of_values})
list_of_ppes = calculate_ppe_from_bayesian_aggregation(dict_without_one_image = combined_dict_without_one_image, dict_with_one_image = combined_dict_with_one_image, user_ids = inner_sample)
for ppe in list_of_ppes:
citizen_ppes.append(ppe)
for composite_key, list_of_values in combined_dict_with_one_image.items():
expert_ans = list_of_values['expert_consensus_excluding_tied']
list_of_expert_answers.append(expert_ans)
convert_expert_list = boolstr_to_floatstr(list_of_expert_answers)
y_true = np.array(convert_expert_list)
y_scores = np.array(citizen_ppes)
auc = roc_auc_score(y_true, y_scores)
auc_array[count_samples,count_subjects] = auc
count_samples += 1
count_subjects += 1
print "Saving AUC matrix..."
np.save("auc_matrix.npy",auc_array)
ppe_uw = float(sum(list_of_unweighted_answers)) / number
list_of_positive_probability_estimates_no_uw.append(ppe_uw)
#### User weighting applied
ppe_w = float(sum(list_of_weighted_answers))/sum(list_of_weights)
list_of_positive_probability_estimates_uw.append(ppe_w)
# Convert experts
convert_expert_list = boolstr_to_floatstr(list_of_expert_answers)
y_true = np.array(convert_expert_list)
# Unweighted version
y_scores_no_uw = np.array(list_of_positive_probability_estimates_no_uw)
auc_no_uw = roc_auc_score(y_true, y_scores_no_uw)
list_of_inner_aucs_no_uw.append(auc_no_uw)
array_of_aucs[count_samples,count_images,count_subjects,1] = auc_no_uw
# Weighted version
y_scores_uw = np.array(list_of_positive_probability_estimates_uw)
auc_uw = roc_auc_score(y_true, y_scores_uw)
list_of_inner_aucs_uw.append(auc_uw)
array_of_aucs[count_samples,count_images,count_subjects,0] = auc_uw
count_samples += 1
# Unweighted version
average_auc_no_uw = float(sum(list_of_inner_aucs_no_uw))/len(list_of_inner_aucs_no_uw)
list_of_aucs_for_an_image_no_uw.append(average_auc_no_uw)
# Weighted version
