p_weighted_citizen_answer = float(sum(list_of_weighted_answers))/sum(list_of_weights)
# list_of_positive_probability_estimates.append(p_weighted_citizen_answer)
weighted_scores.append(p_weighted_citizen_answer)
#### Calculate AUC
# convert_expert_list = boolstr_to_floatstr(list_of_expert_answers)
# y_true = np.array(convert_expert_list)
# y_scores = np.array(list_of_positive_probability_estimates)
# print(leave_one_out, sum(y_true), len(y_scores))
# auc = roc_auc_score(y_true, y_scores)
# list_of_auc.append(auc)
# print(float(sum(list_of_auc))/len(list_of_auc))
#### Plot ROC curves
converted_y_list = boolstr_to_floatstr(y)
y_true_roc = np.array(converted_y_list)
y_unweighted_scores_roc = np.array(unweighted_scores)
y_weighted_scores_roc = np.array(weighted_scores)
fpr_uw, tpr_uw, thresholds_uw = metrics.roc_curve(y_true_roc, y_unweighted_scores_roc)
fpr_w, tpr_w, thresholds_w = metrics.roc_curve(y_true_roc, y_weighted_scores_roc)
# print(fpr_uw, tpr_uw, thresholds_uw)
# print(fpr_w, tpr_w, thresholds_w)
plt.figure()
plt.plot(fpr_uw, tpr_uw, c = 'green')
plt.plot(fpr_w, tpr_w, c = 'black')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.show()
# store the information in the dict excluding the image left out
combined_dict_without_one_image.update({composite_key:list_of_values})
# else if the image chosen to be left out above is equal to the image for this composite key
elif leave_one_out == image_ck:
# store the information in the dict including the image left out
combined_dict_with_one_image.update({composite_key:list_of_values})
list_of_ppes = calculate_ppe_from_bayesian_aggregation(dict_without_one_image = combined_dict_without_one_image, dict_with_one_image = combined_dict_with_one_image, user_ids = inner_sample)
for ppe in list_of_ppes:
citizen_ppes.append(ppe)
for composite_key, list_of_values in combined_dict_with_one_image.items():
expert_ans = list_of_values['expert_consensus_excluding_tied']
list_of_expert_answers.append(expert_ans)
convert_expert_list = boolstr_to_floatstr(list_of_expert_answers)
y_true = np.array(convert_expert_list)
y_scores = np.array(citizen_ppes)
auc = roc_auc_score(y_true, y_scores)
auc_array[count_samples,count_subjects] = auc
count_samples += 1
count_subjects += 1
print "Saving AUC matrix..."
np.save("auc_matrix.npy",auc_array)
