p_weighted_citizen_answer = float(sum(list_of_weighted_answers))/sum(list_of_weights) # list_of_positive_probability_estimates.append(p_weighted_citizen_answer) weighted_scores.append(p_weighted_citizen_answer) #### Calculate AUC # convert_expert_list = boolstr_to_floatstr(list_of_expert_answers) # y_true = np.array(convert_expert_list) # y_scores = np.array(list_of_positive_probability_estimates) # print(leave_one_out, sum(y_true), len(y_scores)) # auc = roc_auc_score(y_true, y_scores) # list_of_auc.append(auc) # print(float(sum(list_of_auc))/len(list_of_auc)) #### Plot ROC curves converted_y_list = boolstr_to_floatstr(y) y_true_roc = np.array(converted_y_list) y_unweighted_scores_roc = np.array(unweighted_scores) y_weighted_scores_roc = np.array(weighted_scores) fpr_uw, tpr_uw, thresholds_uw = metrics.roc_curve(y_true_roc, y_unweighted_scores_roc) fpr_w, tpr_w, thresholds_w = metrics.roc_curve(y_true_roc, y_weighted_scores_roc) # print(fpr_uw, tpr_uw, thresholds_uw) # print(fpr_w, tpr_w, thresholds_w) plt.figure() plt.plot(fpr_uw, tpr_uw, c = 'green') plt.plot(fpr_w, tpr_w, c = 'black') plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.show()
# store the information in the dict excluding the image left out combined_dict_without_one_image.update({composite_key:list_of_values}) # else if the image chosen to be left out above is equal to the image for this composite key elif leave_one_out == image_ck: # store the information in the dict including the image left out combined_dict_with_one_image.update({composite_key:list_of_values}) list_of_ppes = calculate_ppe_from_bayesian_aggregation(dict_without_one_image = combined_dict_without_one_image, dict_with_one_image = combined_dict_with_one_image, user_ids = inner_sample) for ppe in list_of_ppes: citizen_ppes.append(ppe) for composite_key, list_of_values in combined_dict_with_one_image.items(): expert_ans = list_of_values['expert_consensus_excluding_tied'] list_of_expert_answers.append(expert_ans) convert_expert_list = boolstr_to_floatstr(list_of_expert_answers) y_true = np.array(convert_expert_list) y_scores = np.array(citizen_ppes) auc = roc_auc_score(y_true, y_scores) auc_array[count_samples,count_subjects] = auc count_samples += 1 count_subjects += 1 print "Saving AUC matrix..." np.save("auc_matrix.npy",auc_array)