# store the information in the dict excluding the image left out combined_dict_without_one_image.update({composite_key:list_of_values}) # else if the image chosen to be left out above is equal to the image for this composite key elif leave_one_out == image_ck: # store the information in the dict including the image left out combined_dict_with_one_image.update({composite_key:list_of_values}) list_of_ppes = calculate_ppe_from_bayesian_aggregation(dict_without_one_image = combined_dict_without_one_image, dict_with_one_image = combined_dict_with_one_image, user_ids = inner_sample) for ppe in list_of_ppes: citizen_ppes.append(ppe) for composite_key, list_of_values in combined_dict_with_one_image.items(): expert_ans = list_of_values['expert_consensus_excluding_tied'] list_of_expert_answers.append(expert_ans) convert_expert_list = boolstr_to_floatstr(list_of_expert_answers) y_true = np.array(convert_expert_list) y_scores = np.array(citizen_ppes) auc = roc_auc_score(y_true, y_scores) auc_array[count_samples,count_subjects] = auc count_samples += 1 count_subjects += 1 print "Saving AUC matrix..." np.save("auc_matrix.npy",auc_array)
ppe_uw = float(sum(list_of_unweighted_answers)) / number list_of_positive_probability_estimates_no_uw.append(ppe_uw) #### User weighting applied ppe_w = float(sum(list_of_weighted_answers))/sum(list_of_weights) list_of_positive_probability_estimates_uw.append(ppe_w) # Convert experts convert_expert_list = boolstr_to_floatstr(list_of_expert_answers) y_true = np.array(convert_expert_list) # Unweighted version y_scores_no_uw = np.array(list_of_positive_probability_estimates_no_uw) auc_no_uw = roc_auc_score(y_true, y_scores_no_uw) list_of_inner_aucs_no_uw.append(auc_no_uw) array_of_aucs[count_samples,count_images,count_subjects,1] = auc_no_uw # Weighted version y_scores_uw = np.array(list_of_positive_probability_estimates_uw) auc_uw = roc_auc_score(y_true, y_scores_uw) list_of_inner_aucs_uw.append(auc_uw) array_of_aucs[count_samples,count_images,count_subjects,0] = auc_uw count_samples += 1 # Unweighted version average_auc_no_uw = float(sum(list_of_inner_aucs_no_uw))/len(list_of_inner_aucs_no_uw) list_of_aucs_for_an_image_no_uw.append(average_auc_no_uw) # Weighted version