def _validate_metadata(metadata, pats_predicted): """Compute validation score for given patients. """ errors = [] for pid in pats_predicted: prediction_pid = metadata['predictions'][pid-1] assert prediction_pid['patient'] == pid error_sys = utils.CRSP(prediction_pid["systole_average"], data_loader.regular_labels[pid-1, 1]) error_dia = utils.CRSP(prediction_pid["diastole_average"], data_loader.regular_labels[pid-1, 2]) errors += [error_sys, error_dia] return np.mean(errors)
def get_validate_crps(predictions, labels): errors = [] for patient in validation_patients_indices: prediction = predictions[patient-1] if "systole_average" in prediction and prediction["systole_average"] is not None: assert patient == labels[patient-1, 0] error = utils.CRSP(prediction["systole_average"], labels[patient-1, 1]) errors.append(error) error = utils.CRSP(prediction["diastole_average"], labels[patient-1, 2]) errors.append(error) if len(errors)>0: errors = np.array(errors) estimated_CRSP = np.mean(errors) return estimated_CRSP else: return utils.maxfloat
def merge_all_prediction_files(prediction_file_location = INTERMEDIATE_PREDICTIONS_PATH, redo_tta = True): submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time() # calculate the average distribution regular_labels = _load_file(_TRAIN_LABELS_PATH) average_systole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,1]]), axis=0)) average_diastole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,2]]), axis=0)) ss_expert_pkl_files = sorted([] +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout_seqshift_96.pkl") +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_big_leaky_after_seqshift.pkl") +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_big.pkl") +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_zoom_mask_leaky_after.pkl") +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout.pkl") +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky_after.pkl") +glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_maxout.pkl") +glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_nomask.pkl") +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky.pkl") +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom.pkl") +glob.glob(prediction_file_location+"je_ss_jonisc64small_360_gauss_longer.pkl") +glob.glob(prediction_file_location+"j6_2ch_128mm.pkl") +glob.glob(prediction_file_location+"j6_2ch_96mm.pkl") +glob.glob(prediction_file_location+"je_ss_jonisc80_framemax.pkl") +glob.glob(prediction_file_location+"j6_2ch_128mm_96.pkl") +glob.glob(prediction_file_location+"j6_4ch.pkl") +glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll.pkl") +glob.glob(prediction_file_location+"j6_4ch_32mm_specialist.pkl") +glob.glob(prediction_file_location+"j6_4ch_128mm_specialist.pkl") +glob.glob(prediction_file_location+"je_ss_jonisc64_leaky_convroll.pkl") +glob.glob(prediction_file_location+"je_ss_jonisc80small_360_gauss_longer_augzoombright.pkl") +glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll_augzoombright.pkl") +glob.glob(prediction_file_location+"j6_2ch_128mm_zoom.pkl") +glob.glob(prediction_file_location+"j6_2ch_128mm_skew.pkl") +glob.glob(prediction_file_location+"je_ss_jonisc64small_360.pkl") ) fp_expert_pkl_files = sorted([] +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout_seqshift_96.pkl") +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_big_leaky_after_seqshift.pkl") +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_big.pkl") +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_zoom_mask_leaky_after.pkl") +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout.pkl") +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky_after.pkl") +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky.pkl") +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom.pkl") +glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered.pkl") +glob.glob(prediction_file_location+"je_os_fixedaggr_rellocframe.pkl") +glob.glob(prediction_file_location+"je_meta_fixedaggr_filtered.pkl") +glob.glob(prediction_file_location+"je_meta_fixedaggr_framemax_reg.pkl") +glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv.pkl") +glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv_augzoombright_short.pkl") +glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered_discs.pkl") +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright.pkl") +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale64small_filtered_longer.pkl") +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright_betterdist.pkl") +glob.glob(prediction_file_location+"je_os_segmentandintegrate_smartsigma_dropout.pkl") ) everything = sorted([] +glob.glob(prediction_file_location+"*.pkl") ) expert_pkl_files = ss_expert_pkl_files + fp_expert_pkl_files print("found %d/44 files" % len(expert_pkl_files)) """ # filter expert_pkl_files for file in expert_pkl_files[:]: try: with open(file, 'r') as f: print "testing file",file.split('/')[-1] data = pickle.load(f) if 'predictions' not in data.keys(): expert_pkl_files.remove(file) print " -> removed" except: print sys.exc_info()[0] expert_pkl_files.remove(file) print " -> removed" """ NUM_EXPERTS = len(expert_pkl_files) NUM_VALIDATIONS = len(validation_patients_indices) NUM_TESTS = len(test_patients_indices) systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32') diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32') systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool') diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool') test_systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32') test_diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32') test_systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool') test_diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool') for i,file in enumerate(expert_pkl_files): with open(file, 'r') as f: print() print("loading file",file.split('/')[-1]) predictions = pickle.load(f)['predictions'] if redo_tta: best_average_method = normalav best_average_crps = utils.maxfloat for average_method in [geomav, normalav, prodav, weighted_geom_method ]: calculate_tta_average(predictions, average_method, average_systole, average_diastole) crps = get_validate_crps(predictions, regular_labels) print(string.rjust(average_method.__name__,25),"->",crps) if crps<best_average_crps: best_average_method = average_method best_average_crps = crps print(" I choose you,", best_average_method.__name__) calculate_tta_average(predictions, best_average_method, average_systole, average_diastole) print(" validation loss:", get_validate_crps(predictions, regular_labels)) for j,patient in enumerate(validation_patients_indices): prediction = predictions[patient-1] # average distributions get zero weight later on if "systole_average" in prediction and prediction["systole_average"] is not None: systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"] else: systole_masked_expert_predictions_matrix[i,j] = False if "diastole_average" in prediction and prediction["diastole_average"] is not None: diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"] else: diastole_masked_expert_predictions_matrix[i,j] = False for j,patient in enumerate(test_patients_indices): prediction = predictions[patient-1] # average distributions get zero weight later on if "systole_average" in prediction and prediction["systole_average"] is not None: test_systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"] else: test_systole_masked_expert_predictions_matrix[i,j] = False if "diastole_average" in prediction and prediction["diastole_average"] is not None: test_diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"] else: test_diastole_masked_expert_predictions_matrix[i,j] = False del predictions # can be LOADS of data cv = [id-1 for id in validation_patients_indices] systole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,1].flatten()]) systole_expert_weight, first_pass_sys_loss, systole_optimal_params = get_optimal_ensemble_weights_for_these_experts( expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'), prediction_matrix=systole_expert_predictions_matrix, mask_matrix=systole_masked_expert_predictions_matrix, labels=systole_valid_labels, average_distribution=average_systole, ) cv = [id-1 for id in validation_patients_indices] diastole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,2].flatten()]) diastole_expert_weight, first_pass_dia_loss, diastole_optimal_params = get_optimal_ensemble_weights_for_these_experts( expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'), prediction_matrix=diastole_expert_predictions_matrix, mask_matrix=diastole_masked_expert_predictions_matrix, labels=diastole_valid_labels, average_distribution=average_diastole, ) # print the final weight of every expert print(" Systole: Diastole: Name:") for expert_name, systole_weight, diastole_weight in zip(expert_pkl_files, systole_expert_weight, diastole_expert_weight): print(string.rjust("%.3f%%" % (100*systole_weight), 10), end=' ') print(string.rjust("%.3f%%" % (100*diastole_weight), 10), end=' ') print(expert_name.split('/')[-1]) print() print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2)) print() print() print("Average the experts according to these weights to find the final distribution") final_predictions = [{ "patient": i+1, "final_systole": None, "final_diastole": None } for i in range(NUM_PATIENTS)] generate_final_predictions( final_predictions=final_predictions, prediction_tag="final_systole", expert_predictions_matrix=systole_expert_predictions_matrix, masked_expert_predictions_matrix=systole_masked_expert_predictions_matrix, test_expert_predictions_matrix=test_systole_expert_predictions_matrix, test_masked_expert_predictions_matrix=test_systole_masked_expert_predictions_matrix, optimal_params=systole_optimal_params, average_distribution=average_systole, valid_labels=systole_valid_labels, expert_pkl_files=expert_pkl_files, expert_weight=systole_expert_weight, disagreement_cutoff=0.01 # 0.01 ) generate_final_predictions( final_predictions=final_predictions, prediction_tag="final_diastole", expert_predictions_matrix=diastole_expert_predictions_matrix, masked_expert_predictions_matrix=diastole_masked_expert_predictions_matrix, test_expert_predictions_matrix=test_diastole_expert_predictions_matrix, test_masked_expert_predictions_matrix=test_diastole_masked_expert_predictions_matrix, optimal_params=diastole_optimal_params, average_distribution=average_diastole, valid_labels=diastole_valid_labels, expert_pkl_files=expert_pkl_files, expert_weight=diastole_expert_weight, disagreement_cutoff=0.015 # diastole has about 50% more error ) print() print("Calculating training and validation set scores for reference") validation_dict = {} for patient_ids, set_name in [(validation_patients_indices, "validation")]: errors = [] for patient in patient_ids: prediction = final_predictions[patient-1] if "final_systole" in prediction: assert patient == regular_labels[patient-1, 0] error1 = utils.CRSP(prediction["final_systole"], regular_labels[patient-1, 1]) errors.append(error1) prediction["systole_crps_error"] = error1 error2 = utils.CRSP(prediction["final_diastole"], regular_labels[patient-1, 2]) errors.append(error2) prediction["diastole_crps_error"] = error1 prediction["average_crps_error"] = 0.5*error1 + 0.5*error2 if len(errors)>0: errors = np.array(errors) estimated_CRSP = np.mean(errors) print(" %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP)) validation_dict[set_name] = estimated_CRSP else: print(" %s kaggle loss: not calculated" % (string.rjust(set_name, 12))) print("WARNING: both of the previous are overfitted!") print() print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2)) print() print("dumping submission file to %s" % submission_path) with open(submission_path, 'w') as csvfile: csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) csvwriter.writerow(['Id'] + ['P%d'%i for i in range(600)]) for prediction in final_predictions: # the submission only has patients 501 to 700 if prediction["patient"] in test_patients_indices: if "final_diastole" not in prediction or "final_systole" not in prediction: raise Exception("Not all test-set patients were predicted") csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_diastole"].flatten()]) csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_systole"].flatten()]) print("submission file dumped")