def _validate_metadata(metadata, pats_predicted):
"""Compute validation score for given patients.
"""
errors = []
for pid in pats_predicted:
prediction_pid = metadata['predictions'][pid-1]
assert prediction_pid['patient'] == pid
error_sys = utils.CRSP(prediction_pid["systole_average"], data_loader.regular_labels[pid-1, 1])
error_dia = utils.CRSP(prediction_pid["diastole_average"], data_loader.regular_labels[pid-1, 2])
errors += [error_sys, error_dia]
return np.mean(errors)
def get_validate_crps(predictions, labels):
errors = []
for patient in validation_patients_indices:
prediction = predictions[patient-1]
if "systole_average" in prediction and prediction["systole_average"] is not None:
assert patient == labels[patient-1, 0]
error = utils.CRSP(prediction["systole_average"], labels[patient-1, 1])
errors.append(error)
error = utils.CRSP(prediction["diastole_average"], labels[patient-1, 2])
errors.append(error)
if len(errors)>0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
return estimated_CRSP
else:
return utils.maxfloat
def merge_all_prediction_files(prediction_file_location = INTERMEDIATE_PREDICTIONS_PATH,
redo_tta = True):
submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
# calculate the average distribution
regular_labels = _load_file(_TRAIN_LABELS_PATH)
average_systole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,1]]), axis=0))
average_diastole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,2]]), axis=0))
ss_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_nomask.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360_gauss_longer.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm.pkl")
+glob.glob(prediction_file_location+"j6_2ch_96mm.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_framemax.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_96.pkl")
+glob.glob(prediction_file_location+"j6_4ch.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"j6_4ch_32mm_specialist.pkl")
+glob.glob(prediction_file_location+"j6_4ch_128mm_specialist.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80small_360_gauss_longer_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll_augzoombright.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_zoom.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_skew.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360.pkl")
)
fp_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_rellocframe.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_filtered.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_framemax_reg.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv_augzoombright_short.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered_discs.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale64small_filtered_longer.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright_betterdist.pkl")
+glob.glob(prediction_file_location+"je_os_segmentandintegrate_smartsigma_dropout.pkl")
)
everything = sorted([]
+glob.glob(prediction_file_location+"*.pkl")
)
expert_pkl_files = ss_expert_pkl_files + fp_expert_pkl_files
print("found %d/44 files" % len(expert_pkl_files))
"""
# filter expert_pkl_files
for file in expert_pkl_files[:]:
try:
with open(file, 'r') as f:
print "testing file",file.split('/')[-1]
data = pickle.load(f)
if 'predictions' not in data.keys():
expert_pkl_files.remove(file)
print " -> removed"
except:
print sys.exc_info()[0]
expert_pkl_files.remove(file)
print " -> removed"
"""
NUM_EXPERTS = len(expert_pkl_files)
NUM_VALIDATIONS = len(validation_patients_indices)
NUM_TESTS = len(test_patients_indices)
systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
test_systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
test_diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
for i,file in enumerate(expert_pkl_files):
with open(file, 'r') as f:
print()
print("loading file",file.split('/')[-1])
predictions = pickle.load(f)['predictions']
if redo_tta:
best_average_method = normalav
best_average_crps = utils.maxfloat
for average_method in [geomav,
normalav,
prodav,
weighted_geom_method
]:
calculate_tta_average(predictions, average_method, average_systole, average_diastole)
crps = get_validate_crps(predictions, regular_labels)
print(string.rjust(average_method.__name__,25),"->",crps)
if crps<best_average_crps:
best_average_method = average_method
best_average_crps = crps
print(" I choose you,", best_average_method.__name__)
calculate_tta_average(predictions, best_average_method, average_systole, average_diastole)
print(" validation loss:", get_validate_crps(predictions, regular_labels))
for j,patient in enumerate(validation_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
diastole_masked_expert_predictions_matrix[i,j] = False
for j,patient in enumerate(test_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
test_systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
test_systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
test_diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
test_diastole_masked_expert_predictions_matrix[i,j] = False
del predictions # can be LOADS of data
cv = [id-1 for id in validation_patients_indices]
systole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,1].flatten()])
systole_expert_weight, first_pass_sys_loss, systole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=systole_expert_predictions_matrix,
mask_matrix=systole_masked_expert_predictions_matrix,
labels=systole_valid_labels,
average_distribution=average_systole,
)
cv = [id-1 for id in validation_patients_indices]
diastole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,2].flatten()])
diastole_expert_weight, first_pass_dia_loss, diastole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=diastole_expert_predictions_matrix,
mask_matrix=diastole_masked_expert_predictions_matrix,
labels=diastole_valid_labels,
average_distribution=average_diastole,
)
# print the final weight of every expert
print(" Systole: Diastole: Name:")
for expert_name, systole_weight, diastole_weight in zip(expert_pkl_files, systole_expert_weight, diastole_expert_weight):
print(string.rjust("%.3f%%" % (100*systole_weight), 10), end=' ')
print(string.rjust("%.3f%%" % (100*diastole_weight), 10), end=' ')
print(expert_name.split('/')[-1])
print()
print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2))
print()
print()
print("Average the experts according to these weights to find the final distribution")
final_predictions = [{
"patient": i+1,
"final_systole": None,
"final_diastole": None
} for i in range(NUM_PATIENTS)]
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_systole",
expert_predictions_matrix=systole_expert_predictions_matrix,
masked_expert_predictions_matrix=systole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_systole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_systole_masked_expert_predictions_matrix,
optimal_params=systole_optimal_params,
average_distribution=average_systole,
valid_labels=systole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=systole_expert_weight,
disagreement_cutoff=0.01 # 0.01
)
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_diastole",
expert_predictions_matrix=diastole_expert_predictions_matrix,
masked_expert_predictions_matrix=diastole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_diastole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_diastole_masked_expert_predictions_matrix,
optimal_params=diastole_optimal_params,
average_distribution=average_diastole,
valid_labels=diastole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=diastole_expert_weight,
disagreement_cutoff=0.015 # diastole has about 50% more error
)
print()
print("Calculating training and validation set scores for reference")
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation")]:
errors = []
for patient in patient_ids:
prediction = final_predictions[patient-1]
if "final_systole" in prediction:
assert patient == regular_labels[patient-1, 0]
error1 = utils.CRSP(prediction["final_systole"], regular_labels[patient-1, 1])
errors.append(error1)
prediction["systole_crps_error"] = error1
error2 = utils.CRSP(prediction["final_diastole"], regular_labels[patient-1, 2])
errors.append(error2)
prediction["diastole_crps_error"] = error1
prediction["average_crps_error"] = 0.5*error1 + 0.5*error2
if len(errors)>0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print(" %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP))
validation_dict[set_name] = estimated_CRSP
else:
print(" %s kaggle loss: not calculated" % (string.rjust(set_name, 12)))
print("WARNING: both of the previous are overfitted!")
print()
print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2))
print()
print("dumping submission file to %s" % submission_path)
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d'%i for i in range(600)])
for prediction in final_predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in test_patients_indices:
if "final_diastole" not in prediction or "final_systole" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_diastole"].flatten()])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_systole"].flatten()])
print("submission file dumped")