def _create_label_matrix():
systole_valid_labels = np.array([
utils.cumulative_one_hot(v)
for v in data_loader.regular_labels[:, 1].flatten()
])
diastole_valid_labels = np.array([
utils.cumulative_one_hot(v)
for v in data_loader.regular_labels[:, 2].flatten()
])
return systole_valid_labels, diastole_valid_labels
def _create_label_matrix():
systole_valid_labels = np.array(
[utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:,1].flatten()])
diastole_valid_labels = np.array(
[utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:,2].flatten()])
return systole_valid_labels, diastole_valid_labels
pats_predicted = {set:[] for set in data_loader.patient_folders}
for prediction in metadata['predictions']:
pid = prediction['patient']
pset = data_loader.id_to_index_map[pid][0]
if _is_empty_prediction(prediction):
pats_not_predicted[pset].append(pid)
else:
pats_predicted[pset].append(pid)
return pats_not_predicted, pats_predicted
##### TTA AVERAGING ######
#========================#
average_systole = postprocess.make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:,1]]), axis=0))
average_diastole = postprocess.make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:,2]]), axis=0))
def generate_information_weight_matrix(expert_predictions, average_distribution, eps=1e-14, KL_weight = 1.0, cross_entropy_weight=1.0, expert_weights=None):
pdf = utils.cdf_to_pdf(expert_predictions)
average_pdf = utils.cdf_to_pdf(average_distribution)
average_pdf[average_pdf<=0] = np.min(average_pdf[average_pdf>0])/2 # KL is not defined when Q=0 and P is not
inside = pdf * (np.log(pdf) - np.log(average_pdf[None,None,:]))
inside[pdf<=0] = 0 # (xlog(x) of zero is zero)
KL_distance_from_average = np.sum(inside, axis=2) # (NUM_EXPERTS, NUM_VALIDATIONS)
assert np.isfinite(KL_distance_from_average).all()
clipped_predictions = np.clip(expert_predictions, 0.0, 1.0)
cross_entropy_per_sample = - ( average_distribution[None,None,:] * np.log( clipped_predictions+eps) +\
(1.-average_distribution[None,None,:]) * np.log(1.-clipped_predictions+eps) )
for prediction in metadata['predictions']:
pid = prediction['patient']
pset = data_loader.id_to_index_map[pid][0]
if _is_empty_prediction(prediction):
pats_not_predicted[pset].append(pid)
else:
pats_predicted[pset].append(pid)
return pats_not_predicted, pats_predicted
##### TTA AVERAGING ######
#========================#
average_systole = postprocess.make_monotone_distribution(
np.mean(np.array([
utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:, 1]
]),
axis=0))
average_diastole = postprocess.make_monotone_distribution(
np.mean(np.array([
utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:, 2]
]),
axis=0))
def generate_information_weight_matrix(expert_predictions,
average_distribution,
eps=1e-14,
KL_weight=1.0,
cross_entropy_weight=1.0,
expert_weights=None):
def merge_all_prediction_files(prediction_file_location = INTERMEDIATE_PREDICTIONS_PATH,
redo_tta = True):
submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
# calculate the average distribution
regular_labels = _load_file(_TRAIN_LABELS_PATH)
average_systole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,1]]), axis=0))
average_diastole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,2]]), axis=0))
ss_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_nomask.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360_gauss_longer.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm.pkl")
+glob.glob(prediction_file_location+"j6_2ch_96mm.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_framemax.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_96.pkl")
+glob.glob(prediction_file_location+"j6_4ch.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"j6_4ch_32mm_specialist.pkl")
+glob.glob(prediction_file_location+"j6_4ch_128mm_specialist.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80small_360_gauss_longer_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll_augzoombright.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_zoom.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_skew.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360.pkl")
)
fp_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_rellocframe.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_filtered.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_framemax_reg.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv_augzoombright_short.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered_discs.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale64small_filtered_longer.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright_betterdist.pkl")
+glob.glob(prediction_file_location+"je_os_segmentandintegrate_smartsigma_dropout.pkl")
)
everything = sorted([]
+glob.glob(prediction_file_location+"*.pkl")
)
expert_pkl_files = ss_expert_pkl_files + fp_expert_pkl_files
print("found %d/44 files" % len(expert_pkl_files))
"""
# filter expert_pkl_files
for file in expert_pkl_files[:]:
try:
with open(file, 'r') as f:
print "testing file",file.split('/')[-1]
data = pickle.load(f)
if 'predictions' not in data.keys():
expert_pkl_files.remove(file)
print " -> removed"
except:
print sys.exc_info()[0]
expert_pkl_files.remove(file)
print " -> removed"
"""
NUM_EXPERTS = len(expert_pkl_files)
NUM_VALIDATIONS = len(validation_patients_indices)
NUM_TESTS = len(test_patients_indices)
systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
test_systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
test_diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
for i,file in enumerate(expert_pkl_files):
with open(file, 'r') as f:
print()
print("loading file",file.split('/')[-1])
predictions = pickle.load(f)['predictions']
if redo_tta:
best_average_method = normalav
best_average_crps = utils.maxfloat
for average_method in [geomav,
normalav,
prodav,
weighted_geom_method
]:
calculate_tta_average(predictions, average_method, average_systole, average_diastole)
crps = get_validate_crps(predictions, regular_labels)
print(string.rjust(average_method.__name__,25),"->",crps)
if crps<best_average_crps:
best_average_method = average_method
best_average_crps = crps
print(" I choose you,", best_average_method.__name__)
calculate_tta_average(predictions, best_average_method, average_systole, average_diastole)
print(" validation loss:", get_validate_crps(predictions, regular_labels))
for j,patient in enumerate(validation_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
diastole_masked_expert_predictions_matrix[i,j] = False
for j,patient in enumerate(test_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
test_systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
test_systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
test_diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
test_diastole_masked_expert_predictions_matrix[i,j] = False
del predictions # can be LOADS of data
cv = [id-1 for id in validation_patients_indices]
systole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,1].flatten()])
systole_expert_weight, first_pass_sys_loss, systole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=systole_expert_predictions_matrix,
mask_matrix=systole_masked_expert_predictions_matrix,
labels=systole_valid_labels,
average_distribution=average_systole,
)
cv = [id-1 for id in validation_patients_indices]
diastole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,2].flatten()])
diastole_expert_weight, first_pass_dia_loss, diastole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=diastole_expert_predictions_matrix,
mask_matrix=diastole_masked_expert_predictions_matrix,
labels=diastole_valid_labels,
average_distribution=average_diastole,
)
# print the final weight of every expert
print(" Systole: Diastole: Name:")
for expert_name, systole_weight, diastole_weight in zip(expert_pkl_files, systole_expert_weight, diastole_expert_weight):
print(string.rjust("%.3f%%" % (100*systole_weight), 10), end=' ')
print(string.rjust("%.3f%%" % (100*diastole_weight), 10), end=' ')
print(expert_name.split('/')[-1])
print()
print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2))
print()
print()
print("Average the experts according to these weights to find the final distribution")
final_predictions = [{
"patient": i+1,
"final_systole": None,
"final_diastole": None
} for i in range(NUM_PATIENTS)]
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_systole",
expert_predictions_matrix=systole_expert_predictions_matrix,
masked_expert_predictions_matrix=systole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_systole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_systole_masked_expert_predictions_matrix,
optimal_params=systole_optimal_params,
average_distribution=average_systole,
valid_labels=systole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=systole_expert_weight,
disagreement_cutoff=0.01 # 0.01
)
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_diastole",
expert_predictions_matrix=diastole_expert_predictions_matrix,
masked_expert_predictions_matrix=diastole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_diastole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_diastole_masked_expert_predictions_matrix,
optimal_params=diastole_optimal_params,
average_distribution=average_diastole,
valid_labels=diastole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=diastole_expert_weight,
disagreement_cutoff=0.015 # diastole has about 50% more error
)
print()
print("Calculating training and validation set scores for reference")
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation")]:
errors = []
for patient in patient_ids:
prediction = final_predictions[patient-1]
if "final_systole" in prediction:
assert patient == regular_labels[patient-1, 0]
error1 = utils.CRSP(prediction["final_systole"], regular_labels[patient-1, 1])
errors.append(error1)
prediction["systole_crps_error"] = error1
error2 = utils.CRSP(prediction["final_diastole"], regular_labels[patient-1, 2])
errors.append(error2)
prediction["diastole_crps_error"] = error1
prediction["average_crps_error"] = 0.5*error1 + 0.5*error2
if len(errors)>0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print(" %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP))
validation_dict[set_name] = estimated_CRSP
else:
print(" %s kaggle loss: not calculated" % (string.rjust(set_name, 12)))
print("WARNING: both of the previous are overfitted!")
print()
print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2))
print()
print("dumping submission file to %s" % submission_path)
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d'%i for i in range(600)])
for prediction in final_predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in test_patients_indices:
if "final_diastole" not in prediction or "final_systole" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_diastole"].flatten()])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_systole"].flatten()])
print("submission file dumped")
def merge_all_prediction_files(prediction_file_location = INTERMEDIATE_PREDICTIONS_PATH,
redo_tta = True):
submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
# calculate the average distribution
regular_labels = _load_file(_TRAIN_LABELS_PATH)
average_systole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,1]]), axis=0))
average_diastole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,2]]), axis=0))
ss_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_nomask.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360_gauss_longer.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm.pkl")
+glob.glob(prediction_file_location+"j6_2ch_96mm.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_framemax.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_96.pkl")
+glob.glob(prediction_file_location+"j6_4ch.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"j6_4ch_32mm_specialist.pkl")
+glob.glob(prediction_file_location+"j6_4ch_128mm_specialist.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80small_360_gauss_longer_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll_augzoombright.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_zoom.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_skew.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360.pkl")
)
fp_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_rellocframe.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_filtered.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_framemax_reg.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv_augzoombright_short.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered_discs.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale64small_filtered_longer.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright_betterdist.pkl")
+glob.glob(prediction_file_location+"je_os_segmentandintegrate_smartsigma_dropout.pkl")
)
everything = sorted([]
+glob.glob(prediction_file_location+"*.pkl")
)
expert_pkl_files = ss_expert_pkl_files + fp_expert_pkl_files
print "found %d/44 files" % len(expert_pkl_files)
"""
# filter expert_pkl_files
for file in expert_pkl_files[:]:
try:
with open(file, 'r') as f:
print "testing file",file.split('/')[-1]
data = pickle.load(f)
if 'predictions' not in data.keys():
expert_pkl_files.remove(file)
print " -> removed"
except:
print sys.exc_info()[0]
expert_pkl_files.remove(file)
print " -> removed"
"""
NUM_EXPERTS = len(expert_pkl_files)
NUM_VALIDATIONS = len(validation_patients_indices)
NUM_TESTS = len(test_patients_indices)
systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
test_systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
test_diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
for i,file in enumerate(expert_pkl_files):
with open(file, 'r') as f:
print
print "loading file",file.split('/')[-1]
predictions = pickle.load(f)['predictions']
if redo_tta:
best_average_method = normalav
best_average_crps = utils.maxfloat
for average_method in [geomav,
normalav,
prodav,
weighted_geom_method
]:
calculate_tta_average(predictions, average_method, average_systole, average_diastole)
crps = get_validate_crps(predictions, regular_labels)
print string.rjust(average_method.__name__,25),"->",crps
if crps<best_average_crps:
best_average_method = average_method
best_average_crps = crps
print " I choose you,", best_average_method.__name__
calculate_tta_average(predictions, best_average_method, average_systole, average_diastole)
print " validation loss:", get_validate_crps(predictions, regular_labels)
for j,patient in enumerate(validation_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
diastole_masked_expert_predictions_matrix[i,j] = False
for j,patient in enumerate(test_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
test_systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
test_systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
test_diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
test_diastole_masked_expert_predictions_matrix[i,j] = False
del predictions # can be LOADS of data
cv = [id-1 for id in validation_patients_indices]
systole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,1].flatten()])
systole_expert_weight, first_pass_sys_loss, systole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=systole_expert_predictions_matrix,
mask_matrix=systole_masked_expert_predictions_matrix,
labels=systole_valid_labels,
average_distribution=average_systole,
)
cv = [id-1 for id in validation_patients_indices]
diastole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,2].flatten()])
diastole_expert_weight, first_pass_dia_loss, diastole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=diastole_expert_predictions_matrix,
mask_matrix=diastole_masked_expert_predictions_matrix,
labels=diastole_valid_labels,
average_distribution=average_diastole,
)
# print the final weight of every expert
print " Systole: Diastole: Name:"
for expert_name, systole_weight, diastole_weight in zip(expert_pkl_files, systole_expert_weight, diastole_expert_weight):
print string.rjust("%.3f%%" % (100*systole_weight), 10),
print string.rjust("%.3f%%" % (100*diastole_weight), 10),
print expert_name.split('/')[-1]
print
print "estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2)
print
print
print "Average the experts according to these weights to find the final distribution"
final_predictions = [{
"patient": i+1,
"final_systole": None,
"final_diastole": None
} for i in xrange(NUM_PATIENTS)]
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_systole",
expert_predictions_matrix=systole_expert_predictions_matrix,
masked_expert_predictions_matrix=systole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_systole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_systole_masked_expert_predictions_matrix,
optimal_params=systole_optimal_params,
average_distribution=average_systole,
valid_labels=systole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=systole_expert_weight,
disagreement_cutoff=0.01 # 0.01
)
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_diastole",
expert_predictions_matrix=diastole_expert_predictions_matrix,
masked_expert_predictions_matrix=diastole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_diastole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_diastole_masked_expert_predictions_matrix,
optimal_params=diastole_optimal_params,
average_distribution=average_diastole,
valid_labels=diastole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=diastole_expert_weight,
disagreement_cutoff=0.015 # diastole has about 50% more error
)
print
print "Calculating training and validation set scores for reference"
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation")]:
errors = []
for patient in patient_ids:
prediction = final_predictions[patient-1]
if "final_systole" in prediction:
assert patient == regular_labels[patient-1, 0]
error1 = utils.CRSP(prediction["final_systole"], regular_labels[patient-1, 1])
errors.append(error1)
prediction["systole_crps_error"] = error1
error2 = utils.CRSP(prediction["final_diastole"], regular_labels[patient-1, 2])
errors.append(error2)
prediction["diastole_crps_error"] = error1
prediction["average_crps_error"] = 0.5*error1 + 0.5*error2
if len(errors)>0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print " %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP)
validation_dict[set_name] = estimated_CRSP
else:
print " %s kaggle loss: not calculated" % (string.rjust(set_name, 12))
print "WARNING: both of the previous are overfitted!"
print
print "estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2)
print
print "dumping submission file to %s" % submission_path
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d'%i for i in xrange(600)])
for prediction in final_predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in test_patients_indices:
if "final_diastole" not in prediction or "final_systole" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_diastole"].flatten()])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_systole"].flatten()])
print "submission file dumped"
