def calculate_tta_average(predictions, average_method, average_systole, average_diastole):
already_printed = False
for prediction in predictions:
if prediction["systole"].size>0 and prediction["diastole"].size>0:
assert np.isfinite(prediction["systole"][None,:,:]).all()
assert np.isfinite(prediction["diastole"][None,:,:]).all()
prediction["systole_average"] = average_method(prediction["systole"][None,:,:], average=average_systole)
prediction["diastole_average"] = average_method(prediction["diastole"][None,:,:], average=average_diastole)
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
if not already_printed:
#print "WARNING: These distributions are not distributions"
already_printed = True
prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
prediction["systole_average"] = None
prediction["diastole_average"] = None
else:
# average distributions get zero weight later on
#prediction["systole_average"] = average_systole
#prediction["diastole_average"] = average_diastole
prediction["systole_average"] = None
prediction["diastole_average"] = None
def calculate_tta_average(predictions, average_method, average_systole, average_diastole):
already_printed = False
for prediction in predictions:
if prediction["systole"].size>0 and prediction["diastole"].size>0:
assert np.isfinite(prediction["systole"][None,:,:]).all()
assert np.isfinite(prediction["diastole"][None,:,:]).all()
prediction["systole_average"] = average_method(prediction["systole"][None,:,:], average=average_systole)
prediction["diastole_average"] = average_method(prediction["diastole"][None,:,:], average=average_diastole)
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
if not already_printed:
#print "WARNING: These distributions are not distributions"
already_printed = True
prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
prediction["systole_average"] = None
prediction["diastole_average"] = None
else:
# average distributions get zero weight later on
#prediction["systole_average"] = average_systole
#prediction["diastole_average"] = average_diastole
prediction["systole_average"] = None
prediction["diastole_average"] = None
def _make_valid_distributions(metadata):
for prediction in metadata['predictions']:
prediction["systole_average"] = postprocess.make_monotone_distribution(
prediction["systole_average"])
postprocess.test_if_valid_distribution(prediction["systole_average"])
prediction[
"diastole_average"] = postprocess.make_monotone_distribution(
prediction["diastole_average"])
postprocess.test_if_valid_distribution(prediction["diastole_average"])
def dump_predictions(result_ensemble, submission_path):
_, _, test_idx = _get_train_val_test_ids()
test_ids = np.where(test_idx)[0] + 1
preds_sys = result_ensemble["predictions_systole"][np.where(test_idx)]
preds_dia = result_ensemble["predictions_diastole"][np.where(test_idx)]
print "dumping submission file to %s" % submission_path
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d'%i for i in xrange(600)])
for pid, pat_pred_sys, pat_pred_dia in zip(test_ids, preds_sys, preds_dia):
pat_pred_sys = postprocess.make_monotone_distribution(pat_pred_sys)
pat_pred_dia = postprocess.make_monotone_distribution(pat_pred_dia)
csvwriter.writerow(["%d_Diastole" % pid] + ["%.18f" % p for p in pat_pred_dia.flatten()])
csvwriter.writerow(["%d_Systole" % pid] + ["%.18f" % p for p in pat_pred_sys.flatten()])
print "submission file dumped"
def _create_ensembles(metadatas):
# aggregate predictions and targets
mask, preds_sys, preds_dia = _create_prediction_matrix(metadatas)
targets_sys, targets_dia = _create_label_matrix()
print(mask.mean())
# initialise weights
nr_models = len(metadatas)
w_init = np.ones((nr_models, ), dtype='float32') / nr_models
# split data
_, val_idx, test_idx = _get_train_val_test_ids()
mask_val, mask_test = mask[:, val_idx], mask[:, test_idx]
targets_sys_val = targets_sys[val_idx[:len(targets_sys)], :]
targets_dia_val = targets_dia[val_idx[:len(targets_dia)], :]
preds_sys_val, preds_sys_test = preds_sys[:,
val_idx, :], preds_sys[:,
test_idx, :]
preds_dia_val, preds_dia_test = preds_dia[:,
val_idx, :], preds_dia[:,
test_idx, :]
## CREATE SYSTOLE AND DIASTOLE PREDICTION USING LOOXVAL
if DO_XVAL:
print("Making systole ensemble")
print(" Doing leave one patient out xval")
nr_val_patients = len(targets_sys_val)
train_errs_sys = []
val_errs_sys = []
w_iters_sys = []
for val_pid in range(nr_val_patients):
print(" - fold %d" % val_pid)
w_iter = _find_weights(
w_init,
np.hstack(
(preds_sys_val[:, :val_pid], preds_sys_val[:,
val_pid + 1:])),
np.vstack((targets_sys_val[:val_pid],
targets_sys_val[val_pid + 1:])),
np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
)
train_err = _eval_weights(
w_iter,
np.hstack(
(preds_sys_val[:, :val_pid], preds_sys_val[:,
val_pid + 1:])),
np.vstack((targets_sys_val[:val_pid],
targets_sys_val[val_pid + 1:])),
np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
)
val_err = _eval_weights(
w_iter,
preds_sys_val[:, val_pid:val_pid + 1],
targets_sys_val[val_pid:val_pid + 1],
mask_val[:, val_pid:val_pid + 1],
)
print(" train_err: %.4f, val_err: %.4f" %
(train_err, val_err))
train_errs_sys.append(train_err)
val_errs_sys.append(val_err)
w_iters_sys.append(w_iter)
expected_systole_loss = np.mean(val_errs_sys)
print(" average train err: %.4f" % np.mean(train_errs_sys))
print(" average valid err: %.4f" % np.mean(val_errs_sys))
print("Making diastole ensemble")
print(" Doing leave one patient out xval")
nr_val_patients = len(targets_dia_val)
train_errs_dia = []
val_errs_dia = []
w_iters_dia = []
for val_pid in range(nr_val_patients):
print(" - fold %d" % val_pid)
w_iter = _find_weights(
w_init,
np.hstack(
(preds_dia_val[:, :val_pid], preds_dia_val[:,
val_pid + 1:])),
np.vstack((targets_dia_val[:val_pid],
targets_dia_val[val_pid + 1:])),
np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
)
train_err = _eval_weights(
w_iter,
np.hstack(
(preds_dia_val[:, :val_pid], preds_dia_val[:,
val_pid + 1:])),
np.vstack((targets_dia_val[:val_pid],
targets_dia_val[val_pid + 1:])),
np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
)
val_err = _eval_weights(
w_iter,
preds_dia_val[:, val_pid:val_pid + 1],
targets_dia_val[val_pid:val_pid + 1],
mask_val[:, val_pid:val_pid + 1],
)
print(" train_err: %.4f, val_err: %.4f" %
(train_err, val_err))
train_errs_dia.append(train_err)
val_errs_dia.append(val_err)
w_iters_dia.append(w_iter)
expected_diastole_loss = np.mean(val_errs_dia)
print(" average train err: %.4f" % np.mean(train_errs_dia))
print(" average valid err: %.4f" % np.mean(val_errs_dia))
## MAKE ENSEMBLE USING THE FULL VALIDATION SET
print("Fitting weights on the entire validation set")
w_sys = _find_weights(w_init, preds_sys_val, targets_sys_val, mask_val)
w_dia = _find_weights(w_init, preds_dia_val, targets_dia_val, mask_val)
# Print the result
print()
print("dia sys ")
sort_key = lambda x: -x[1] - x[2]
for metadata, weight_sys, weight_dia in sorted(zip(
metadatas, w_sys, w_dia),
key=sort_key):
print("%4.1f %4.1f : %s" % (weight_sys * 100, weight_dia * 100,
metadata["configuration_file"]))
## SELECT THE TOP MODELS AND RETRAIN ONCE MORE
print()
print("Selecting the top %d models and retraining" % SELECT_TOP)
sorted_models, sorted_w_sys, sorted_w_dia = list(
zip(*sorted(zip(metadatas, w_sys, w_dia), key=sort_key)))
top_models = sorted_models[:SELECT_TOP]
w_init_sys_top = np.array(sorted_w_sys[:SELECT_TOP]) / np.array(
sorted_w_sys[:SELECT_TOP]).sum()
w_init_dia_top = np.array(sorted_w_dia[:SELECT_TOP]) / np.array(
sorted_w_dia[:SELECT_TOP]).sum()
mask_top, preds_sys_top, preds_dia_top = _create_prediction_matrix(
top_models)
mask_top_val, mask_top_test = mask_top[:, val_idx], mask_top[:, test_idx]
preds_sys_top_val, preds_sys_top_test = preds_sys_top[:,
val_idx, :], preds_sys_top[:,
test_idx, :]
preds_dia_top_val, preds_dia_top_test = preds_dia_top[:,
val_idx, :], preds_dia_top[:,
test_idx, :]
w_sys_top = _find_weights(w_init_sys_top, preds_sys_top_val,
targets_sys_val, mask_top_val)
w_dia_top = _find_weights(w_init_dia_top, preds_dia_top_val,
targets_dia_val, mask_top_val)
sys_err = _eval_weights(w_sys_top, preds_sys_top_val, targets_sys_val,
mask_top_val)
dia_err = _eval_weights(w_dia_top, preds_dia_top_val, targets_dia_val,
mask_top_val)
print()
print("dia sys ")
sort_key = lambda x: -x[1] - x[2]
for metadata, weight_sys, weight_dia in zip(top_models, w_sys_top,
w_dia_top):
print("%4.1f %4.1f : %s" % (weight_sys * 100, weight_dia * 100,
metadata["configuration_file"]))
print("Final scores on the validation set:")
print(" systole: %.4f" % sys_err)
print(" diastole: %.4f" % dia_err)
print(" average: %.4f" % ((sys_err + dia_err) / 2.0))
if DO_XVAL:
print("Expected leaderboard scores:")
print(" systole: %.4f" % expected_systole_loss)
print(" diastole: %.4f" % expected_diastole_loss)
print(" average: %.4f" %
((expected_systole_loss + expected_diastole_loss) / 2.0))
## COMPUTE TEST PREDICTIONS
test_ids = np.where(test_idx)[0] + 1
preds_sys = _compute_predictions_ensemble(w_sys_top, preds_sys_top_test,
mask_top_test)
preds_dia = _compute_predictions_ensemble(w_dia_top, preds_dia_top_test,
mask_top_test)
submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
print("dumping submission file to %s" % submission_path)
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d' % i for i in range(600)])
for pid, pat_pred_sys, pat_pred_dia in zip(test_ids, preds_sys,
preds_dia):
pat_pred_sys = postprocess.make_monotone_distribution(pat_pred_sys)
pat_pred_dia = postprocess.make_monotone_distribution(pat_pred_dia)
csvwriter.writerow(["%d_Diastole" % pid] +
["%.18f" % p for p in pat_pred_dia.flatten()])
csvwriter.writerow(["%d_Systole" % pid] +
["%.18f" % p for p in pat_pred_sys.flatten()])
print("submission file dumped")
def _make_valid_distributions(metadata):
for prediction in metadata['predictions']:
prediction["systole_average"] = postprocess.make_monotone_distribution(prediction["systole_average"])
postprocess.test_if_valid_distribution(prediction["systole_average"])
prediction["diastole_average"] = postprocess.make_monotone_distribution(prediction["diastole_average"])
postprocess.test_if_valid_distribution(prediction["diastole_average"])
pats_predicted = {set:[] for set in data_loader.patient_folders}
for prediction in metadata['predictions']:
pid = prediction['patient']
pset = data_loader.id_to_index_map[pid][0]
if _is_empty_prediction(prediction):
pats_not_predicted[pset].append(pid)
else:
pats_predicted[pset].append(pid)
return pats_not_predicted, pats_predicted
##### TTA AVERAGING ######
#========================#
average_systole = postprocess.make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:,1]]), axis=0))
average_diastole = postprocess.make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:,2]]), axis=0))
def generate_information_weight_matrix(expert_predictions, average_distribution, eps=1e-14, KL_weight = 1.0, cross_entropy_weight=1.0, expert_weights=None):
pdf = utils.cdf_to_pdf(expert_predictions)
average_pdf = utils.cdf_to_pdf(average_distribution)
average_pdf[average_pdf<=0] = np.min(average_pdf[average_pdf>0])/2 # KL is not defined when Q=0 and P is not
inside = pdf * (np.log(pdf) - np.log(average_pdf[None,None,:]))
inside[pdf<=0] = 0 # (xlog(x) of zero is zero)
KL_distance_from_average = np.sum(inside, axis=2) # (NUM_EXPERTS, NUM_VALIDATIONS)
assert np.isfinite(KL_distance_from_average).all()
clipped_predictions = np.clip(expert_predictions, 0.0, 1.0)
cross_entropy_per_sample = - ( average_distribution[None,None,:] * np.log( clipped_predictions+eps) +\
(1.-average_distribution[None,None,:]) * np.log(1.-clipped_predictions+eps) )
def predict_slice_model(expid, outfile, mfile=None):
metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)
if theano.config.optimizer != "fast_run":
print "WARNING: not running in fast mode!"
print "Build model"
interface_layers = config().build_model()
output_layers = interface_layers["outputs"]
input_layers = interface_layers["inputs"]
top_layer = lasagne.layers.MergeLayer(
incomings=output_layers.values()
)
_check_slicemodel(input_layers)
# Print the architecture
_print_architecture(top_layer)
xs_shared = {
key: lasagne.utils.shared_empty(dim=len(l_in.output_shape), dtype='float32') for (key, l_in) in input_layers.iteritems()
}
idx = T.lscalar('idx')
givens = dict()
for key in input_layers.keys():
if key=="sunny":
givens[input_layers[key].input_var] = xs_shared[key][idx*config().sunny_batch_size:(idx+1)*config().sunny_batch_size]
else:
givens[input_layers[key].input_var] = xs_shared[key][idx*config().batch_size:(idx+1)*config().batch_size]
network_outputs = [
lasagne.layers.helper.get_output(network_output_layer, deterministic=True)
for network_output_layer in output_layers.values()
]
iter_test = theano.function([idx], network_outputs + theano_printer.get_the_stuff_to_print(),
givens=givens, on_unused_input="ignore",
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
)
print "Load model parameters for resuming"
resume_metadata = np.load(metadata_path)
lasagne.layers.set_all_param_values(top_layer, resume_metadata['param_values'])
num_batches_chunk = config().batches_per_chunk
num_batches = get_number_of_test_batches()
num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))
chunks_train_idcs = range(1, num_chunks+1)
create_test_gen = partial(config().create_test_gen,
required_input_keys = xs_shared.keys(),
required_output_keys = ["patients", "slices"],
)
print "Generate predictions with this model"
start_time = time.time()
prev_time = start_time
predictions = [{"patient": i+1,
"slices": {
slice_id: {
"systole": np.zeros((0,600)),
"diastole": np.zeros((0,600))
} for slice_id in data_loader.get_slice_ids_for_patient(i+1)
}
} for i in xrange(NUM_PATIENTS)]
# Loop over data and generate predictions
for e, test_data in izip(itertools.count(start=1), buffering.buffered_gen_threaded(create_test_gen())):
print " load testing data onto GPU"
for key in xs_shared:
xs_shared[key].set_value(test_data["input"][key])
patient_ids = test_data["output"]["patients"]
slice_ids = test_data["output"]["slices"]
print " patients:", " ".join(map(str, patient_ids))
print " chunk %d/%d" % (e, num_chunks)
for b in xrange(num_batches_chunk):
iter_result = iter_test(b)
network_outputs = tuple(iter_result[:len(output_layers)])
network_outputs_dict = {output_layers.keys()[i]: network_outputs[i] for i in xrange(len(output_layers))}
kaggle_systoles, kaggle_diastoles = config().postprocess(network_outputs_dict)
kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype('float64'), kaggle_diastoles.astype('float64')
for idx, (patient_id, slice_id) in enumerate(
zip(patient_ids[b*config().batch_size:(b+1)*config().batch_size],
slice_ids[b*config().batch_size:(b+1)*config().batch_size])):
if patient_id != 0:
index = patient_id-1
patient_data = predictions[index]
assert patient_id==patient_data["patient"]
patient_slice_data = patient_data["slices"][slice_id]
patient_slice_data["systole"] = np.concatenate((patient_slice_data["systole"], kaggle_systoles[idx:idx+1,:]),axis=0)
patient_slice_data["diastole"] = np.concatenate((patient_slice_data["diastole"], kaggle_diastoles[idx:idx+1,:]),axis=0)
now = time.time()
time_since_start = now - start_time
time_since_prev = now - prev_time
prev_time = now
est_time_left = time_since_start * (float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
eta = datetime.now() + timedelta(seconds=est_time_left)
eta_str = eta.strftime("%c")
print " %s since start (%.2f s)" % (utils.hms(time_since_start), time_since_prev)
print " estimated %s to go (ETA: %s)" % (utils.hms(est_time_left), eta_str)
print
# Average predictions
already_printed = False
for prediction in predictions:
for prediction_slice_id in prediction["slices"]:
prediction_slice = prediction["slices"][prediction_slice_id]
if prediction_slice["systole"].size>0 and prediction_slice["diastole"].size>0:
average_method = getattr(config(), 'tta_average_method', partial(np.mean, axis=0))
prediction_slice["systole_average"] = average_method(prediction_slice["systole"])
prediction_slice["diastole_average"] = average_method(prediction_slice["diastole"])
try:
test_if_valid_distribution(prediction_slice["systole_average"])
test_if_valid_distribution(prediction_slice["diastole_average"])
except:
if not already_printed:
print "WARNING: These distributions are not distributions"
already_printed = True
prediction_slice["systole_average"] = make_monotone_distribution(prediction_slice["systole_average"])
prediction_slice["diastole_average"] = make_monotone_distribution(prediction_slice["diastole_average"])
print "Calculating training and validation set scores for reference"
# Add CRPS scores to the predictions
# Iterate over train and validation sets
for patient_ids, set_name in [(validation_patients_indices, "validation"),
(train_patients_indices, "train")]:
# Iterate over patients in the set
for patient in patient_ids:
prediction = predictions[patient-1]
# Iterate over the slices
for slice_id in prediction["slices"]:
prediction_slice = prediction["slices"][slice_id]
if "systole_average" in prediction_slice:
assert patient == regular_labels[patient-1, 0]
error_sys = CRSP(prediction_slice["systole_average"], regular_labels[patient-1, 1])
prediction_slice["systole_CRPS"] = error_sys
prediction_slice["target_systole"] = regular_labels[patient-1, 1]
error_dia = CRSP(prediction_slice["diastole_average"], regular_labels[patient-1, 2])
prediction_slice["diastole_CRPS"] = error_dia
prediction_slice["target_diastole"] = regular_labels[patient-1, 2]
prediction_slice["CRPS"] = 0.5 * error_sys + 0.5 * error_dia
print "dumping prediction file to %s" % outfile
with open(outfile, 'w') as f:
pickle.dump({
'metadata_path': metadata_path,
'configuration_file': config().__name__,
'git_revision_hash': utils.get_git_revision_hash(),
'experiment_id': expid,
'time_since_start': time_since_start,
'param_values': lasagne.layers.get_all_param_values(top_layer),
'predictions_per_slice': predictions,
}, f, pickle.HIGHEST_PROTOCOL)
print "prediction file dumped"
return
def _create_ensembles(metadatas):
# aggregate predictions and targets
mask, preds_sys, preds_dia = _create_prediction_matrix(metadatas)
targets_sys, targets_dia = _create_label_matrix()
print mask.mean()
# initialise weights
nr_models = len(metadatas)
w_init = np.ones((nr_models,), dtype='float32') / nr_models
# split data
_, val_idx, test_idx = _get_train_val_test_ids()
mask_val, mask_test = mask[:, val_idx], mask[:, test_idx]
targets_sys_val = targets_sys[val_idx[:len(targets_sys)], :]
targets_dia_val = targets_dia[val_idx[:len(targets_dia)], :]
preds_sys_val, preds_sys_test = preds_sys[:, val_idx, :], preds_sys[:, test_idx, :]
preds_dia_val, preds_dia_test = preds_dia[:, val_idx, :], preds_dia[:, test_idx, :]
## CREATE SYSTOLE AND DIASTOLE PREDICTION USING LOOXVAL
if DO_XVAL:
print "Making systole ensemble"
print " Doing leave one patient out xval"
nr_val_patients = len(targets_sys_val)
train_errs_sys = []
val_errs_sys = []
w_iters_sys = []
for val_pid in xrange(nr_val_patients):
print " - fold %d" % val_pid
w_iter = _find_weights(
w_init,
np.hstack((preds_sys_val[:, :val_pid], preds_sys_val[:, val_pid + 1:])),
np.vstack((targets_sys_val[:val_pid], targets_sys_val[val_pid + 1:])),
np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
)
train_err = _eval_weights(
w_iter,
np.hstack((preds_sys_val[:, :val_pid], preds_sys_val[:, val_pid + 1:])),
np.vstack((targets_sys_val[:val_pid], targets_sys_val[val_pid + 1:])),
np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
)
val_err = _eval_weights(w_iter,
preds_sys_val[:, val_pid:val_pid + 1],
targets_sys_val[val_pid:val_pid + 1],
mask_val[:, val_pid:val_pid + 1],
)
print " train_err: %.4f, val_err: %.4f" % (train_err, val_err)
train_errs_sys.append(train_err)
val_errs_sys.append(val_err)
w_iters_sys.append(w_iter)
expected_systole_loss = np.mean(val_errs_sys)
print " average train err: %.4f" % np.mean(train_errs_sys)
print " average valid err: %.4f" % np.mean(val_errs_sys)
print "Making diastole ensemble"
print " Doing leave one patient out xval"
nr_val_patients = len(targets_dia_val)
train_errs_dia = []
val_errs_dia = []
w_iters_dia = []
for val_pid in xrange(nr_val_patients):
print " - fold %d" % val_pid
w_iter = _find_weights(
w_init,
np.hstack((preds_dia_val[:, :val_pid], preds_dia_val[:, val_pid + 1:])),
np.vstack((targets_dia_val[:val_pid], targets_dia_val[val_pid + 1:])),
np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
)
train_err = _eval_weights(
w_iter,
np.hstack((preds_dia_val[:, :val_pid], preds_dia_val[:, val_pid + 1:])),
np.vstack((targets_dia_val[:val_pid], targets_dia_val[val_pid + 1:])),
np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
)
val_err = _eval_weights(w_iter,
preds_dia_val[:, val_pid:val_pid + 1],
targets_dia_val[val_pid:val_pid + 1],
mask_val[:, val_pid:val_pid + 1],
)
print " train_err: %.4f, val_err: %.4f" % (train_err, val_err)
train_errs_dia.append(train_err)
val_errs_dia.append(val_err)
w_iters_dia.append(w_iter)
expected_diastole_loss = np.mean(val_errs_dia)
print " average train err: %.4f" % np.mean(train_errs_dia)
print " average valid err: %.4f" % np.mean(val_errs_dia)
## MAKE ENSEMBLE USING THE FULL VALIDATION SET
print "Fitting weights on the entire validation set"
w_sys = _find_weights(w_init, preds_sys_val, targets_sys_val, mask_val)
w_dia = _find_weights(w_init, preds_dia_val, targets_dia_val, mask_val)
# Print the result
print
print "dia sys "
sort_key = lambda x: - x[1] - x[2]
for metadata, weight_sys, weight_dia in sorted(zip(metadatas, w_sys, w_dia), key=sort_key):
print "%4.1f %4.1f : %s" % (weight_sys * 100, weight_dia * 100, metadata["configuration_file"])
## SELECT THE TOP MODELS AND RETRAIN ONCE MORE
print
print "Selecting the top %d models and retraining" % SELECT_TOP
sorted_models, sorted_w_sys, sorted_w_dia = zip(*sorted(zip(metadatas, w_sys, w_dia), key=sort_key))
top_models = sorted_models[:SELECT_TOP]
w_init_sys_top = np.array(sorted_w_sys[:SELECT_TOP]) / np.array(sorted_w_sys[:SELECT_TOP]).sum()
w_init_dia_top = np.array(sorted_w_dia[:SELECT_TOP]) / np.array(sorted_w_dia[:SELECT_TOP]).sum()
mask_top, preds_sys_top, preds_dia_top = _create_prediction_matrix(top_models)
mask_top_val, mask_top_test = mask_top[:, val_idx], mask_top[:, test_idx]
preds_sys_top_val, preds_sys_top_test = preds_sys_top[:, val_idx, :], preds_sys_top[:, test_idx, :]
preds_dia_top_val, preds_dia_top_test = preds_dia_top[:, val_idx, :], preds_dia_top[:, test_idx, :]
w_sys_top = _find_weights(w_init_sys_top, preds_sys_top_val, targets_sys_val, mask_top_val)
w_dia_top = _find_weights(w_init_dia_top, preds_dia_top_val, targets_dia_val, mask_top_val)
sys_err = _eval_weights(w_sys_top, preds_sys_top_val, targets_sys_val, mask_top_val)
dia_err = _eval_weights(w_dia_top, preds_dia_top_val, targets_dia_val, mask_top_val)
print
print "dia sys "
sort_key = lambda x: - x[1] - x[2]
for metadata, weight_sys, weight_dia in zip(top_models, w_sys_top, w_dia_top):
print "%4.1f %4.1f : %s" % (weight_sys * 100, weight_dia * 100, metadata["configuration_file"])
print "Final scores on the validation set:"
print " systole: %.4f" % sys_err
print " diastole: %.4f" % dia_err
print " average: %.4f" % ((sys_err + dia_err) / 2.0)
if DO_XVAL:
print "Expected leaderboard scores:"
print " systole: %.4f" % expected_systole_loss
print " diastole: %.4f" % expected_diastole_loss
print " average: %.4f" % ((expected_systole_loss + expected_diastole_loss) / 2.0)
## COMPUTE TEST PREDICTIONS
test_ids = np.where(test_idx)[0] + 1
preds_sys = _compute_predictions_ensemble(w_sys_top, preds_sys_top_test, mask_top_test)
preds_dia = _compute_predictions_ensemble(w_dia_top, preds_dia_top_test, mask_top_test)
submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
print "dumping submission file to %s" % submission_path
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d' % i for i in xrange(600)])
for pid, pat_pred_sys, pat_pred_dia in zip(test_ids, preds_sys, preds_dia):
pat_pred_sys = postprocess.make_monotone_distribution(pat_pred_sys)
pat_pred_dia = postprocess.make_monotone_distribution(pat_pred_dia)
csvwriter.writerow(["%d_Diastole" % pid] + ["%.18f" % p for p in pat_pred_dia.flatten()])
csvwriter.writerow(["%d_Systole" % pid] + ["%.18f" % p for p in pat_pred_sys.flatten()])
print "submission file dumped"
def predict_model(expid, mfile=None):
metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)
prediction_path = INTERMEDIATE_PREDICTIONS_PATH + "%s.pkl" % expid
submission_path = SUBMISSION_PATH + "%s.csv" % expid
if theano.config.optimizer != "fast_run":
print("WARNING: not running in fast mode!")
print("Using")
print(" %s" % metadata_path)
print("To generate")
print(" %s" % prediction_path)
print(" %s" % submission_path)
print("Build model")
interface_layers = config().build_model()
output_layers = interface_layers["outputs"]
input_layers = interface_layers["inputs"]
top_layer = lasagne.layers.MergeLayer(
incomings=list(output_layers.values()))
all_layers = lasagne.layers.get_all_layers(top_layer)
num_params = lasagne.layers.count_params(top_layer)
print(" number of parameters: %d" % num_params)
print(string.ljust(" layer output shapes:", 36), end=' ')
print(string.ljust("#params:", 10), end=' ')
print("output shape:")
for layer in all_layers[:-1]:
name = string.ljust(layer.__class__.__name__, 32)
num_param = sum(
[np.prod(p.get_value().shape) for p in layer.get_params()])
num_param = string.ljust(num_param.__str__(), 10)
print(" %s %s %s" % (name, num_param, layer.output_shape))
xs_shared = {
key: lasagne.utils.shared_empty(dim=len(l_in.output_shape),
dtype='float32')
for (key, l_in) in input_layers.items()
}
idx = T.lscalar('idx')
givens = dict()
for key in list(input_layers.keys()):
if key == "sunny":
givens[input_layers[key].input_var] = xs_shared[key][idx * config(
).sunny_batch_size:(idx + 1) * config().sunny_batch_size]
else:
givens[input_layers[key].
input_var] = xs_shared[key][idx *
config().batch_size:(idx + 1) *
config().batch_size]
network_outputs = [
lasagne.layers.helper.get_output(network_output_layer,
deterministic=True)
for network_output_layer in list(output_layers.values())
]
iter_test = theano.function(
[idx],
network_outputs + theano_printer.get_the_stuff_to_print(),
givens=givens,
on_unused_input="ignore",
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
)
print("Load model parameters for resuming")
resume_metadata = np.load(metadata_path)
lasagne.layers.set_all_param_values(top_layer,
resume_metadata['param_values'])
num_batches_chunk = config().batches_per_chunk
num_batches = get_number_of_test_batches()
num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))
chunks_train_idcs = list(range(1, num_chunks + 1))
data_loader.filter_patient_folders()
create_test_gen = partial(
config().create_test_gen,
required_input_keys=list(xs_shared.keys()),
required_output_keys=[
"patients", "classification_correction_function"
],
)
print("Generate predictions with this model")
start_time = time.time()
prev_time = start_time
predictions = [{
"patient": i + 1,
"systole": np.zeros((0, 600)),
"diastole": np.zeros((0, 600))
} for i in range(NUM_PATIENTS)]
for e, test_data in zip(itertools.count(start=1),
buffering.buffered_gen_threaded(
create_test_gen())):
print(" load testing data onto GPU")
for key in xs_shared:
xs_shared[key].set_value(test_data["input"][key])
patient_ids = test_data["output"]["patients"]
classification_correction = test_data["output"][
"classification_correction_function"]
print(" patients:", " ".join(map(str, patient_ids)))
print(" chunk %d/%d" % (e, num_chunks))
for b in range(num_batches_chunk):
iter_result = iter_test(b)
network_outputs = tuple(iter_result[:len(output_layers)])
network_outputs_dict = {
list(output_layers.keys())[i]: network_outputs[i]
for i in range(len(output_layers))
}
kaggle_systoles, kaggle_diastoles = config().postprocess(
network_outputs_dict)
kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype(
'float64'), kaggle_diastoles.astype('float64')
for idx, patient_id in enumerate(
patient_ids[b * config().batch_size:(b + 1) *
config().batch_size]):
if patient_id != 0:
index = patient_id - 1
patient_data = predictions[index]
assert patient_id == patient_data["patient"]
kaggle_systole = kaggle_systoles[idx:idx + 1, :]
kaggle_diastole = kaggle_diastoles[idx:idx + 1, :]
assert np.isfinite(kaggle_systole).all() and np.isfinite(
kaggle_systole).all()
kaggle_systole = classification_correction[
b * config().batch_size + idx](kaggle_systole)
kaggle_diastole = classification_correction[
b * config().batch_size + idx](kaggle_diastole)
assert np.isfinite(kaggle_systole).all() and np.isfinite(
kaggle_systole).all()
patient_data["systole"] = np.concatenate(
(patient_data["systole"], kaggle_systole), axis=0)
patient_data["diastole"] = np.concatenate(
(patient_data["diastole"], kaggle_diastole), axis=0)
now = time.time()
time_since_start = now - start_time
time_since_prev = now - prev_time
prev_time = now
est_time_left = time_since_start * (
float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
eta = datetime.now() + timedelta(seconds=est_time_left)
eta_str = eta.strftime("%c")
print(" %s since start (%.2f s)" %
(utils.hms(time_since_start), time_since_prev))
print(" estimated %s to go (ETA: %s)" %
(utils.hms(est_time_left), eta_str))
print()
already_printed = False
for prediction in predictions:
if prediction["systole"].size > 0 and prediction["diastole"].size > 0:
average_method = getattr(config(), 'tta_average_method',
partial(np.mean, axis=0))
prediction["systole_average"] = average_method(
prediction["systole"])
prediction["diastole_average"] = average_method(
prediction["diastole"])
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
if not already_printed:
print("WARNING: These distributions are not distributions")
already_printed = True
prediction["systole_average"] = make_monotone_distribution(
prediction["systole_average"])
prediction["diastole_average"] = make_monotone_distribution(
prediction["diastole_average"])
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
print("Calculating training and validation set scores for reference")
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation"),
(train_patients_indices, "train")]:
errors = []
for patient in patient_ids:
prediction = predictions[patient - 1]
if "systole_average" in prediction:
assert patient == regular_labels[patient - 1, 0]
error = CRSP(prediction["systole_average"],
regular_labels[patient - 1, 1])
errors.append(error)
error = CRSP(prediction["diastole_average"],
regular_labels[patient - 1, 2])
errors.append(error)
if len(errors) > 0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print(" %s kaggle loss: %f" %
(string.rjust(set_name, 12), estimated_CRSP))
validation_dict[set_name] = estimated_CRSP
else:
print(" %s kaggle loss: not calculated" %
(string.rjust(set_name, 12)))
print("dumping prediction file to %s" % prediction_path)
with open(prediction_path, 'w') as f:
pickle.dump(
{
'metadata_path': metadata_path,
'prediction_path': prediction_path,
'submission_path': submission_path,
'configuration_file': config().__name__,
'git_revision_hash': utils.get_git_revision_hash(),
'experiment_id': expid,
'time_since_start': time_since_start,
'param_values': lasagne.layers.get_all_param_values(top_layer),
'predictions': predictions,
'validation_errors': validation_dict,
}, f, pickle.HIGHEST_PROTOCOL)
print("prediction file dumped")
print("dumping submission file to %s" % submission_path)
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d' % i for i in range(600)])
for prediction in predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in data_loader.test_patients_indices:
if "diastole_average" not in prediction or "systole_average" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + [
"%.18f" % p
for p in prediction["diastole_average"].flatten()
])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + [
"%.18f" % p
for p in prediction["systole_average"].flatten()
])
print("submission file dumped")
return
def generate_final_predictions(
final_predictions,
prediction_tag,
expert_predictions_matrix,
masked_expert_predictions_matrix,
test_expert_predictions_matrix,
test_masked_expert_predictions_matrix,
optimal_params,
average_distribution,
valid_labels,
expert_pkl_files,
expert_weight,
disagreement_cutoff):
cache_dict = dict()
for final_prediction in final_predictions:
patient_id = final_prediction['patient']
if patient_id in train_patients_indices:
continue
print(" final prediction of patient %d" % patient_id)
reoptimized = False
# get me the data for this patient
# (NUM_EXPERTS, 600)
if patient_id in validation_patients_indices:
idx = validation_patients_indices.index(patient_id)
prediction_matrix = expert_predictions_matrix[:, idx, :]
mask_matrix = masked_expert_predictions_matrix[:, idx]
elif patient_id in test_patients_indices:
idx = test_patients_indices.index(patient_id)
prediction_matrix = test_expert_predictions_matrix[:, idx, :]
mask_matrix = test_masked_expert_predictions_matrix[:, idx]
else:
raise "This patient is neither train, validation or test?"
# Is there an expert in the set, which did not predict this patient?
# if so, re-optimize our weights on the validation set!
# so, if a model is unavailble, but should be available, re-optimize!
if np.logical_and(np.logical_not(mask_matrix), (expert_weight!=0.0)).any():
if hash_mask(mask_matrix) in cache_dict:
(optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
else:
expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
expert_mask=mask_matrix,
prediction_matrix=expert_predictions_matrix,
mask_matrix=masked_expert_predictions_matrix,
labels=valid_labels,
average_distribution=average_distribution,
)
cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
reoptimized = True
else:
optimal_params_for_this_patient = optimal_params
while True: # do-while: break condition at the end
last_mask_matrix = mask_matrix
final_prediction[prediction_tag] = optimize_expert_weights(
expert_predictions=prediction_matrix[:,None,:],
mask_matrix=mask_matrix[:,None],
average_distribution=average_distribution,
do_optimization=False,
optimal_params=optimal_params_for_this_patient,
)
## find the disagreement between models
# and remove the ones who disagree too much with the average
disagreement = find_disagreement(
expert_predictions=prediction_matrix,
mask_matrix=mask_matrix,
ground_truth=final_prediction[prediction_tag]
)
mask_matrix = (disagreement<disagreement_cutoff)
## RETRAIN THESE ENSEMBLE WEIGHTS ON THE VALIDATION SET IF NEEDED!
if ((last_mask_matrix!=mask_matrix).any()
and np.sum(mask_matrix)!=0):
if hash_mask(mask_matrix) in cache_dict:
(optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
else:
expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
expert_mask=mask_matrix,
prediction_matrix=expert_predictions_matrix,
mask_matrix=masked_expert_predictions_matrix,
labels=valid_labels,
average_distribution=average_distribution,
)
cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
reoptimized = True
continue
else:
break
try:
test_if_valid_distribution(final_prediction[prediction_tag])
except:
final_prediction[prediction_tag] = make_monotone_distribution(final_prediction[prediction_tag])
test_if_valid_distribution(final_prediction[prediction_tag])
if reoptimized:
print(" Weight: Name:")
for expert_name, weight in zip(expert_pkl_files, expert_weight):
if weight>0.:
print(string.rjust("%.3f%%" % (100*weight), 12), end=' ')
print(expert_name.split('/')[-1])
def merge_all_prediction_files(prediction_file_location = INTERMEDIATE_PREDICTIONS_PATH,
redo_tta = True):
submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
# calculate the average distribution
regular_labels = _load_file(_TRAIN_LABELS_PATH)
average_systole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,1]]), axis=0))
average_diastole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,2]]), axis=0))
ss_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_nomask.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360_gauss_longer.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm.pkl")
+glob.glob(prediction_file_location+"j6_2ch_96mm.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_framemax.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_96.pkl")
+glob.glob(prediction_file_location+"j6_4ch.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"j6_4ch_32mm_specialist.pkl")
+glob.glob(prediction_file_location+"j6_4ch_128mm_specialist.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80small_360_gauss_longer_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll_augzoombright.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_zoom.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_skew.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360.pkl")
)
fp_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_rellocframe.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_filtered.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_framemax_reg.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv_augzoombright_short.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered_discs.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale64small_filtered_longer.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright_betterdist.pkl")
+glob.glob(prediction_file_location+"je_os_segmentandintegrate_smartsigma_dropout.pkl")
)
everything = sorted([]
+glob.glob(prediction_file_location+"*.pkl")
)
expert_pkl_files = ss_expert_pkl_files + fp_expert_pkl_files
print("found %d/44 files" % len(expert_pkl_files))
"""
# filter expert_pkl_files
for file in expert_pkl_files[:]:
try:
with open(file, 'r') as f:
print "testing file",file.split('/')[-1]
data = pickle.load(f)
if 'predictions' not in data.keys():
expert_pkl_files.remove(file)
print " -> removed"
except:
print sys.exc_info()[0]
expert_pkl_files.remove(file)
print " -> removed"
"""
NUM_EXPERTS = len(expert_pkl_files)
NUM_VALIDATIONS = len(validation_patients_indices)
NUM_TESTS = len(test_patients_indices)
systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
test_systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
test_diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
for i,file in enumerate(expert_pkl_files):
with open(file, 'r') as f:
print()
print("loading file",file.split('/')[-1])
predictions = pickle.load(f)['predictions']
if redo_tta:
best_average_method = normalav
best_average_crps = utils.maxfloat
for average_method in [geomav,
normalav,
prodav,
weighted_geom_method
]:
calculate_tta_average(predictions, average_method, average_systole, average_diastole)
crps = get_validate_crps(predictions, regular_labels)
print(string.rjust(average_method.__name__,25),"->",crps)
if crps<best_average_crps:
best_average_method = average_method
best_average_crps = crps
print(" I choose you,", best_average_method.__name__)
calculate_tta_average(predictions, best_average_method, average_systole, average_diastole)
print(" validation loss:", get_validate_crps(predictions, regular_labels))
for j,patient in enumerate(validation_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
diastole_masked_expert_predictions_matrix[i,j] = False
for j,patient in enumerate(test_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
test_systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
test_systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
test_diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
test_diastole_masked_expert_predictions_matrix[i,j] = False
del predictions # can be LOADS of data
cv = [id-1 for id in validation_patients_indices]
systole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,1].flatten()])
systole_expert_weight, first_pass_sys_loss, systole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=systole_expert_predictions_matrix,
mask_matrix=systole_masked_expert_predictions_matrix,
labels=systole_valid_labels,
average_distribution=average_systole,
)
cv = [id-1 for id in validation_patients_indices]
diastole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,2].flatten()])
diastole_expert_weight, first_pass_dia_loss, diastole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=diastole_expert_predictions_matrix,
mask_matrix=diastole_masked_expert_predictions_matrix,
labels=diastole_valid_labels,
average_distribution=average_diastole,
)
# print the final weight of every expert
print(" Systole: Diastole: Name:")
for expert_name, systole_weight, diastole_weight in zip(expert_pkl_files, systole_expert_weight, diastole_expert_weight):
print(string.rjust("%.3f%%" % (100*systole_weight), 10), end=' ')
print(string.rjust("%.3f%%" % (100*diastole_weight), 10), end=' ')
print(expert_name.split('/')[-1])
print()
print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2))
print()
print()
print("Average the experts according to these weights to find the final distribution")
final_predictions = [{
"patient": i+1,
"final_systole": None,
"final_diastole": None
} for i in range(NUM_PATIENTS)]
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_systole",
expert_predictions_matrix=systole_expert_predictions_matrix,
masked_expert_predictions_matrix=systole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_systole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_systole_masked_expert_predictions_matrix,
optimal_params=systole_optimal_params,
average_distribution=average_systole,
valid_labels=systole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=systole_expert_weight,
disagreement_cutoff=0.01 # 0.01
)
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_diastole",
expert_predictions_matrix=diastole_expert_predictions_matrix,
masked_expert_predictions_matrix=diastole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_diastole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_diastole_masked_expert_predictions_matrix,
optimal_params=diastole_optimal_params,
average_distribution=average_diastole,
valid_labels=diastole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=diastole_expert_weight,
disagreement_cutoff=0.015 # diastole has about 50% more error
)
print()
print("Calculating training and validation set scores for reference")
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation")]:
errors = []
for patient in patient_ids:
prediction = final_predictions[patient-1]
if "final_systole" in prediction:
assert patient == regular_labels[patient-1, 0]
error1 = utils.CRSP(prediction["final_systole"], regular_labels[patient-1, 1])
errors.append(error1)
prediction["systole_crps_error"] = error1
error2 = utils.CRSP(prediction["final_diastole"], regular_labels[patient-1, 2])
errors.append(error2)
prediction["diastole_crps_error"] = error1
prediction["average_crps_error"] = 0.5*error1 + 0.5*error2
if len(errors)>0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print(" %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP))
validation_dict[set_name] = estimated_CRSP
else:
print(" %s kaggle loss: not calculated" % (string.rjust(set_name, 12)))
print("WARNING: both of the previous are overfitted!")
print()
print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2))
print()
print("dumping submission file to %s" % submission_path)
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d'%i for i in range(600)])
for prediction in final_predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in test_patients_indices:
if "final_diastole" not in prediction or "final_systole" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_diastole"].flatten()])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_systole"].flatten()])
print("submission file dumped")
def predict_model(expid, mfile=None):
metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)
prediction_path = INTERMEDIATE_PREDICTIONS_PATH + "%s.pkl" % expid
submission_path = SUBMISSION_PATH + "%s.csv" % expid
if theano.config.optimizer != "fast_run":
print "WARNING: not running in fast mode!"
print "Using"
print " %s" % metadata_path
print "To generate"
print " %s" % prediction_path
print " %s" % submission_path
print "Build model"
interface_layers = config().build_model()
output_layers = interface_layers["outputs"]
input_layers = interface_layers["inputs"]
top_layer = lasagne.layers.MergeLayer(
incomings=output_layers.values()
)
all_layers = lasagne.layers.get_all_layers(top_layer)
num_params = lasagne.layers.count_params(top_layer)
print " number of parameters: %d" % num_params
print string.ljust(" layer output shapes:",36),
print string.ljust("#params:",10),
print "output shape:"
for layer in all_layers[:-1]:
name = string.ljust(layer.__class__.__name__, 32)
num_param = sum([np.prod(p.get_value().shape) for p in layer.get_params()])
num_param = string.ljust(num_param.__str__(), 10)
print " %s %s %s" % (name, num_param, layer.output_shape)
xs_shared = {
key: lasagne.utils.shared_empty(dim=len(l_in.output_shape), dtype='float32') for (key, l_in) in input_layers.iteritems()
}
idx = T.lscalar('idx')
givens = dict()
for key in input_layers.keys():
if key=="sunny":
givens[input_layers[key].input_var] = xs_shared[key][idx*config().sunny_batch_size:(idx+1)*config().sunny_batch_size]
else:
givens[input_layers[key].input_var] = xs_shared[key][idx*config().batch_size:(idx+1)*config().batch_size]
network_outputs = [
lasagne.layers.helper.get_output(network_output_layer, deterministic=True)
for network_output_layer in output_layers.values()
]
iter_test = theano.function([idx], network_outputs + theano_printer.get_the_stuff_to_print(),
givens=givens, on_unused_input="ignore",
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
)
print "Load model parameters for resuming"
resume_metadata = np.load(metadata_path)
lasagne.layers.set_all_param_values(top_layer, resume_metadata['param_values'])
num_batches_chunk = config().batches_per_chunk
num_batches = get_number_of_test_batches()
num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))
chunks_train_idcs = range(1, num_chunks+1)
data_loader.filter_patient_folders()
create_test_gen = partial(config().create_test_gen,
required_input_keys = xs_shared.keys(),
required_output_keys = ["patients", "classification_correction_function"],
)
print "Generate predictions with this model"
start_time = time.time()
prev_time = start_time
predictions = [{"patient": i+1,
"systole": np.zeros((0,600)),
"diastole": np.zeros((0,600))
} for i in xrange(NUM_PATIENTS)]
for e, test_data in izip(itertools.count(start=1), buffering.buffered_gen_threaded(create_test_gen())):
print " load testing data onto GPU"
for key in xs_shared:
xs_shared[key].set_value(test_data["input"][key])
patient_ids = test_data["output"]["patients"]
classification_correction = test_data["output"]["classification_correction_function"]
print " patients:", " ".join(map(str, patient_ids))
print " chunk %d/%d" % (e, num_chunks)
for b in xrange(num_batches_chunk):
iter_result = iter_test(b)
network_outputs = tuple(iter_result[:len(output_layers)])
network_outputs_dict = {output_layers.keys()[i]: network_outputs[i] for i in xrange(len(output_layers))}
kaggle_systoles, kaggle_diastoles = config().postprocess(network_outputs_dict)
kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype('float64'), kaggle_diastoles.astype('float64')
for idx, patient_id in enumerate(patient_ids[b*config().batch_size:(b+1)*config().batch_size]):
if patient_id != 0:
index = patient_id-1
patient_data = predictions[index]
assert patient_id==patient_data["patient"]
kaggle_systole = kaggle_systoles[idx:idx+1,:]
kaggle_diastole = kaggle_diastoles[idx:idx+1,:]
assert np.isfinite(kaggle_systole).all() and np.isfinite(kaggle_systole).all()
kaggle_systole = classification_correction[b*config().batch_size + idx](kaggle_systole)
kaggle_diastole = classification_correction[b*config().batch_size + idx](kaggle_diastole)
assert np.isfinite(kaggle_systole).all() and np.isfinite(kaggle_systole).all()
patient_data["systole"] = np.concatenate((patient_data["systole"], kaggle_systole ),axis=0)
patient_data["diastole"] = np.concatenate((patient_data["diastole"], kaggle_diastole ),axis=0)
now = time.time()
time_since_start = now - start_time
time_since_prev = now - prev_time
prev_time = now
est_time_left = time_since_start * (float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
eta = datetime.now() + timedelta(seconds=est_time_left)
eta_str = eta.strftime("%c")
print " %s since start (%.2f s)" % (utils.hms(time_since_start), time_since_prev)
print " estimated %s to go (ETA: %s)" % (utils.hms(est_time_left), eta_str)
print
already_printed = False
for prediction in predictions:
if prediction["systole"].size>0 and prediction["diastole"].size>0:
average_method = getattr(config(), 'tta_average_method', partial(np.mean, axis=0))
prediction["systole_average"] = average_method(prediction["systole"])
prediction["diastole_average"] = average_method(prediction["diastole"])
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
if not already_printed:
print "WARNING: These distributions are not distributions"
already_printed = True
prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
print "Calculating training and validation set scores for reference"
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation"),
(train_patients_indices, "train")]:
errors = []
for patient in patient_ids:
prediction = predictions[patient-1]
if "systole_average" in prediction:
assert patient == regular_labels[patient-1, 0]
error = CRSP(prediction["systole_average"], regular_labels[patient-1, 1])
errors.append(error)
error = CRSP(prediction["diastole_average"], regular_labels[patient-1, 2])
errors.append(error)
if len(errors)>0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print " %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP)
validation_dict[set_name] = estimated_CRSP
else:
print " %s kaggle loss: not calculated" % (string.rjust(set_name, 12))
print "dumping prediction file to %s" % prediction_path
with open(prediction_path, 'w') as f:
pickle.dump({
'metadata_path': metadata_path,
'prediction_path': prediction_path,
'submission_path': submission_path,
'configuration_file': config().__name__,
'git_revision_hash': utils.get_git_revision_hash(),
'experiment_id': expid,
'time_since_start': time_since_start,
'param_values': lasagne.layers.get_all_param_values(top_layer),
'predictions': predictions,
'validation_errors': validation_dict,
}, f, pickle.HIGHEST_PROTOCOL)
print "prediction file dumped"
print "dumping submission file to %s" % submission_path
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d'%i for i in xrange(600)])
for prediction in predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in data_loader.test_patients_indices:
if "diastole_average" not in prediction or "systole_average" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["diastole_average"].flatten()])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["systole_average"].flatten()])
print "submission file dumped"
return
def generate_final_predictions(
final_predictions,
prediction_tag,
expert_predictions_matrix,
masked_expert_predictions_matrix,
test_expert_predictions_matrix,
test_masked_expert_predictions_matrix,
optimal_params,
average_distribution,
valid_labels,
expert_pkl_files,
expert_weight,
disagreement_cutoff):
cache_dict = dict()
for final_prediction in final_predictions:
patient_id = final_prediction['patient']
if patient_id in train_patients_indices:
continue
print " final prediction of patient %d" % patient_id
reoptimized = False
# get me the data for this patient
# (NUM_EXPERTS, 600)
if patient_id in validation_patients_indices:
idx = validation_patients_indices.index(patient_id)
prediction_matrix = expert_predictions_matrix[:, idx, :]
mask_matrix = masked_expert_predictions_matrix[:, idx]
elif patient_id in test_patients_indices:
idx = test_patients_indices.index(patient_id)
prediction_matrix = test_expert_predictions_matrix[:, idx, :]
mask_matrix = test_masked_expert_predictions_matrix[:, idx]
else:
raise "This patient is neither train, validation or test?"
# Is there an expert in the set, which did not predict this patient?
# if so, re-optimize our weights on the validation set!
# so, if a model is unavailble, but should be available, re-optimize!
if np.logical_and(np.logical_not(mask_matrix), (expert_weight!=0.0)).any():
if hash_mask(mask_matrix) in cache_dict:
(optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
else:
expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
expert_mask=mask_matrix,
prediction_matrix=expert_predictions_matrix,
mask_matrix=masked_expert_predictions_matrix,
labels=valid_labels,
average_distribution=average_distribution,
)
cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
reoptimized = True
else:
optimal_params_for_this_patient = optimal_params
while True: # do-while: break condition at the end
last_mask_matrix = mask_matrix
final_prediction[prediction_tag] = optimize_expert_weights(
expert_predictions=prediction_matrix[:,None,:],
mask_matrix=mask_matrix[:,None],
average_distribution=average_distribution,
do_optimization=False,
optimal_params=optimal_params_for_this_patient,
)
## find the disagreement between models
# and remove the ones who disagree too much with the average
disagreement = find_disagreement(
expert_predictions=prediction_matrix,
mask_matrix=mask_matrix,
ground_truth=final_prediction[prediction_tag]
)
mask_matrix = (disagreement<disagreement_cutoff)
## RETRAIN THESE ENSEMBLE WEIGHTS ON THE VALIDATION SET IF NEEDED!
if ((last_mask_matrix!=mask_matrix).any()
and np.sum(mask_matrix)!=0):
if hash_mask(mask_matrix) in cache_dict:
(optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
else:
expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
expert_mask=mask_matrix,
prediction_matrix=expert_predictions_matrix,
mask_matrix=masked_expert_predictions_matrix,
labels=valid_labels,
average_distribution=average_distribution,
)
cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
reoptimized = True
continue
else:
break
try:
test_if_valid_distribution(final_prediction[prediction_tag])
except:
final_prediction[prediction_tag] = make_monotone_distribution(final_prediction[prediction_tag])
test_if_valid_distribution(final_prediction[prediction_tag])
if reoptimized:
print " Weight: Name:"
for expert_name, weight in zip(expert_pkl_files, expert_weight):
if weight>0.:
print string.rjust("%.3f%%" % (100*weight), 12),
print expert_name.split('/')[-1]
def merge_all_prediction_files(prediction_file_location = INTERMEDIATE_PREDICTIONS_PATH,
redo_tta = True):
submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
# calculate the average distribution
regular_labels = _load_file(_TRAIN_LABELS_PATH)
average_systole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,1]]), axis=0))
average_diastole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,2]]), axis=0))
ss_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_nomask.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360_gauss_longer.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm.pkl")
+glob.glob(prediction_file_location+"j6_2ch_96mm.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_framemax.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_96.pkl")
+glob.glob(prediction_file_location+"j6_4ch.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"j6_4ch_32mm_specialist.pkl")
+glob.glob(prediction_file_location+"j6_4ch_128mm_specialist.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64_leaky_convroll.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80small_360_gauss_longer_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll_augzoombright.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_zoom.pkl")
+glob.glob(prediction_file_location+"j6_2ch_128mm_skew.pkl")
+glob.glob(prediction_file_location+"je_ss_jonisc64small_360.pkl")
)
fp_expert_pkl_files = sorted([]
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout_seqshift_96.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_big_leaky_after_seqshift.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_big.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky_after.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky.pkl")
+glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_rellocframe.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_filtered.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_framemax_reg.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv_augzoombright_short.pkl")
+glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered_discs.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale64small_filtered_longer.pkl")
+glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright_betterdist.pkl")
+glob.glob(prediction_file_location+"je_os_segmentandintegrate_smartsigma_dropout.pkl")
)
everything = sorted([]
+glob.glob(prediction_file_location+"*.pkl")
)
expert_pkl_files = ss_expert_pkl_files + fp_expert_pkl_files
print "found %d/44 files" % len(expert_pkl_files)
"""
# filter expert_pkl_files
for file in expert_pkl_files[:]:
try:
with open(file, 'r') as f:
print "testing file",file.split('/')[-1]
data = pickle.load(f)
if 'predictions' not in data.keys():
expert_pkl_files.remove(file)
print " -> removed"
except:
print sys.exc_info()[0]
expert_pkl_files.remove(file)
print " -> removed"
"""
NUM_EXPERTS = len(expert_pkl_files)
NUM_VALIDATIONS = len(validation_patients_indices)
NUM_TESTS = len(test_patients_indices)
systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
test_systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
test_systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
test_diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
for i,file in enumerate(expert_pkl_files):
with open(file, 'r') as f:
print
print "loading file",file.split('/')[-1]
predictions = pickle.load(f)['predictions']
if redo_tta:
best_average_method = normalav
best_average_crps = utils.maxfloat
for average_method in [geomav,
normalav,
prodav,
weighted_geom_method
]:
calculate_tta_average(predictions, average_method, average_systole, average_diastole)
crps = get_validate_crps(predictions, regular_labels)
print string.rjust(average_method.__name__,25),"->",crps
if crps<best_average_crps:
best_average_method = average_method
best_average_crps = crps
print " I choose you,", best_average_method.__name__
calculate_tta_average(predictions, best_average_method, average_systole, average_diastole)
print " validation loss:", get_validate_crps(predictions, regular_labels)
for j,patient in enumerate(validation_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
diastole_masked_expert_predictions_matrix[i,j] = False
for j,patient in enumerate(test_patients_indices):
prediction = predictions[patient-1]
# average distributions get zero weight later on
if "systole_average" in prediction and prediction["systole_average"] is not None:
test_systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
else:
test_systole_masked_expert_predictions_matrix[i,j] = False
if "diastole_average" in prediction and prediction["diastole_average"] is not None:
test_diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
else:
test_diastole_masked_expert_predictions_matrix[i,j] = False
del predictions # can be LOADS of data
cv = [id-1 for id in validation_patients_indices]
systole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,1].flatten()])
systole_expert_weight, first_pass_sys_loss, systole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=systole_expert_predictions_matrix,
mask_matrix=systole_masked_expert_predictions_matrix,
labels=systole_valid_labels,
average_distribution=average_systole,
)
cv = [id-1 for id in validation_patients_indices]
diastole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,2].flatten()])
diastole_expert_weight, first_pass_dia_loss, diastole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
prediction_matrix=diastole_expert_predictions_matrix,
mask_matrix=diastole_masked_expert_predictions_matrix,
labels=diastole_valid_labels,
average_distribution=average_diastole,
)
# print the final weight of every expert
print " Systole: Diastole: Name:"
for expert_name, systole_weight, diastole_weight in zip(expert_pkl_files, systole_expert_weight, diastole_expert_weight):
print string.rjust("%.3f%%" % (100*systole_weight), 10),
print string.rjust("%.3f%%" % (100*diastole_weight), 10),
print expert_name.split('/')[-1]
print
print "estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2)
print
print
print "Average the experts according to these weights to find the final distribution"
final_predictions = [{
"patient": i+1,
"final_systole": None,
"final_diastole": None
} for i in xrange(NUM_PATIENTS)]
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_systole",
expert_predictions_matrix=systole_expert_predictions_matrix,
masked_expert_predictions_matrix=systole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_systole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_systole_masked_expert_predictions_matrix,
optimal_params=systole_optimal_params,
average_distribution=average_systole,
valid_labels=systole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=systole_expert_weight,
disagreement_cutoff=0.01 # 0.01
)
generate_final_predictions(
final_predictions=final_predictions,
prediction_tag="final_diastole",
expert_predictions_matrix=diastole_expert_predictions_matrix,
masked_expert_predictions_matrix=diastole_masked_expert_predictions_matrix,
test_expert_predictions_matrix=test_diastole_expert_predictions_matrix,
test_masked_expert_predictions_matrix=test_diastole_masked_expert_predictions_matrix,
optimal_params=diastole_optimal_params,
average_distribution=average_diastole,
valid_labels=diastole_valid_labels,
expert_pkl_files=expert_pkl_files,
expert_weight=diastole_expert_weight,
disagreement_cutoff=0.015 # diastole has about 50% more error
)
print
print "Calculating training and validation set scores for reference"
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation")]:
errors = []
for patient in patient_ids:
prediction = final_predictions[patient-1]
if "final_systole" in prediction:
assert patient == regular_labels[patient-1, 0]
error1 = utils.CRSP(prediction["final_systole"], regular_labels[patient-1, 1])
errors.append(error1)
prediction["systole_crps_error"] = error1
error2 = utils.CRSP(prediction["final_diastole"], regular_labels[patient-1, 2])
errors.append(error2)
prediction["diastole_crps_error"] = error1
prediction["average_crps_error"] = 0.5*error1 + 0.5*error2
if len(errors)>0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print " %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP)
validation_dict[set_name] = estimated_CRSP
else:
print " %s kaggle loss: not calculated" % (string.rjust(set_name, 12))
print "WARNING: both of the previous are overfitted!"
print
print "estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2)
print
print "dumping submission file to %s" % submission_path
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d'%i for i in xrange(600)])
for prediction in final_predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in test_patients_indices:
if "final_diastole" not in prediction or "final_systole" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_diastole"].flatten()])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_systole"].flatten()])
print "submission file dumped"