def _make_valid_distributions(metadata):
for prediction in metadata['predictions']:
prediction["systole_average"] = postprocess.make_monotone_distribution(
prediction["systole_average"])
postprocess.test_if_valid_distribution(prediction["systole_average"])
prediction[
"diastole_average"] = postprocess.make_monotone_distribution(
prediction["diastole_average"])
postprocess.test_if_valid_distribution(prediction["diastole_average"])
def calculate_tta_average(predictions, average_method, average_systole, average_diastole):
already_printed = False
for prediction in predictions:
if prediction["systole"].size>0 and prediction["diastole"].size>0:
assert np.isfinite(prediction["systole"][None,:,:]).all()
assert np.isfinite(prediction["diastole"][None,:,:]).all()
prediction["systole_average"] = average_method(prediction["systole"][None,:,:], average=average_systole)
prediction["diastole_average"] = average_method(prediction["diastole"][None,:,:], average=average_diastole)
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
if not already_printed:
#print "WARNING: These distributions are not distributions"
already_printed = True
prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
prediction["systole_average"] = None
prediction["diastole_average"] = None
else:
# average distributions get zero weight later on
#prediction["systole_average"] = average_systole
#prediction["diastole_average"] = average_diastole
prediction["systole_average"] = None
prediction["diastole_average"] = None
def calculate_tta_average(predictions, average_method, average_systole, average_diastole):
already_printed = False
for prediction in predictions:
if prediction["systole"].size>0 and prediction["diastole"].size>0:
assert np.isfinite(prediction["systole"][None,:,:]).all()
assert np.isfinite(prediction["diastole"][None,:,:]).all()
prediction["systole_average"] = average_method(prediction["systole"][None,:,:], average=average_systole)
prediction["diastole_average"] = average_method(prediction["diastole"][None,:,:], average=average_diastole)
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
if not already_printed:
#print "WARNING: These distributions are not distributions"
already_printed = True
prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
prediction["systole_average"] = None
prediction["diastole_average"] = None
else:
# average distributions get zero weight later on
#prediction["systole_average"] = average_systole
#prediction["diastole_average"] = average_diastole
prediction["systole_average"] = None
prediction["diastole_average"] = None
def _make_valid_distributions(metadata):
for prediction in metadata['predictions']:
prediction["systole_average"] = postprocess.make_monotone_distribution(prediction["systole_average"])
postprocess.test_if_valid_distribution(prediction["systole_average"])
prediction["diastole_average"] = postprocess.make_monotone_distribution(prediction["diastole_average"])
postprocess.test_if_valid_distribution(prediction["diastole_average"])
def predict_slice_model(expid, outfile, mfile=None):
metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)
if theano.config.optimizer != "fast_run":
print "WARNING: not running in fast mode!"
print "Build model"
interface_layers = config().build_model()
output_layers = interface_layers["outputs"]
input_layers = interface_layers["inputs"]
top_layer = lasagne.layers.MergeLayer(
incomings=output_layers.values()
)
_check_slicemodel(input_layers)
# Print the architecture
_print_architecture(top_layer)
xs_shared = {
key: lasagne.utils.shared_empty(dim=len(l_in.output_shape), dtype='float32') for (key, l_in) in input_layers.iteritems()
}
idx = T.lscalar('idx')
givens = dict()
for key in input_layers.keys():
if key=="sunny":
givens[input_layers[key].input_var] = xs_shared[key][idx*config().sunny_batch_size:(idx+1)*config().sunny_batch_size]
else:
givens[input_layers[key].input_var] = xs_shared[key][idx*config().batch_size:(idx+1)*config().batch_size]
network_outputs = [
lasagne.layers.helper.get_output(network_output_layer, deterministic=True)
for network_output_layer in output_layers.values()
]
iter_test = theano.function([idx], network_outputs + theano_printer.get_the_stuff_to_print(),
givens=givens, on_unused_input="ignore",
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
)
print "Load model parameters for resuming"
resume_metadata = np.load(metadata_path)
lasagne.layers.set_all_param_values(top_layer, resume_metadata['param_values'])
num_batches_chunk = config().batches_per_chunk
num_batches = get_number_of_test_batches()
num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))
chunks_train_idcs = range(1, num_chunks+1)
create_test_gen = partial(config().create_test_gen,
required_input_keys = xs_shared.keys(),
required_output_keys = ["patients", "slices"],
)
print "Generate predictions with this model"
start_time = time.time()
prev_time = start_time
predictions = [{"patient": i+1,
"slices": {
slice_id: {
"systole": np.zeros((0,600)),
"diastole": np.zeros((0,600))
} for slice_id in data_loader.get_slice_ids_for_patient(i+1)
}
} for i in xrange(NUM_PATIENTS)]
# Loop over data and generate predictions
for e, test_data in izip(itertools.count(start=1), buffering.buffered_gen_threaded(create_test_gen())):
print " load testing data onto GPU"
for key in xs_shared:
xs_shared[key].set_value(test_data["input"][key])
patient_ids = test_data["output"]["patients"]
slice_ids = test_data["output"]["slices"]
print " patients:", " ".join(map(str, patient_ids))
print " chunk %d/%d" % (e, num_chunks)
for b in xrange(num_batches_chunk):
iter_result = iter_test(b)
network_outputs = tuple(iter_result[:len(output_layers)])
network_outputs_dict = {output_layers.keys()[i]: network_outputs[i] for i in xrange(len(output_layers))}
kaggle_systoles, kaggle_diastoles = config().postprocess(network_outputs_dict)
kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype('float64'), kaggle_diastoles.astype('float64')
for idx, (patient_id, slice_id) in enumerate(
zip(patient_ids[b*config().batch_size:(b+1)*config().batch_size],
slice_ids[b*config().batch_size:(b+1)*config().batch_size])):
if patient_id != 0:
index = patient_id-1
patient_data = predictions[index]
assert patient_id==patient_data["patient"]
patient_slice_data = patient_data["slices"][slice_id]
patient_slice_data["systole"] = np.concatenate((patient_slice_data["systole"], kaggle_systoles[idx:idx+1,:]),axis=0)
patient_slice_data["diastole"] = np.concatenate((patient_slice_data["diastole"], kaggle_diastoles[idx:idx+1,:]),axis=0)
now = time.time()
time_since_start = now - start_time
time_since_prev = now - prev_time
prev_time = now
est_time_left = time_since_start * (float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
eta = datetime.now() + timedelta(seconds=est_time_left)
eta_str = eta.strftime("%c")
print " %s since start (%.2f s)" % (utils.hms(time_since_start), time_since_prev)
print " estimated %s to go (ETA: %s)" % (utils.hms(est_time_left), eta_str)
print
# Average predictions
already_printed = False
for prediction in predictions:
for prediction_slice_id in prediction["slices"]:
prediction_slice = prediction["slices"][prediction_slice_id]
if prediction_slice["systole"].size>0 and prediction_slice["diastole"].size>0:
average_method = getattr(config(), 'tta_average_method', partial(np.mean, axis=0))
prediction_slice["systole_average"] = average_method(prediction_slice["systole"])
prediction_slice["diastole_average"] = average_method(prediction_slice["diastole"])
try:
test_if_valid_distribution(prediction_slice["systole_average"])
test_if_valid_distribution(prediction_slice["diastole_average"])
except:
if not already_printed:
print "WARNING: These distributions are not distributions"
already_printed = True
prediction_slice["systole_average"] = make_monotone_distribution(prediction_slice["systole_average"])
prediction_slice["diastole_average"] = make_monotone_distribution(prediction_slice["diastole_average"])
print "Calculating training and validation set scores for reference"
# Add CRPS scores to the predictions
# Iterate over train and validation sets
for patient_ids, set_name in [(validation_patients_indices, "validation"),
(train_patients_indices, "train")]:
# Iterate over patients in the set
for patient in patient_ids:
prediction = predictions[patient-1]
# Iterate over the slices
for slice_id in prediction["slices"]:
prediction_slice = prediction["slices"][slice_id]
if "systole_average" in prediction_slice:
assert patient == regular_labels[patient-1, 0]
error_sys = CRSP(prediction_slice["systole_average"], regular_labels[patient-1, 1])
prediction_slice["systole_CRPS"] = error_sys
prediction_slice["target_systole"] = regular_labels[patient-1, 1]
error_dia = CRSP(prediction_slice["diastole_average"], regular_labels[patient-1, 2])
prediction_slice["diastole_CRPS"] = error_dia
prediction_slice["target_diastole"] = regular_labels[patient-1, 2]
prediction_slice["CRPS"] = 0.5 * error_sys + 0.5 * error_dia
print "dumping prediction file to %s" % outfile
with open(outfile, 'w') as f:
pickle.dump({
'metadata_path': metadata_path,
'configuration_file': config().__name__,
'git_revision_hash': utils.get_git_revision_hash(),
'experiment_id': expid,
'time_since_start': time_since_start,
'param_values': lasagne.layers.get_all_param_values(top_layer),
'predictions_per_slice': predictions,
}, f, pickle.HIGHEST_PROTOCOL)
print "prediction file dumped"
return
def predict_model(expid, mfile=None):
metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)
prediction_path = INTERMEDIATE_PREDICTIONS_PATH + "%s.pkl" % expid
submission_path = SUBMISSION_PATH + "%s.csv" % expid
if theano.config.optimizer != "fast_run":
print("WARNING: not running in fast mode!")
print("Using")
print(" %s" % metadata_path)
print("To generate")
print(" %s" % prediction_path)
print(" %s" % submission_path)
print("Build model")
interface_layers = config().build_model()
output_layers = interface_layers["outputs"]
input_layers = interface_layers["inputs"]
top_layer = lasagne.layers.MergeLayer(
incomings=list(output_layers.values()))
all_layers = lasagne.layers.get_all_layers(top_layer)
num_params = lasagne.layers.count_params(top_layer)
print(" number of parameters: %d" % num_params)
print(string.ljust(" layer output shapes:", 36), end=' ')
print(string.ljust("#params:", 10), end=' ')
print("output shape:")
for layer in all_layers[:-1]:
name = string.ljust(layer.__class__.__name__, 32)
num_param = sum(
[np.prod(p.get_value().shape) for p in layer.get_params()])
num_param = string.ljust(num_param.__str__(), 10)
print(" %s %s %s" % (name, num_param, layer.output_shape))
xs_shared = {
key: lasagne.utils.shared_empty(dim=len(l_in.output_shape),
dtype='float32')
for (key, l_in) in input_layers.items()
}
idx = T.lscalar('idx')
givens = dict()
for key in list(input_layers.keys()):
if key == "sunny":
givens[input_layers[key].input_var] = xs_shared[key][idx * config(
).sunny_batch_size:(idx + 1) * config().sunny_batch_size]
else:
givens[input_layers[key].
input_var] = xs_shared[key][idx *
config().batch_size:(idx + 1) *
config().batch_size]
network_outputs = [
lasagne.layers.helper.get_output(network_output_layer,
deterministic=True)
for network_output_layer in list(output_layers.values())
]
iter_test = theano.function(
[idx],
network_outputs + theano_printer.get_the_stuff_to_print(),
givens=givens,
on_unused_input="ignore",
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
)
print("Load model parameters for resuming")
resume_metadata = np.load(metadata_path)
lasagne.layers.set_all_param_values(top_layer,
resume_metadata['param_values'])
num_batches_chunk = config().batches_per_chunk
num_batches = get_number_of_test_batches()
num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))
chunks_train_idcs = list(range(1, num_chunks + 1))
data_loader.filter_patient_folders()
create_test_gen = partial(
config().create_test_gen,
required_input_keys=list(xs_shared.keys()),
required_output_keys=[
"patients", "classification_correction_function"
],
)
print("Generate predictions with this model")
start_time = time.time()
prev_time = start_time
predictions = [{
"patient": i + 1,
"systole": np.zeros((0, 600)),
"diastole": np.zeros((0, 600))
} for i in range(NUM_PATIENTS)]
for e, test_data in zip(itertools.count(start=1),
buffering.buffered_gen_threaded(
create_test_gen())):
print(" load testing data onto GPU")
for key in xs_shared:
xs_shared[key].set_value(test_data["input"][key])
patient_ids = test_data["output"]["patients"]
classification_correction = test_data["output"][
"classification_correction_function"]
print(" patients:", " ".join(map(str, patient_ids)))
print(" chunk %d/%d" % (e, num_chunks))
for b in range(num_batches_chunk):
iter_result = iter_test(b)
network_outputs = tuple(iter_result[:len(output_layers)])
network_outputs_dict = {
list(output_layers.keys())[i]: network_outputs[i]
for i in range(len(output_layers))
}
kaggle_systoles, kaggle_diastoles = config().postprocess(
network_outputs_dict)
kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype(
'float64'), kaggle_diastoles.astype('float64')
for idx, patient_id in enumerate(
patient_ids[b * config().batch_size:(b + 1) *
config().batch_size]):
if patient_id != 0:
index = patient_id - 1
patient_data = predictions[index]
assert patient_id == patient_data["patient"]
kaggle_systole = kaggle_systoles[idx:idx + 1, :]
kaggle_diastole = kaggle_diastoles[idx:idx + 1, :]
assert np.isfinite(kaggle_systole).all() and np.isfinite(
kaggle_systole).all()
kaggle_systole = classification_correction[
b * config().batch_size + idx](kaggle_systole)
kaggle_diastole = classification_correction[
b * config().batch_size + idx](kaggle_diastole)
assert np.isfinite(kaggle_systole).all() and np.isfinite(
kaggle_systole).all()
patient_data["systole"] = np.concatenate(
(patient_data["systole"], kaggle_systole), axis=0)
patient_data["diastole"] = np.concatenate(
(patient_data["diastole"], kaggle_diastole), axis=0)
now = time.time()
time_since_start = now - start_time
time_since_prev = now - prev_time
prev_time = now
est_time_left = time_since_start * (
float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
eta = datetime.now() + timedelta(seconds=est_time_left)
eta_str = eta.strftime("%c")
print(" %s since start (%.2f s)" %
(utils.hms(time_since_start), time_since_prev))
print(" estimated %s to go (ETA: %s)" %
(utils.hms(est_time_left), eta_str))
print()
already_printed = False
for prediction in predictions:
if prediction["systole"].size > 0 and prediction["diastole"].size > 0:
average_method = getattr(config(), 'tta_average_method',
partial(np.mean, axis=0))
prediction["systole_average"] = average_method(
prediction["systole"])
prediction["diastole_average"] = average_method(
prediction["diastole"])
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
if not already_printed:
print("WARNING: These distributions are not distributions")
already_printed = True
prediction["systole_average"] = make_monotone_distribution(
prediction["systole_average"])
prediction["diastole_average"] = make_monotone_distribution(
prediction["diastole_average"])
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
print("Calculating training and validation set scores for reference")
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation"),
(train_patients_indices, "train")]:
errors = []
for patient in patient_ids:
prediction = predictions[patient - 1]
if "systole_average" in prediction:
assert patient == regular_labels[patient - 1, 0]
error = CRSP(prediction["systole_average"],
regular_labels[patient - 1, 1])
errors.append(error)
error = CRSP(prediction["diastole_average"],
regular_labels[patient - 1, 2])
errors.append(error)
if len(errors) > 0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print(" %s kaggle loss: %f" %
(string.rjust(set_name, 12), estimated_CRSP))
validation_dict[set_name] = estimated_CRSP
else:
print(" %s kaggle loss: not calculated" %
(string.rjust(set_name, 12)))
print("dumping prediction file to %s" % prediction_path)
with open(prediction_path, 'w') as f:
pickle.dump(
{
'metadata_path': metadata_path,
'prediction_path': prediction_path,
'submission_path': submission_path,
'configuration_file': config().__name__,
'git_revision_hash': utils.get_git_revision_hash(),
'experiment_id': expid,
'time_since_start': time_since_start,
'param_values': lasagne.layers.get_all_param_values(top_layer),
'predictions': predictions,
'validation_errors': validation_dict,
}, f, pickle.HIGHEST_PROTOCOL)
print("prediction file dumped")
print("dumping submission file to %s" % submission_path)
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d' % i for i in range(600)])
for prediction in predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in data_loader.test_patients_indices:
if "diastole_average" not in prediction or "systole_average" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + [
"%.18f" % p
for p in prediction["diastole_average"].flatten()
])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + [
"%.18f" % p
for p in prediction["systole_average"].flatten()
])
print("submission file dumped")
return
def generate_final_predictions(
final_predictions,
prediction_tag,
expert_predictions_matrix,
masked_expert_predictions_matrix,
test_expert_predictions_matrix,
test_masked_expert_predictions_matrix,
optimal_params,
average_distribution,
valid_labels,
expert_pkl_files,
expert_weight,
disagreement_cutoff):
cache_dict = dict()
for final_prediction in final_predictions:
patient_id = final_prediction['patient']
if patient_id in train_patients_indices:
continue
print(" final prediction of patient %d" % patient_id)
reoptimized = False
# get me the data for this patient
# (NUM_EXPERTS, 600)
if patient_id in validation_patients_indices:
idx = validation_patients_indices.index(patient_id)
prediction_matrix = expert_predictions_matrix[:, idx, :]
mask_matrix = masked_expert_predictions_matrix[:, idx]
elif patient_id in test_patients_indices:
idx = test_patients_indices.index(patient_id)
prediction_matrix = test_expert_predictions_matrix[:, idx, :]
mask_matrix = test_masked_expert_predictions_matrix[:, idx]
else:
raise "This patient is neither train, validation or test?"
# Is there an expert in the set, which did not predict this patient?
# if so, re-optimize our weights on the validation set!
# so, if a model is unavailble, but should be available, re-optimize!
if np.logical_and(np.logical_not(mask_matrix), (expert_weight!=0.0)).any():
if hash_mask(mask_matrix) in cache_dict:
(optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
else:
expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
expert_mask=mask_matrix,
prediction_matrix=expert_predictions_matrix,
mask_matrix=masked_expert_predictions_matrix,
labels=valid_labels,
average_distribution=average_distribution,
)
cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
reoptimized = True
else:
optimal_params_for_this_patient = optimal_params
while True: # do-while: break condition at the end
last_mask_matrix = mask_matrix
final_prediction[prediction_tag] = optimize_expert_weights(
expert_predictions=prediction_matrix[:,None,:],
mask_matrix=mask_matrix[:,None],
average_distribution=average_distribution,
do_optimization=False,
optimal_params=optimal_params_for_this_patient,
)
## find the disagreement between models
# and remove the ones who disagree too much with the average
disagreement = find_disagreement(
expert_predictions=prediction_matrix,
mask_matrix=mask_matrix,
ground_truth=final_prediction[prediction_tag]
)
mask_matrix = (disagreement<disagreement_cutoff)
## RETRAIN THESE ENSEMBLE WEIGHTS ON THE VALIDATION SET IF NEEDED!
if ((last_mask_matrix!=mask_matrix).any()
and np.sum(mask_matrix)!=0):
if hash_mask(mask_matrix) in cache_dict:
(optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
else:
expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
expert_mask=mask_matrix,
prediction_matrix=expert_predictions_matrix,
mask_matrix=masked_expert_predictions_matrix,
labels=valid_labels,
average_distribution=average_distribution,
)
cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
reoptimized = True
continue
else:
break
try:
test_if_valid_distribution(final_prediction[prediction_tag])
except:
final_prediction[prediction_tag] = make_monotone_distribution(final_prediction[prediction_tag])
test_if_valid_distribution(final_prediction[prediction_tag])
if reoptimized:
print(" Weight: Name:")
for expert_name, weight in zip(expert_pkl_files, expert_weight):
if weight>0.:
print(string.rjust("%.3f%%" % (100*weight), 12), end=' ')
print(expert_name.split('/')[-1])
def predict_model(expid, mfile=None):
metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)
prediction_path = INTERMEDIATE_PREDICTIONS_PATH + "%s.pkl" % expid
submission_path = SUBMISSION_PATH + "%s.csv" % expid
if theano.config.optimizer != "fast_run":
print "WARNING: not running in fast mode!"
print "Using"
print " %s" % metadata_path
print "To generate"
print " %s" % prediction_path
print " %s" % submission_path
print "Build model"
interface_layers = config().build_model()
output_layers = interface_layers["outputs"]
input_layers = interface_layers["inputs"]
top_layer = lasagne.layers.MergeLayer(
incomings=output_layers.values()
)
all_layers = lasagne.layers.get_all_layers(top_layer)
num_params = lasagne.layers.count_params(top_layer)
print " number of parameters: %d" % num_params
print string.ljust(" layer output shapes:",36),
print string.ljust("#params:",10),
print "output shape:"
for layer in all_layers[:-1]:
name = string.ljust(layer.__class__.__name__, 32)
num_param = sum([np.prod(p.get_value().shape) for p in layer.get_params()])
num_param = string.ljust(num_param.__str__(), 10)
print " %s %s %s" % (name, num_param, layer.output_shape)
xs_shared = {
key: lasagne.utils.shared_empty(dim=len(l_in.output_shape), dtype='float32') for (key, l_in) in input_layers.iteritems()
}
idx = T.lscalar('idx')
givens = dict()
for key in input_layers.keys():
if key=="sunny":
givens[input_layers[key].input_var] = xs_shared[key][idx*config().sunny_batch_size:(idx+1)*config().sunny_batch_size]
else:
givens[input_layers[key].input_var] = xs_shared[key][idx*config().batch_size:(idx+1)*config().batch_size]
network_outputs = [
lasagne.layers.helper.get_output(network_output_layer, deterministic=True)
for network_output_layer in output_layers.values()
]
iter_test = theano.function([idx], network_outputs + theano_printer.get_the_stuff_to_print(),
givens=givens, on_unused_input="ignore",
# mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
)
print "Load model parameters for resuming"
resume_metadata = np.load(metadata_path)
lasagne.layers.set_all_param_values(top_layer, resume_metadata['param_values'])
num_batches_chunk = config().batches_per_chunk
num_batches = get_number_of_test_batches()
num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))
chunks_train_idcs = range(1, num_chunks+1)
data_loader.filter_patient_folders()
create_test_gen = partial(config().create_test_gen,
required_input_keys = xs_shared.keys(),
required_output_keys = ["patients", "classification_correction_function"],
)
print "Generate predictions with this model"
start_time = time.time()
prev_time = start_time
predictions = [{"patient": i+1,
"systole": np.zeros((0,600)),
"diastole": np.zeros((0,600))
} for i in xrange(NUM_PATIENTS)]
for e, test_data in izip(itertools.count(start=1), buffering.buffered_gen_threaded(create_test_gen())):
print " load testing data onto GPU"
for key in xs_shared:
xs_shared[key].set_value(test_data["input"][key])
patient_ids = test_data["output"]["patients"]
classification_correction = test_data["output"]["classification_correction_function"]
print " patients:", " ".join(map(str, patient_ids))
print " chunk %d/%d" % (e, num_chunks)
for b in xrange(num_batches_chunk):
iter_result = iter_test(b)
network_outputs = tuple(iter_result[:len(output_layers)])
network_outputs_dict = {output_layers.keys()[i]: network_outputs[i] for i in xrange(len(output_layers))}
kaggle_systoles, kaggle_diastoles = config().postprocess(network_outputs_dict)
kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype('float64'), kaggle_diastoles.astype('float64')
for idx, patient_id in enumerate(patient_ids[b*config().batch_size:(b+1)*config().batch_size]):
if patient_id != 0:
index = patient_id-1
patient_data = predictions[index]
assert patient_id==patient_data["patient"]
kaggle_systole = kaggle_systoles[idx:idx+1,:]
kaggle_diastole = kaggle_diastoles[idx:idx+1,:]
assert np.isfinite(kaggle_systole).all() and np.isfinite(kaggle_systole).all()
kaggle_systole = classification_correction[b*config().batch_size + idx](kaggle_systole)
kaggle_diastole = classification_correction[b*config().batch_size + idx](kaggle_diastole)
assert np.isfinite(kaggle_systole).all() and np.isfinite(kaggle_systole).all()
patient_data["systole"] = np.concatenate((patient_data["systole"], kaggle_systole ),axis=0)
patient_data["diastole"] = np.concatenate((patient_data["diastole"], kaggle_diastole ),axis=0)
now = time.time()
time_since_start = now - start_time
time_since_prev = now - prev_time
prev_time = now
est_time_left = time_since_start * (float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
eta = datetime.now() + timedelta(seconds=est_time_left)
eta_str = eta.strftime("%c")
print " %s since start (%.2f s)" % (utils.hms(time_since_start), time_since_prev)
print " estimated %s to go (ETA: %s)" % (utils.hms(est_time_left), eta_str)
print
already_printed = False
for prediction in predictions:
if prediction["systole"].size>0 and prediction["diastole"].size>0:
average_method = getattr(config(), 'tta_average_method', partial(np.mean, axis=0))
prediction["systole_average"] = average_method(prediction["systole"])
prediction["diastole_average"] = average_method(prediction["diastole"])
try:
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
except:
if not already_printed:
print "WARNING: These distributions are not distributions"
already_printed = True
prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
test_if_valid_distribution(prediction["systole_average"])
test_if_valid_distribution(prediction["diastole_average"])
print "Calculating training and validation set scores for reference"
validation_dict = {}
for patient_ids, set_name in [(validation_patients_indices, "validation"),
(train_patients_indices, "train")]:
errors = []
for patient in patient_ids:
prediction = predictions[patient-1]
if "systole_average" in prediction:
assert patient == regular_labels[patient-1, 0]
error = CRSP(prediction["systole_average"], regular_labels[patient-1, 1])
errors.append(error)
error = CRSP(prediction["diastole_average"], regular_labels[patient-1, 2])
errors.append(error)
if len(errors)>0:
errors = np.array(errors)
estimated_CRSP = np.mean(errors)
print " %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP)
validation_dict[set_name] = estimated_CRSP
else:
print " %s kaggle loss: not calculated" % (string.rjust(set_name, 12))
print "dumping prediction file to %s" % prediction_path
with open(prediction_path, 'w') as f:
pickle.dump({
'metadata_path': metadata_path,
'prediction_path': prediction_path,
'submission_path': submission_path,
'configuration_file': config().__name__,
'git_revision_hash': utils.get_git_revision_hash(),
'experiment_id': expid,
'time_since_start': time_since_start,
'param_values': lasagne.layers.get_all_param_values(top_layer),
'predictions': predictions,
'validation_errors': validation_dict,
}, f, pickle.HIGHEST_PROTOCOL)
print "prediction file dumped"
print "dumping submission file to %s" % submission_path
with open(submission_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(['Id'] + ['P%d'%i for i in xrange(600)])
for prediction in predictions:
# the submission only has patients 501 to 700
if prediction["patient"] in data_loader.test_patients_indices:
if "diastole_average" not in prediction or "systole_average" not in prediction:
raise Exception("Not all test-set patients were predicted")
csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["diastole_average"].flatten()])
csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["systole_average"].flatten()])
print "submission file dumped"
return
def generate_final_predictions(
final_predictions,
prediction_tag,
expert_predictions_matrix,
masked_expert_predictions_matrix,
test_expert_predictions_matrix,
test_masked_expert_predictions_matrix,
optimal_params,
average_distribution,
valid_labels,
expert_pkl_files,
expert_weight,
disagreement_cutoff):
cache_dict = dict()
for final_prediction in final_predictions:
patient_id = final_prediction['patient']
if patient_id in train_patients_indices:
continue
print " final prediction of patient %d" % patient_id
reoptimized = False
# get me the data for this patient
# (NUM_EXPERTS, 600)
if patient_id in validation_patients_indices:
idx = validation_patients_indices.index(patient_id)
prediction_matrix = expert_predictions_matrix[:, idx, :]
mask_matrix = masked_expert_predictions_matrix[:, idx]
elif patient_id in test_patients_indices:
idx = test_patients_indices.index(patient_id)
prediction_matrix = test_expert_predictions_matrix[:, idx, :]
mask_matrix = test_masked_expert_predictions_matrix[:, idx]
else:
raise "This patient is neither train, validation or test?"
# Is there an expert in the set, which did not predict this patient?
# if so, re-optimize our weights on the validation set!
# so, if a model is unavailble, but should be available, re-optimize!
if np.logical_and(np.logical_not(mask_matrix), (expert_weight!=0.0)).any():
if hash_mask(mask_matrix) in cache_dict:
(optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
else:
expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
expert_mask=mask_matrix,
prediction_matrix=expert_predictions_matrix,
mask_matrix=masked_expert_predictions_matrix,
labels=valid_labels,
average_distribution=average_distribution,
)
cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
reoptimized = True
else:
optimal_params_for_this_patient = optimal_params
while True: # do-while: break condition at the end
last_mask_matrix = mask_matrix
final_prediction[prediction_tag] = optimize_expert_weights(
expert_predictions=prediction_matrix[:,None,:],
mask_matrix=mask_matrix[:,None],
average_distribution=average_distribution,
do_optimization=False,
optimal_params=optimal_params_for_this_patient,
)
## find the disagreement between models
# and remove the ones who disagree too much with the average
disagreement = find_disagreement(
expert_predictions=prediction_matrix,
mask_matrix=mask_matrix,
ground_truth=final_prediction[prediction_tag]
)
mask_matrix = (disagreement<disagreement_cutoff)
## RETRAIN THESE ENSEMBLE WEIGHTS ON THE VALIDATION SET IF NEEDED!
if ((last_mask_matrix!=mask_matrix).any()
and np.sum(mask_matrix)!=0):
if hash_mask(mask_matrix) in cache_dict:
(optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
else:
expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
expert_mask=mask_matrix,
prediction_matrix=expert_predictions_matrix,
mask_matrix=masked_expert_predictions_matrix,
labels=valid_labels,
average_distribution=average_distribution,
)
cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
reoptimized = True
continue
else:
break
try:
test_if_valid_distribution(final_prediction[prediction_tag])
except:
final_prediction[prediction_tag] = make_monotone_distribution(final_prediction[prediction_tag])
test_if_valid_distribution(final_prediction[prediction_tag])
if reoptimized:
print " Weight: Name:"
for expert_name, weight in zip(expert_pkl_files, expert_weight):
if weight>0.:
print string.rjust("%.3f%%" % (100*weight), 12),
print expert_name.split('/')[-1]