def fix_submission(submission_path_meta, submission_path_ss, submission_path_mix, threshold=0.03):
pid2cdf0_meta, pid2cdf1_meta = read_submission(submission_path_meta)
pid2cdf0_ss, pid2cdf1_ss = read_submission(submission_path_ss)
pid2cdf0_mix, pid2cdf1_mix = read_submission(submission_path_mix)
pid2cdf0_fixed, pid2cdf1_fixed = {}, {}
patient_ids = pid2cdf0_meta.keys()
assert patient_ids == pid2cdf0_ss.keys()
assert patient_ids == pid2cdf0_mix.keys()
# no prediction from metamodel -> take prediction from slice model
for pid in patient_ids:
pid2cdf0_fixed[pid] = pid2cdf0_ss[pid] if pid2cdf0_meta[pid] is None else pid2cdf0_meta[pid]
pid2cdf1_fixed[pid] = pid2cdf1_ss[pid] if pid2cdf1_meta[pid] is None else pid2cdf1_meta[pid]
# metamodel disagrees with slice model -> take prediction from mixture ensemble
for pid in patient_ids:
crps0 = utils_heart.crps(pid2cdf0_fixed[pid], pid2cdf0_ss[pid])
if crps0 > threshold:
print 'sys', pid, crps0
pid2cdf0_fixed[pid] = pid2cdf0_mix[pid] if crps0 > threshold else pid2cdf0_fixed[pid]
crps1 = utils_heart.crps(pid2cdf1_fixed[pid], pid2cdf1_ss[pid])
if crps1 > threshold:
print 'dst', pid, crps1
pid2cdf1_fixed[pid] = pid2cdf1_mix[pid] if crps1 > threshold else pid2cdf1_fixed[pid]
fixed_predictions = {}
for pid in patient_ids:
fixed_predictions[pid] = [pid2cdf0_fixed[pid], pid2cdf1_fixed[pid]]
meta_expid = submission_path_meta.split('-')[-1].replace('.csv', '')
ss_expid = submission_path_ss.split('-')[-1].replace('.csv', '')
mix_expid = submission_path_mix.split('-')[-1].replace('.csv', '')
fixed_submission_path = SUBMISSION_PATH + 'ira_%s-%s-%s.csv' % (meta_expid, ss_expid, mix_expid)
utils.save_submission(fixed_predictions, fixed_submission_path)
print 'Submission save to', fixed_submission_path
assert patient_data['patient'] == patient_id
patient_data["systole"] = np.concatenate(
(patient_data["systole"], systole_prediction[None, :]), axis=0)
patient_data["diastole"] = np.concatenate(
(patient_data["diastole"], diastole_prediction[None, :]), axis=0)
avg_patient_predictions = config().get_avg_patient_predictions(
batch_predictions, batch_ids, mean=mean)
patient_targets = utils_heart.get_patient_average_heaviside_predictions(
batch_targets, batch_ids, mean=mean)
assert avg_patient_predictions.viewkeys() == patient_targets.viewkeys()
crpss_sys, crpss_dst = [], []
for id in avg_patient_predictions.iterkeys():
crpss_sys.append(
utils_heart.crps(avg_patient_predictions[id][0],
patient_targets[id][0]))
crpss_dst.append(
utils_heart.crps(avg_patient_predictions[id][1],
patient_targets[id][1]))
crps0, crps1 = np.mean(crpss_sys), np.mean(crpss_dst)
print '\nValidation CRPS: ', crps0, crps1, 0.5 * (crps0 + crps1)
utils.save_pkl(avg_patient_predictions, valid_prediction_path)
print ' validation predictions saved to %s' % valid_prediction_path
print
# test
test_data_iterator = config().test_data_iterator
if n_tta_iterations == 1:
test_data_iterator.transformation_params = config(
).valid_transformation_params
mu_predictions.append(iter_mu())
sigma_predictions.append(iter_sigma())
pid2mu = utils_heart.get_patient_normparam_prediction(mu_predictions, batch_ids)
pid2sigma = utils_heart.get_patient_normparam_prediction(sigma_predictions, batch_ids)
valid_pid2musigma = {}
for pid in pid2mu.keys():
valid_pid2musigma[pid] = {'mu': pid2mu[pid], 'sigma': pid2sigma[pid]}
valid_avg_patient_predictions = config().get_avg_patient_predictions(batch_predictions, batch_ids, mean='geometric')
patient_targets = utils_heart.get_patient_average_heaviside_predictions(batch_targets, batch_ids)
assert valid_avg_patient_predictions.keys() == patient_targets.keys()
crpss_sys, crpss_dst = [], []
for id in valid_avg_patient_predictions.keys():
crpss_sys.append(utils_heart.crps(valid_avg_patient_predictions[id][0], patient_targets[id][0]))
crpss_dst.append(utils_heart.crps(valid_avg_patient_predictions[id][1], patient_targets[id][1]))
print(id, 0.5 * (crpss_sys[-1] + crpss_dst[-1]), crpss_sys[-1], crpss_dst[-1])
crps0, crps1 = np.mean(crpss_sys), np.mean(crpss_dst)
print('Valid crps average: ', 0.5 * (crps0 + crps1))
print(' generating predictions for the train set')
train_data_iterator = config().train_data_iterator
train_data_iterator.transformation_params = config().valid_transformation_params
train_data_iterator.full_batch = False
train_data_iterator.random = False
train_data_iterator.infinite = False
batch_predictions, batch_targets, batch_ids = [], [], []
mu_predictions.append(iter_mu())
sigma_predictions.append(iter_sigma())
pid2mu = utils_heart.get_patient_normparam_prediction(mu_predictions, batch_ids)
pid2sigma = utils_heart.get_patient_normparam_prediction(sigma_predictions, batch_ids)
valid_pid2musigma = {}
for pid in pid2mu.iterkeys():
valid_pid2musigma[pid] = {'mu': pid2mu[pid], 'sigma': pid2sigma[pid]}
valid_avg_patient_predictions = config().get_avg_patient_predictions(batch_predictions, batch_ids, mean='geometric')
patient_targets = utils_heart.get_patient_average_heaviside_predictions(batch_targets, batch_ids)
assert valid_avg_patient_predictions.viewkeys() == patient_targets.viewkeys()
crpss_sys, crpss_dst = [], []
for id in valid_avg_patient_predictions.iterkeys():
crpss_sys.append(utils_heart.crps(valid_avg_patient_predictions[id][0], patient_targets[id][0]))
crpss_dst.append(utils_heart.crps(valid_avg_patient_predictions[id][1], patient_targets[id][1]))
print id, 0.5 * (crpss_sys[-1] + crpss_dst[-1]), crpss_sys[-1], crpss_dst[-1]
crps0, crps1 = np.mean(crpss_sys), np.mean(crpss_dst)
print 'Valid crps average: ', 0.5 * (crps0 + crps1)
print ' generating predictions for the train set'
train_data_iterator = config().train_data_iterator
train_data_iterator.transformation_params = config().valid_transformation_params
train_data_iterator.full_batch = False
train_data_iterator.random = False
train_data_iterator.infinite = False
batch_predictions, batch_targets, batch_ids = [], [], []
pid2sigma = utils_heart.get_patient_normparam_prediction(
sigma_predictions, batch_ids)
valid_pid2musigma = {}
for pid in pid2mu.iterkeys():
valid_pid2musigma[pid] = {'mu': pid2mu[pid], 'sigma': pid2sigma[pid]}
valid_avg_patient_predictions = config().get_avg_patient_predictions(
batch_predictions, batch_ids, mean='geometric')
patient_targets = utils_heart.get_patient_average_heaviside_predictions(
batch_targets, batch_ids)
assert valid_avg_patient_predictions.viewkeys() == patient_targets.viewkeys()
crpss_sys, crpss_dst = [], []
for id in valid_avg_patient_predictions.iterkeys():
crpss_sys.append(
utils_heart.crps(valid_avg_patient_predictions[id][0],
patient_targets[id][0]))
crpss_dst.append(
utils_heart.crps(valid_avg_patient_predictions[id][1],
patient_targets[id][1]))
print id, 0.5 * (crpss_sys[-1] +
crpss_dst[-1]), crpss_sys[-1], crpss_dst[-1]
crps0, crps1 = np.mean(crpss_sys), np.mean(crpss_dst)
print 'Valid crps average: ', 0.5 * (crps0 + crps1)
print ' generating predictions for the train set'
train_data_iterator = config().train_data_iterator
train_data_iterator.transformation_params = config(
).valid_transformation_params
train_data_iterator.full_batch = False
