def fix_submission(submission_path_meta, submission_path_ss, submission_path_mix, threshold=0.03): pid2cdf0_meta, pid2cdf1_meta = read_submission(submission_path_meta) pid2cdf0_ss, pid2cdf1_ss = read_submission(submission_path_ss) pid2cdf0_mix, pid2cdf1_mix = read_submission(submission_path_mix) pid2cdf0_fixed, pid2cdf1_fixed = {}, {} patient_ids = pid2cdf0_meta.keys() assert patient_ids == pid2cdf0_ss.keys() assert patient_ids == pid2cdf0_mix.keys() # no prediction from metamodel -> take prediction from slice model for pid in patient_ids: pid2cdf0_fixed[pid] = pid2cdf0_ss[pid] if pid2cdf0_meta[pid] is None else pid2cdf0_meta[pid] pid2cdf1_fixed[pid] = pid2cdf1_ss[pid] if pid2cdf1_meta[pid] is None else pid2cdf1_meta[pid] # metamodel disagrees with slice model -> take prediction from mixture ensemble for pid in patient_ids: crps0 = utils_heart.crps(pid2cdf0_fixed[pid], pid2cdf0_ss[pid]) if crps0 > threshold: print 'sys', pid, crps0 pid2cdf0_fixed[pid] = pid2cdf0_mix[pid] if crps0 > threshold else pid2cdf0_fixed[pid] crps1 = utils_heart.crps(pid2cdf1_fixed[pid], pid2cdf1_ss[pid]) if crps1 > threshold: print 'dst', pid, crps1 pid2cdf1_fixed[pid] = pid2cdf1_mix[pid] if crps1 > threshold else pid2cdf1_fixed[pid] fixed_predictions = {} for pid in patient_ids: fixed_predictions[pid] = [pid2cdf0_fixed[pid], pid2cdf1_fixed[pid]] meta_expid = submission_path_meta.split('-')[-1].replace('.csv', '') ss_expid = submission_path_ss.split('-')[-1].replace('.csv', '') mix_expid = submission_path_mix.split('-')[-1].replace('.csv', '') fixed_submission_path = SUBMISSION_PATH + 'ira_%s-%s-%s.csv' % (meta_expid, ss_expid, mix_expid) utils.save_submission(fixed_predictions, fixed_submission_path) print 'Submission save to', fixed_submission_path
assert patient_data['patient'] == patient_id patient_data["systole"] = np.concatenate( (patient_data["systole"], systole_prediction[None, :]), axis=0) patient_data["diastole"] = np.concatenate( (patient_data["diastole"], diastole_prediction[None, :]), axis=0) avg_patient_predictions = config().get_avg_patient_predictions( batch_predictions, batch_ids, mean=mean) patient_targets = utils_heart.get_patient_average_heaviside_predictions( batch_targets, batch_ids, mean=mean) assert avg_patient_predictions.viewkeys() == patient_targets.viewkeys() crpss_sys, crpss_dst = [], [] for id in avg_patient_predictions.iterkeys(): crpss_sys.append( utils_heart.crps(avg_patient_predictions[id][0], patient_targets[id][0])) crpss_dst.append( utils_heart.crps(avg_patient_predictions[id][1], patient_targets[id][1])) crps0, crps1 = np.mean(crpss_sys), np.mean(crpss_dst) print '\nValidation CRPS: ', crps0, crps1, 0.5 * (crps0 + crps1) utils.save_pkl(avg_patient_predictions, valid_prediction_path) print ' validation predictions saved to %s' % valid_prediction_path print # test test_data_iterator = config().test_data_iterator if n_tta_iterations == 1: test_data_iterator.transformation_params = config( ).valid_transformation_params
mu_predictions.append(iter_mu()) sigma_predictions.append(iter_sigma()) pid2mu = utils_heart.get_patient_normparam_prediction(mu_predictions, batch_ids) pid2sigma = utils_heart.get_patient_normparam_prediction(sigma_predictions, batch_ids) valid_pid2musigma = {} for pid in pid2mu.keys(): valid_pid2musigma[pid] = {'mu': pid2mu[pid], 'sigma': pid2sigma[pid]} valid_avg_patient_predictions = config().get_avg_patient_predictions(batch_predictions, batch_ids, mean='geometric') patient_targets = utils_heart.get_patient_average_heaviside_predictions(batch_targets, batch_ids) assert valid_avg_patient_predictions.keys() == patient_targets.keys() crpss_sys, crpss_dst = [], [] for id in valid_avg_patient_predictions.keys(): crpss_sys.append(utils_heart.crps(valid_avg_patient_predictions[id][0], patient_targets[id][0])) crpss_dst.append(utils_heart.crps(valid_avg_patient_predictions[id][1], patient_targets[id][1])) print(id, 0.5 * (crpss_sys[-1] + crpss_dst[-1]), crpss_sys[-1], crpss_dst[-1]) crps0, crps1 = np.mean(crpss_sys), np.mean(crpss_dst) print('Valid crps average: ', 0.5 * (crps0 + crps1)) print(' generating predictions for the train set') train_data_iterator = config().train_data_iterator train_data_iterator.transformation_params = config().valid_transformation_params train_data_iterator.full_batch = False train_data_iterator.random = False train_data_iterator.infinite = False batch_predictions, batch_targets, batch_ids = [], [], []
mu_predictions.append(iter_mu()) sigma_predictions.append(iter_sigma()) pid2mu = utils_heart.get_patient_normparam_prediction(mu_predictions, batch_ids) pid2sigma = utils_heart.get_patient_normparam_prediction(sigma_predictions, batch_ids) valid_pid2musigma = {} for pid in pid2mu.iterkeys(): valid_pid2musigma[pid] = {'mu': pid2mu[pid], 'sigma': pid2sigma[pid]} valid_avg_patient_predictions = config().get_avg_patient_predictions(batch_predictions, batch_ids, mean='geometric') patient_targets = utils_heart.get_patient_average_heaviside_predictions(batch_targets, batch_ids) assert valid_avg_patient_predictions.viewkeys() == patient_targets.viewkeys() crpss_sys, crpss_dst = [], [] for id in valid_avg_patient_predictions.iterkeys(): crpss_sys.append(utils_heart.crps(valid_avg_patient_predictions[id][0], patient_targets[id][0])) crpss_dst.append(utils_heart.crps(valid_avg_patient_predictions[id][1], patient_targets[id][1])) print id, 0.5 * (crpss_sys[-1] + crpss_dst[-1]), crpss_sys[-1], crpss_dst[-1] crps0, crps1 = np.mean(crpss_sys), np.mean(crpss_dst) print 'Valid crps average: ', 0.5 * (crps0 + crps1) print ' generating predictions for the train set' train_data_iterator = config().train_data_iterator train_data_iterator.transformation_params = config().valid_transformation_params train_data_iterator.full_batch = False train_data_iterator.random = False train_data_iterator.infinite = False batch_predictions, batch_targets, batch_ids = [], [], []
pid2sigma = utils_heart.get_patient_normparam_prediction( sigma_predictions, batch_ids) valid_pid2musigma = {} for pid in pid2mu.iterkeys(): valid_pid2musigma[pid] = {'mu': pid2mu[pid], 'sigma': pid2sigma[pid]} valid_avg_patient_predictions = config().get_avg_patient_predictions( batch_predictions, batch_ids, mean='geometric') patient_targets = utils_heart.get_patient_average_heaviside_predictions( batch_targets, batch_ids) assert valid_avg_patient_predictions.viewkeys() == patient_targets.viewkeys() crpss_sys, crpss_dst = [], [] for id in valid_avg_patient_predictions.iterkeys(): crpss_sys.append( utils_heart.crps(valid_avg_patient_predictions[id][0], patient_targets[id][0])) crpss_dst.append( utils_heart.crps(valid_avg_patient_predictions[id][1], patient_targets[id][1])) print id, 0.5 * (crpss_sys[-1] + crpss_dst[-1]), crpss_sys[-1], crpss_dst[-1] crps0, crps1 = np.mean(crpss_sys), np.mean(crpss_dst) print 'Valid crps average: ', 0.5 * (crps0 + crps1) print ' generating predictions for the train set' train_data_iterator = config().train_data_iterator train_data_iterator.transformation_params = config( ).valid_transformation_params train_data_iterator.full_batch = False