예제 #1
0
def calculate_tta_average(predictions, average_method, average_systole, average_diastole):
    already_printed = False
    for prediction in predictions:
        if prediction["systole"].size>0 and prediction["diastole"].size>0:
            assert np.isfinite(prediction["systole"][None,:,:]).all()
            assert np.isfinite(prediction["diastole"][None,:,:]).all()
            prediction["systole_average"] = average_method(prediction["systole"][None,:,:], average=average_systole)
            prediction["diastole_average"] = average_method(prediction["diastole"][None,:,:], average=average_diastole)
            try:
                test_if_valid_distribution(prediction["systole_average"])
                test_if_valid_distribution(prediction["diastole_average"])
            except:
                if not already_printed:
                    #print "WARNING: These distributions are not distributions"
                    already_printed = True
                prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
                prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
                try:
                    test_if_valid_distribution(prediction["systole_average"])
                    test_if_valid_distribution(prediction["diastole_average"])
                except:
                    prediction["systole_average"] = None
                    prediction["diastole_average"] = None
        else:
            # average distributions get zero weight later on
            #prediction["systole_average"] = average_systole
            #prediction["diastole_average"] = average_diastole
            prediction["systole_average"] = None
            prediction["diastole_average"] = None
예제 #2
0
def calculate_tta_average(predictions, average_method, average_systole, average_diastole):
    already_printed = False
    for prediction in predictions:
        if prediction["systole"].size>0 and prediction["diastole"].size>0:
            assert np.isfinite(prediction["systole"][None,:,:]).all()
            assert np.isfinite(prediction["diastole"][None,:,:]).all()
            prediction["systole_average"] = average_method(prediction["systole"][None,:,:], average=average_systole)
            prediction["diastole_average"] = average_method(prediction["diastole"][None,:,:], average=average_diastole)
            try:
                test_if_valid_distribution(prediction["systole_average"])
                test_if_valid_distribution(prediction["diastole_average"])
            except:
                if not already_printed:
                    #print "WARNING: These distributions are not distributions"
                    already_printed = True
                prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
                prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
                try:
                    test_if_valid_distribution(prediction["systole_average"])
                    test_if_valid_distribution(prediction["diastole_average"])
                except:
                    prediction["systole_average"] = None
                    prediction["diastole_average"] = None
        else:
            # average distributions get zero weight later on
            #prediction["systole_average"] = average_systole
            #prediction["diastole_average"] = average_diastole
            prediction["systole_average"] = None
            prediction["diastole_average"] = None
예제 #3
0
def _make_valid_distributions(metadata):
    for prediction in metadata['predictions']:
        prediction["systole_average"] = postprocess.make_monotone_distribution(
            prediction["systole_average"])
        postprocess.test_if_valid_distribution(prediction["systole_average"])
        prediction[
            "diastole_average"] = postprocess.make_monotone_distribution(
                prediction["diastole_average"])
        postprocess.test_if_valid_distribution(prediction["diastole_average"])
def dump_predictions(result_ensemble, submission_path):

    _, _, test_idx = _get_train_val_test_ids()
    test_ids = np.where(test_idx)[0] + 1

    preds_sys = result_ensemble["predictions_systole"][np.where(test_idx)]
    preds_dia = result_ensemble["predictions_diastole"][np.where(test_idx)]
    print "dumping submission file to %s" % submission_path
    with open(submission_path, 'w') as csvfile:
        csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        csvwriter.writerow(['Id'] + ['P%d'%i for i in xrange(600)])
        for pid, pat_pred_sys, pat_pred_dia in zip(test_ids, preds_sys, preds_dia):
            pat_pred_sys = postprocess.make_monotone_distribution(pat_pred_sys)
            pat_pred_dia = postprocess.make_monotone_distribution(pat_pred_dia)
            csvwriter.writerow(["%d_Diastole" % pid] + ["%.18f" % p for p in pat_pred_dia.flatten()])
            csvwriter.writerow(["%d_Systole" % pid] + ["%.18f" % p for p in pat_pred_sys.flatten()])
    print "submission file dumped"
예제 #5
0
def _create_ensembles(metadatas):
    # aggregate predictions and targets
    mask, preds_sys, preds_dia = _create_prediction_matrix(metadatas)
    targets_sys, targets_dia = _create_label_matrix()
    print(mask.mean())

    # initialise weights
    nr_models = len(metadatas)
    w_init = np.ones((nr_models, ), dtype='float32') / nr_models

    # split data
    _, val_idx, test_idx = _get_train_val_test_ids()
    mask_val, mask_test = mask[:, val_idx], mask[:, test_idx]
    targets_sys_val = targets_sys[val_idx[:len(targets_sys)], :]
    targets_dia_val = targets_dia[val_idx[:len(targets_dia)], :]
    preds_sys_val, preds_sys_test = preds_sys[:,
                                              val_idx, :], preds_sys[:,
                                                                     test_idx, :]
    preds_dia_val, preds_dia_test = preds_dia[:,
                                              val_idx, :], preds_dia[:,
                                                                     test_idx, :]

    ## CREATE SYSTOLE AND DIASTOLE PREDICTION USING LOOXVAL
    if DO_XVAL:
        print("Making systole ensemble")
        print("  Doing leave one patient out xval")
        nr_val_patients = len(targets_sys_val)
        train_errs_sys = []
        val_errs_sys = []
        w_iters_sys = []
        for val_pid in range(nr_val_patients):
            print("    - fold %d" % val_pid)
            w_iter = _find_weights(
                w_init,
                np.hstack(
                    (preds_sys_val[:, :val_pid], preds_sys_val[:,
                                                               val_pid + 1:])),
                np.vstack((targets_sys_val[:val_pid],
                           targets_sys_val[val_pid + 1:])),
                np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
            )
            train_err = _eval_weights(
                w_iter,
                np.hstack(
                    (preds_sys_val[:, :val_pid], preds_sys_val[:,
                                                               val_pid + 1:])),
                np.vstack((targets_sys_val[:val_pid],
                           targets_sys_val[val_pid + 1:])),
                np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
            )
            val_err = _eval_weights(
                w_iter,
                preds_sys_val[:, val_pid:val_pid + 1],
                targets_sys_val[val_pid:val_pid + 1],
                mask_val[:, val_pid:val_pid + 1],
            )
            print("      train_err: %.4f,  val_err: %.4f" %
                  (train_err, val_err))
            train_errs_sys.append(train_err)
            val_errs_sys.append(val_err)
            w_iters_sys.append(w_iter)

        expected_systole_loss = np.mean(val_errs_sys)
        print("  average train err: %.4f" % np.mean(train_errs_sys))
        print("  average valid err: %.4f" % np.mean(val_errs_sys))

        print("Making diastole ensemble")
        print("  Doing leave one patient out xval")
        nr_val_patients = len(targets_dia_val)
        train_errs_dia = []
        val_errs_dia = []
        w_iters_dia = []
        for val_pid in range(nr_val_patients):
            print("    - fold %d" % val_pid)
            w_iter = _find_weights(
                w_init,
                np.hstack(
                    (preds_dia_val[:, :val_pid], preds_dia_val[:,
                                                               val_pid + 1:])),
                np.vstack((targets_dia_val[:val_pid],
                           targets_dia_val[val_pid + 1:])),
                np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
            )
            train_err = _eval_weights(
                w_iter,
                np.hstack(
                    (preds_dia_val[:, :val_pid], preds_dia_val[:,
                                                               val_pid + 1:])),
                np.vstack((targets_dia_val[:val_pid],
                           targets_dia_val[val_pid + 1:])),
                np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
            )
            val_err = _eval_weights(
                w_iter,
                preds_dia_val[:, val_pid:val_pid + 1],
                targets_dia_val[val_pid:val_pid + 1],
                mask_val[:, val_pid:val_pid + 1],
            )
            print("      train_err: %.4f,  val_err: %.4f" %
                  (train_err, val_err))
            train_errs_dia.append(train_err)
            val_errs_dia.append(val_err)
            w_iters_dia.append(w_iter)

        expected_diastole_loss = np.mean(val_errs_dia)
        print("  average train err: %.4f" % np.mean(train_errs_dia))
        print("  average valid err: %.4f" % np.mean(val_errs_dia))

        ## MAKE ENSEMBLE USING THE FULL VALIDATION SET
        print("Fitting weights on the entire validation set")
        w_sys = _find_weights(w_init, preds_sys_val, targets_sys_val, mask_val)
        w_dia = _find_weights(w_init, preds_dia_val, targets_dia_val, mask_val)

        # Print the result
        print()
        print("dia   sys ")
        sort_key = lambda x: -x[1] - x[2]
        for metadata, weight_sys, weight_dia in sorted(zip(
                metadatas, w_sys, w_dia),
                                                       key=sort_key):
            print("%4.1f  %4.1f : %s" % (weight_sys * 100, weight_dia * 100,
                                         metadata["configuration_file"]))

    ## SELECT THE TOP MODELS AND RETRAIN ONCE MORE
    print()
    print("Selecting the top %d models and retraining" % SELECT_TOP)
    sorted_models, sorted_w_sys, sorted_w_dia = list(
        zip(*sorted(zip(metadatas, w_sys, w_dia), key=sort_key)))
    top_models = sorted_models[:SELECT_TOP]

    w_init_sys_top = np.array(sorted_w_sys[:SELECT_TOP]) / np.array(
        sorted_w_sys[:SELECT_TOP]).sum()
    w_init_dia_top = np.array(sorted_w_dia[:SELECT_TOP]) / np.array(
        sorted_w_dia[:SELECT_TOP]).sum()
    mask_top, preds_sys_top, preds_dia_top = _create_prediction_matrix(
        top_models)

    mask_top_val, mask_top_test = mask_top[:, val_idx], mask_top[:, test_idx]
    preds_sys_top_val, preds_sys_top_test = preds_sys_top[:,
                                                          val_idx, :], preds_sys_top[:,
                                                                                     test_idx, :]
    preds_dia_top_val, preds_dia_top_test = preds_dia_top[:,
                                                          val_idx, :], preds_dia_top[:,
                                                                                     test_idx, :]

    w_sys_top = _find_weights(w_init_sys_top, preds_sys_top_val,
                              targets_sys_val, mask_top_val)
    w_dia_top = _find_weights(w_init_dia_top, preds_dia_top_val,
                              targets_dia_val, mask_top_val)

    sys_err = _eval_weights(w_sys_top, preds_sys_top_val, targets_sys_val,
                            mask_top_val)
    dia_err = _eval_weights(w_dia_top, preds_dia_top_val, targets_dia_val,
                            mask_top_val)

    print()
    print("dia   sys ")
    sort_key = lambda x: -x[1] - x[2]
    for metadata, weight_sys, weight_dia in zip(top_models, w_sys_top,
                                                w_dia_top):
        print("%4.1f  %4.1f : %s" % (weight_sys * 100, weight_dia * 100,
                                     metadata["configuration_file"]))

    print("Final scores on the validation set:")
    print("  systole:  %.4f" % sys_err)
    print("  diastole: %.4f" % dia_err)
    print("  average:  %.4f" % ((sys_err + dia_err) / 2.0))
    if DO_XVAL:
        print("Expected leaderboard scores:")
        print("  systole:  %.4f" % expected_systole_loss)
        print("  diastole: %.4f" % expected_diastole_loss)
        print("  average:  %.4f" %
              ((expected_systole_loss + expected_diastole_loss) / 2.0))

    ## COMPUTE TEST PREDICTIONS
    test_ids = np.where(test_idx)[0] + 1
    preds_sys = _compute_predictions_ensemble(w_sys_top, preds_sys_top_test,
                                              mask_top_test)
    preds_dia = _compute_predictions_ensemble(w_dia_top, preds_dia_top_test,
                                              mask_top_test)

    submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
    print("dumping submission file to %s" % submission_path)
    with open(submission_path, 'w') as csvfile:
        csvwriter = csv.writer(csvfile,
                               delimiter=',',
                               quotechar='|',
                               quoting=csv.QUOTE_MINIMAL)
        csvwriter.writerow(['Id'] + ['P%d' % i for i in range(600)])
        for pid, pat_pred_sys, pat_pred_dia in zip(test_ids, preds_sys,
                                                   preds_dia):
            pat_pred_sys = postprocess.make_monotone_distribution(pat_pred_sys)
            pat_pred_dia = postprocess.make_monotone_distribution(pat_pred_dia)
            csvwriter.writerow(["%d_Diastole" % pid] +
                               ["%.18f" % p for p in pat_pred_dia.flatten()])
            csvwriter.writerow(["%d_Systole" % pid] +
                               ["%.18f" % p for p in pat_pred_sys.flatten()])
    print("submission file dumped")
def _make_valid_distributions(metadata):
    for prediction in metadata['predictions']:
        prediction["systole_average"] = postprocess.make_monotone_distribution(prediction["systole_average"])
        postprocess.test_if_valid_distribution(prediction["systole_average"])
        prediction["diastole_average"] = postprocess.make_monotone_distribution(prediction["diastole_average"])
        postprocess.test_if_valid_distribution(prediction["diastole_average"])
    pats_predicted = {set:[] for set in data_loader.patient_folders}
    for prediction in metadata['predictions']:
        pid = prediction['patient']
        pset = data_loader.id_to_index_map[pid][0]
        if _is_empty_prediction(prediction):
            pats_not_predicted[pset].append(pid)
        else:
            pats_predicted[pset].append(pid)
    return pats_not_predicted, pats_predicted


##### TTA AVERAGING ######
#========================#


average_systole = postprocess.make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:,1]]), axis=0))
average_diastole = postprocess.make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in data_loader.regular_labels[:,2]]), axis=0))


def generate_information_weight_matrix(expert_predictions, average_distribution, eps=1e-14, KL_weight = 1.0, cross_entropy_weight=1.0, expert_weights=None):
    pdf = utils.cdf_to_pdf(expert_predictions)
    average_pdf = utils.cdf_to_pdf(average_distribution)
    average_pdf[average_pdf<=0] = np.min(average_pdf[average_pdf>0])/2  # KL is not defined when Q=0 and P is not
    inside = pdf * (np.log(pdf) - np.log(average_pdf[None,None,:]))
    inside[pdf<=0] = 0  # (xlog(x) of zero is zero)
    KL_distance_from_average = np.sum(inside, axis=2)  # (NUM_EXPERTS, NUM_VALIDATIONS)
    assert np.isfinite(KL_distance_from_average).all()

    clipped_predictions = np.clip(expert_predictions, 0.0, 1.0)
    cross_entropy_per_sample = - (    average_distribution[None,None,:]  * np.log(   clipped_predictions+eps) +\
                                  (1.-average_distribution[None,None,:]) * np.log(1.-clipped_predictions+eps) )
예제 #8
0
def predict_slice_model(expid, outfile, mfile=None):
    metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)

    if theano.config.optimizer != "fast_run":
        print "WARNING: not running in fast mode!"

    print "Build model"
    interface_layers = config().build_model()

    output_layers = interface_layers["outputs"]
    input_layers = interface_layers["inputs"]
    top_layer = lasagne.layers.MergeLayer(
        incomings=output_layers.values()
    )
    _check_slicemodel(input_layers)

    # Print the architecture
    _print_architecture(top_layer)

    xs_shared = {
        key: lasagne.utils.shared_empty(dim=len(l_in.output_shape), dtype='float32') for (key, l_in) in input_layers.iteritems()
    }
    idx = T.lscalar('idx')

    givens = dict()

    for key in input_layers.keys():
        if key=="sunny":
            givens[input_layers[key].input_var] = xs_shared[key][idx*config().sunny_batch_size:(idx+1)*config().sunny_batch_size]
        else:
            givens[input_layers[key].input_var] = xs_shared[key][idx*config().batch_size:(idx+1)*config().batch_size]

    network_outputs = [
        lasagne.layers.helper.get_output(network_output_layer, deterministic=True)
        for network_output_layer in output_layers.values()
    ]

    iter_test = theano.function([idx], network_outputs + theano_printer.get_the_stuff_to_print(),
                                 givens=givens, on_unused_input="ignore",
                                 # mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
                                 )

    print "Load model parameters for resuming"
    resume_metadata = np.load(metadata_path)
    lasagne.layers.set_all_param_values(top_layer, resume_metadata['param_values'])
    num_batches_chunk = config().batches_per_chunk
    num_batches = get_number_of_test_batches()
    num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))

    chunks_train_idcs = range(1, num_chunks+1)

    create_test_gen = partial(config().create_test_gen,
                              required_input_keys = xs_shared.keys(),
                              required_output_keys = ["patients", "slices"],
                              )

    print "Generate predictions with this model"
    start_time = time.time()
    prev_time = start_time


    predictions = [{"patient": i+1,
                    "slices": {
                        slice_id: {
                            "systole": np.zeros((0,600)),
                            "diastole": np.zeros((0,600))
                        } for slice_id in data_loader.get_slice_ids_for_patient(i+1)
                    }
                   } for i in xrange(NUM_PATIENTS)]


    # Loop over data and generate predictions
    for e, test_data in izip(itertools.count(start=1), buffering.buffered_gen_threaded(create_test_gen())):
        print "  load testing data onto GPU"

        for key in xs_shared:
            xs_shared[key].set_value(test_data["input"][key])


        patient_ids = test_data["output"]["patients"]
        slice_ids = test_data["output"]["slices"]
        print "  patients:", " ".join(map(str, patient_ids))
        print "  chunk %d/%d" % (e, num_chunks)

        for b in xrange(num_batches_chunk):
            iter_result = iter_test(b)
            network_outputs = tuple(iter_result[:len(output_layers)])
            network_outputs_dict = {output_layers.keys()[i]: network_outputs[i] for i in xrange(len(output_layers))}
            kaggle_systoles, kaggle_diastoles = config().postprocess(network_outputs_dict)
            kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype('float64'), kaggle_diastoles.astype('float64')
            for idx, (patient_id, slice_id) in enumerate(
                    zip(patient_ids[b*config().batch_size:(b+1)*config().batch_size],
                        slice_ids[b*config().batch_size:(b+1)*config().batch_size])):
                if patient_id != 0:
                    index = patient_id-1
                    patient_data = predictions[index]
                    assert patient_id==patient_data["patient"]
                    patient_slice_data = patient_data["slices"][slice_id]
                    patient_slice_data["systole"] =  np.concatenate((patient_slice_data["systole"],  kaggle_systoles[idx:idx+1,:]),axis=0)
                    patient_slice_data["diastole"] = np.concatenate((patient_slice_data["diastole"], kaggle_diastoles[idx:idx+1,:]),axis=0)

        now = time.time()
        time_since_start = now - start_time
        time_since_prev = now - prev_time
        prev_time = now
        est_time_left = time_since_start * (float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
        eta = datetime.now() + timedelta(seconds=est_time_left)
        eta_str = eta.strftime("%c")
        print "  %s since start (%.2f s)" % (utils.hms(time_since_start), time_since_prev)
        print "  estimated %s to go (ETA: %s)" % (utils.hms(est_time_left), eta_str)
        print

    # Average predictions
    already_printed = False
    for prediction in predictions:
        for prediction_slice_id in prediction["slices"]:
            prediction_slice = prediction["slices"][prediction_slice_id]
            if prediction_slice["systole"].size>0 and prediction_slice["diastole"].size>0:
                average_method =  getattr(config(), 'tta_average_method', partial(np.mean, axis=0))
                prediction_slice["systole_average"] = average_method(prediction_slice["systole"])
                prediction_slice["diastole_average"] = average_method(prediction_slice["diastole"])
                try:
                    test_if_valid_distribution(prediction_slice["systole_average"])
                    test_if_valid_distribution(prediction_slice["diastole_average"])
                except:
                    if not already_printed:
                        print "WARNING: These distributions are not distributions"
                        already_printed = True
                    prediction_slice["systole_average"] = make_monotone_distribution(prediction_slice["systole_average"])
                    prediction_slice["diastole_average"] = make_monotone_distribution(prediction_slice["diastole_average"])


    print "Calculating training and validation set scores for reference"
    # Add CRPS scores to the predictions
    # Iterate over train and validation sets
    for patient_ids, set_name in [(validation_patients_indices, "validation"),
                                      (train_patients_indices,  "train")]:
        # Iterate over patients in the set
        for patient in patient_ids:
            prediction = predictions[patient-1]
            # Iterate over the slices
            for slice_id in prediction["slices"]:
                prediction_slice = prediction["slices"][slice_id]
                if "systole_average" in prediction_slice:
                    assert patient == regular_labels[patient-1, 0]
                    error_sys = CRSP(prediction_slice["systole_average"], regular_labels[patient-1, 1])
                    prediction_slice["systole_CRPS"] = error_sys
                    prediction_slice["target_systole"] = regular_labels[patient-1, 1]
                    error_dia = CRSP(prediction_slice["diastole_average"], regular_labels[patient-1, 2])
                    prediction_slice["diastole_CRPS"] = error_dia
                    prediction_slice["target_diastole"] = regular_labels[patient-1, 2]
                    prediction_slice["CRPS"] = 0.5 * error_sys + 0.5 * error_dia


    print "dumping prediction file to %s" % outfile
    with open(outfile, 'w') as f:
        pickle.dump({
                        'metadata_path': metadata_path,
                        'configuration_file': config().__name__,
                        'git_revision_hash': utils.get_git_revision_hash(),
                        'experiment_id': expid,
                        'time_since_start': time_since_start,
                        'param_values': lasagne.layers.get_all_param_values(top_layer),
                        'predictions_per_slice': predictions,
                    }, f, pickle.HIGHEST_PROTOCOL)
    print "prediction file dumped"


    return
def _create_ensembles(metadatas):
    # aggregate predictions and targets
    mask, preds_sys, preds_dia = _create_prediction_matrix(metadatas)
    targets_sys, targets_dia = _create_label_matrix()
    print mask.mean()

    # initialise weights
    nr_models = len(metadatas)
    w_init = np.ones((nr_models,), dtype='float32') / nr_models

    # split data
    _, val_idx, test_idx = _get_train_val_test_ids()
    mask_val, mask_test = mask[:, val_idx], mask[:, test_idx]
    targets_sys_val = targets_sys[val_idx[:len(targets_sys)], :]
    targets_dia_val = targets_dia[val_idx[:len(targets_dia)], :]
    preds_sys_val, preds_sys_test = preds_sys[:, val_idx, :], preds_sys[:, test_idx, :]
    preds_dia_val, preds_dia_test = preds_dia[:, val_idx, :], preds_dia[:, test_idx, :]

    ## CREATE SYSTOLE AND DIASTOLE PREDICTION USING LOOXVAL
    if DO_XVAL:
        print "Making systole ensemble"
        print "  Doing leave one patient out xval"
        nr_val_patients = len(targets_sys_val)
        train_errs_sys = []
        val_errs_sys = []
        w_iters_sys = []
        for val_pid in xrange(nr_val_patients):
            print "    - fold %d" % val_pid
            w_iter = _find_weights(
                w_init,
                np.hstack((preds_sys_val[:, :val_pid], preds_sys_val[:, val_pid + 1:])),
                np.vstack((targets_sys_val[:val_pid], targets_sys_val[val_pid + 1:])),
                np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
            )
            train_err = _eval_weights(
                w_iter,
                np.hstack((preds_sys_val[:, :val_pid], preds_sys_val[:, val_pid + 1:])),
                np.vstack((targets_sys_val[:val_pid], targets_sys_val[val_pid + 1:])),
                np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
            )
            val_err = _eval_weights(w_iter,
                                    preds_sys_val[:, val_pid:val_pid + 1],
                                    targets_sys_val[val_pid:val_pid + 1],
                                    mask_val[:, val_pid:val_pid + 1],
                                    )
            print "      train_err: %.4f,  val_err: %.4f" % (train_err, val_err)
            train_errs_sys.append(train_err)
            val_errs_sys.append(val_err)
            w_iters_sys.append(w_iter)

        expected_systole_loss = np.mean(val_errs_sys)
        print "  average train err: %.4f" % np.mean(train_errs_sys)
        print "  average valid err: %.4f" % np.mean(val_errs_sys)

        print "Making diastole ensemble"
        print "  Doing leave one patient out xval"
        nr_val_patients = len(targets_dia_val)
        train_errs_dia = []
        val_errs_dia = []
        w_iters_dia = []
        for val_pid in xrange(nr_val_patients):
            print "    - fold %d" % val_pid
            w_iter = _find_weights(
                w_init,
                np.hstack((preds_dia_val[:, :val_pid], preds_dia_val[:, val_pid + 1:])),
                np.vstack((targets_dia_val[:val_pid], targets_dia_val[val_pid + 1:])),
                np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
            )
            train_err = _eval_weights(
                w_iter,
                np.hstack((preds_dia_val[:, :val_pid], preds_dia_val[:, val_pid + 1:])),
                np.vstack((targets_dia_val[:val_pid], targets_dia_val[val_pid + 1:])),
                np.hstack((mask_val[:, :val_pid], mask_val[:, val_pid + 1:])),
            )
            val_err = _eval_weights(w_iter,
                                    preds_dia_val[:, val_pid:val_pid + 1],
                                    targets_dia_val[val_pid:val_pid + 1],
                                    mask_val[:, val_pid:val_pid + 1],
                                    )
            print "      train_err: %.4f,  val_err: %.4f" % (train_err, val_err)
            train_errs_dia.append(train_err)
            val_errs_dia.append(val_err)
            w_iters_dia.append(w_iter)

        expected_diastole_loss = np.mean(val_errs_dia)
        print "  average train err: %.4f" % np.mean(train_errs_dia)
        print "  average valid err: %.4f" % np.mean(val_errs_dia)

        ## MAKE ENSEMBLE USING THE FULL VALIDATION SET
        print "Fitting weights on the entire validation set"
        w_sys = _find_weights(w_init, preds_sys_val, targets_sys_val, mask_val)
        w_dia = _find_weights(w_init, preds_dia_val, targets_dia_val, mask_val)

        # Print the result
        print
        print "dia   sys "
        sort_key = lambda x: - x[1] - x[2]
        for metadata, weight_sys, weight_dia in sorted(zip(metadatas, w_sys, w_dia), key=sort_key):
            print "%4.1f  %4.1f : %s" % (weight_sys * 100, weight_dia * 100, metadata["configuration_file"])

    ## SELECT THE TOP MODELS AND RETRAIN ONCE MORE
    print
    print "Selecting the top %d models and retraining" % SELECT_TOP
    sorted_models, sorted_w_sys, sorted_w_dia = zip(*sorted(zip(metadatas, w_sys, w_dia), key=sort_key))
    top_models = sorted_models[:SELECT_TOP]

    w_init_sys_top = np.array(sorted_w_sys[:SELECT_TOP]) / np.array(sorted_w_sys[:SELECT_TOP]).sum()
    w_init_dia_top = np.array(sorted_w_dia[:SELECT_TOP]) / np.array(sorted_w_dia[:SELECT_TOP]).sum()
    mask_top, preds_sys_top, preds_dia_top = _create_prediction_matrix(top_models)

    mask_top_val, mask_top_test = mask_top[:, val_idx], mask_top[:, test_idx]
    preds_sys_top_val, preds_sys_top_test = preds_sys_top[:, val_idx, :], preds_sys_top[:, test_idx, :]
    preds_dia_top_val, preds_dia_top_test = preds_dia_top[:, val_idx, :], preds_dia_top[:, test_idx, :]

    w_sys_top = _find_weights(w_init_sys_top, preds_sys_top_val, targets_sys_val, mask_top_val)
    w_dia_top = _find_weights(w_init_dia_top, preds_dia_top_val, targets_dia_val, mask_top_val)

    sys_err = _eval_weights(w_sys_top, preds_sys_top_val, targets_sys_val, mask_top_val)
    dia_err = _eval_weights(w_dia_top, preds_dia_top_val, targets_dia_val, mask_top_val)

    print
    print "dia   sys "
    sort_key = lambda x: - x[1] - x[2]
    for metadata, weight_sys, weight_dia in zip(top_models, w_sys_top, w_dia_top):
        print "%4.1f  %4.1f : %s" % (weight_sys * 100, weight_dia * 100, metadata["configuration_file"])

    print "Final scores on the validation set:"
    print "  systole:  %.4f" % sys_err
    print "  diastole: %.4f" % dia_err
    print "  average:  %.4f" % ((sys_err + dia_err) / 2.0)
    if DO_XVAL:
        print "Expected leaderboard scores:"
        print "  systole:  %.4f" % expected_systole_loss
        print "  diastole: %.4f" % expected_diastole_loss
        print "  average:  %.4f" % ((expected_systole_loss + expected_diastole_loss) / 2.0)

    ## COMPUTE TEST PREDICTIONS
    test_ids = np.where(test_idx)[0] + 1
    preds_sys = _compute_predictions_ensemble(w_sys_top, preds_sys_top_test, mask_top_test)
    preds_dia = _compute_predictions_ensemble(w_dia_top, preds_dia_top_test, mask_top_test)

    submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()
    print "dumping submission file to %s" % submission_path
    with open(submission_path, 'w') as csvfile:
        csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        csvwriter.writerow(['Id'] + ['P%d' % i for i in xrange(600)])
        for pid, pat_pred_sys, pat_pred_dia in zip(test_ids, preds_sys, preds_dia):
            pat_pred_sys = postprocess.make_monotone_distribution(pat_pred_sys)
            pat_pred_dia = postprocess.make_monotone_distribution(pat_pred_dia)
            csvwriter.writerow(["%d_Diastole" % pid] + ["%.18f" % p for p in pat_pred_dia.flatten()])
            csvwriter.writerow(["%d_Systole" % pid] + ["%.18f" % p for p in pat_pred_sys.flatten()])
    print "submission file dumped"
예제 #10
0
def predict_model(expid, mfile=None):
    metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)
    prediction_path = INTERMEDIATE_PREDICTIONS_PATH + "%s.pkl" % expid
    submission_path = SUBMISSION_PATH + "%s.csv" % expid

    if theano.config.optimizer != "fast_run":
        print("WARNING: not running in fast mode!")

    print("Using")
    print("  %s" % metadata_path)
    print("To generate")
    print("  %s" % prediction_path)
    print("  %s" % submission_path)

    print("Build model")
    interface_layers = config().build_model()

    output_layers = interface_layers["outputs"]
    input_layers = interface_layers["inputs"]
    top_layer = lasagne.layers.MergeLayer(
        incomings=list(output_layers.values()))
    all_layers = lasagne.layers.get_all_layers(top_layer)
    num_params = lasagne.layers.count_params(top_layer)
    print("  number of parameters: %d" % num_params)
    print(string.ljust("  layer output shapes:", 36), end=' ')
    print(string.ljust("#params:", 10), end=' ')
    print("output shape:")
    for layer in all_layers[:-1]:
        name = string.ljust(layer.__class__.__name__, 32)
        num_param = sum(
            [np.prod(p.get_value().shape) for p in layer.get_params()])
        num_param = string.ljust(num_param.__str__(), 10)
        print("    %s %s %s" % (name, num_param, layer.output_shape))

    xs_shared = {
        key: lasagne.utils.shared_empty(dim=len(l_in.output_shape),
                                        dtype='float32')
        for (key, l_in) in input_layers.items()
    }
    idx = T.lscalar('idx')

    givens = dict()

    for key in list(input_layers.keys()):
        if key == "sunny":
            givens[input_layers[key].input_var] = xs_shared[key][idx * config(
            ).sunny_batch_size:(idx + 1) * config().sunny_batch_size]
        else:
            givens[input_layers[key].
                   input_var] = xs_shared[key][idx *
                                               config().batch_size:(idx + 1) *
                                               config().batch_size]

    network_outputs = [
        lasagne.layers.helper.get_output(network_output_layer,
                                         deterministic=True)
        for network_output_layer in list(output_layers.values())
    ]

    iter_test = theano.function(
        [idx],
        network_outputs + theano_printer.get_the_stuff_to_print(),
        givens=givens,
        on_unused_input="ignore",
        # mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
    )

    print("Load model parameters for resuming")
    resume_metadata = np.load(metadata_path)
    lasagne.layers.set_all_param_values(top_layer,
                                        resume_metadata['param_values'])
    num_batches_chunk = config().batches_per_chunk
    num_batches = get_number_of_test_batches()
    num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))

    chunks_train_idcs = list(range(1, num_chunks + 1))

    data_loader.filter_patient_folders()

    create_test_gen = partial(
        config().create_test_gen,
        required_input_keys=list(xs_shared.keys()),
        required_output_keys=[
            "patients", "classification_correction_function"
        ],
    )

    print("Generate predictions with this model")
    start_time = time.time()
    prev_time = start_time

    predictions = [{
        "patient": i + 1,
        "systole": np.zeros((0, 600)),
        "diastole": np.zeros((0, 600))
    } for i in range(NUM_PATIENTS)]

    for e, test_data in zip(itertools.count(start=1),
                            buffering.buffered_gen_threaded(
                                create_test_gen())):
        print("  load testing data onto GPU")

        for key in xs_shared:
            xs_shared[key].set_value(test_data["input"][key])

        patient_ids = test_data["output"]["patients"]
        classification_correction = test_data["output"][
            "classification_correction_function"]
        print("  patients:", " ".join(map(str, patient_ids)))
        print("  chunk %d/%d" % (e, num_chunks))

        for b in range(num_batches_chunk):
            iter_result = iter_test(b)
            network_outputs = tuple(iter_result[:len(output_layers)])
            network_outputs_dict = {
                list(output_layers.keys())[i]: network_outputs[i]
                for i in range(len(output_layers))
            }
            kaggle_systoles, kaggle_diastoles = config().postprocess(
                network_outputs_dict)
            kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype(
                'float64'), kaggle_diastoles.astype('float64')
            for idx, patient_id in enumerate(
                    patient_ids[b * config().batch_size:(b + 1) *
                                config().batch_size]):
                if patient_id != 0:
                    index = patient_id - 1
                    patient_data = predictions[index]
                    assert patient_id == patient_data["patient"]

                    kaggle_systole = kaggle_systoles[idx:idx + 1, :]
                    kaggle_diastole = kaggle_diastoles[idx:idx + 1, :]
                    assert np.isfinite(kaggle_systole).all() and np.isfinite(
                        kaggle_systole).all()
                    kaggle_systole = classification_correction[
                        b * config().batch_size + idx](kaggle_systole)
                    kaggle_diastole = classification_correction[
                        b * config().batch_size + idx](kaggle_diastole)
                    assert np.isfinite(kaggle_systole).all() and np.isfinite(
                        kaggle_systole).all()
                    patient_data["systole"] = np.concatenate(
                        (patient_data["systole"], kaggle_systole), axis=0)
                    patient_data["diastole"] = np.concatenate(
                        (patient_data["diastole"], kaggle_diastole), axis=0)

        now = time.time()
        time_since_start = now - start_time
        time_since_prev = now - prev_time
        prev_time = now
        est_time_left = time_since_start * (
            float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
        eta = datetime.now() + timedelta(seconds=est_time_left)
        eta_str = eta.strftime("%c")
        print("  %s since start (%.2f s)" %
              (utils.hms(time_since_start), time_since_prev))
        print("  estimated %s to go (ETA: %s)" %
              (utils.hms(est_time_left), eta_str))
        print()

    already_printed = False
    for prediction in predictions:
        if prediction["systole"].size > 0 and prediction["diastole"].size > 0:
            average_method = getattr(config(), 'tta_average_method',
                                     partial(np.mean, axis=0))
            prediction["systole_average"] = average_method(
                prediction["systole"])
            prediction["diastole_average"] = average_method(
                prediction["diastole"])
            try:
                test_if_valid_distribution(prediction["systole_average"])
                test_if_valid_distribution(prediction["diastole_average"])
            except:
                if not already_printed:
                    print("WARNING: These distributions are not distributions")
                    already_printed = True
                prediction["systole_average"] = make_monotone_distribution(
                    prediction["systole_average"])
                prediction["diastole_average"] = make_monotone_distribution(
                    prediction["diastole_average"])
                test_if_valid_distribution(prediction["systole_average"])
                test_if_valid_distribution(prediction["diastole_average"])

    print("Calculating training and validation set scores for reference")

    validation_dict = {}
    for patient_ids, set_name in [(validation_patients_indices, "validation"),
                                  (train_patients_indices, "train")]:
        errors = []
        for patient in patient_ids:
            prediction = predictions[patient - 1]
            if "systole_average" in prediction:
                assert patient == regular_labels[patient - 1, 0]
                error = CRSP(prediction["systole_average"],
                             regular_labels[patient - 1, 1])
                errors.append(error)
                error = CRSP(prediction["diastole_average"],
                             regular_labels[patient - 1, 2])
                errors.append(error)
        if len(errors) > 0:
            errors = np.array(errors)
            estimated_CRSP = np.mean(errors)
            print("  %s kaggle loss: %f" %
                  (string.rjust(set_name, 12), estimated_CRSP))
            validation_dict[set_name] = estimated_CRSP
        else:
            print("  %s kaggle loss: not calculated" %
                  (string.rjust(set_name, 12)))

    print("dumping prediction file to %s" % prediction_path)
    with open(prediction_path, 'w') as f:
        pickle.dump(
            {
                'metadata_path': metadata_path,
                'prediction_path': prediction_path,
                'submission_path': submission_path,
                'configuration_file': config().__name__,
                'git_revision_hash': utils.get_git_revision_hash(),
                'experiment_id': expid,
                'time_since_start': time_since_start,
                'param_values': lasagne.layers.get_all_param_values(top_layer),
                'predictions': predictions,
                'validation_errors': validation_dict,
            }, f, pickle.HIGHEST_PROTOCOL)
    print("prediction file dumped")

    print("dumping submission file to %s" % submission_path)
    with open(submission_path, 'w') as csvfile:
        csvwriter = csv.writer(csvfile,
                               delimiter=',',
                               quotechar='|',
                               quoting=csv.QUOTE_MINIMAL)
        csvwriter.writerow(['Id'] + ['P%d' % i for i in range(600)])
        for prediction in predictions:
            # the submission only has patients 501 to 700
            if prediction["patient"] in data_loader.test_patients_indices:
                if "diastole_average" not in prediction or "systole_average" not in prediction:
                    raise Exception("Not all test-set patients were predicted")
                csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + [
                    "%.18f" % p
                    for p in prediction["diastole_average"].flatten()
                ])
                csvwriter.writerow(["%d_Systole" % prediction["patient"]] + [
                    "%.18f" % p
                    for p in prediction["systole_average"].flatten()
                ])
    print("submission file dumped")

    return
예제 #11
0
def generate_final_predictions(
        final_predictions,
        prediction_tag,
        expert_predictions_matrix,
        masked_expert_predictions_matrix,
        test_expert_predictions_matrix,
        test_masked_expert_predictions_matrix,
        optimal_params,
        average_distribution,
        valid_labels,
        expert_pkl_files,
        expert_weight,
        disagreement_cutoff):
    cache_dict = dict()

    for final_prediction in final_predictions:
        patient_id = final_prediction['patient']
        if patient_id in train_patients_indices:
            continue

        print("   final prediction of patient %d" % patient_id)
        reoptimized = False
        # get me the data for this patient
        # (NUM_EXPERTS, 600)
        if patient_id in validation_patients_indices:
            idx = validation_patients_indices.index(patient_id)
            prediction_matrix = expert_predictions_matrix[:, idx, :]
            mask_matrix = masked_expert_predictions_matrix[:, idx]

        elif patient_id in test_patients_indices:
            idx = test_patients_indices.index(patient_id)
            prediction_matrix = test_expert_predictions_matrix[:, idx, :]
            mask_matrix = test_masked_expert_predictions_matrix[:, idx]

        else:
            raise "This patient is neither train, validation or test?"


        # Is there an expert in the set, which did not predict this patient?
        # if so, re-optimize our weights on the validation set!
        # so, if a model is unavailble, but should be available, re-optimize!
        if np.logical_and(np.logical_not(mask_matrix),  (expert_weight!=0.0)).any():
            if hash_mask(mask_matrix) in cache_dict:
                (optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
            else:
                expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
                    expert_mask=mask_matrix,
                    prediction_matrix=expert_predictions_matrix,
                    mask_matrix=masked_expert_predictions_matrix,
                    labels=valid_labels,
                    average_distribution=average_distribution,
                    )
                cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
            reoptimized = True
        else:
            optimal_params_for_this_patient = optimal_params



        while True: # do-while: break condition at the end
            last_mask_matrix = mask_matrix

            final_prediction[prediction_tag] = optimize_expert_weights(
                            expert_predictions=prediction_matrix[:,None,:],
                            mask_matrix=mask_matrix[:,None],
                            average_distribution=average_distribution,
                            do_optimization=False,
                            optimal_params=optimal_params_for_this_patient,
                            )

            ## find the disagreement between models
            # and remove the ones who disagree too much with the average
            disagreement = find_disagreement(
                expert_predictions=prediction_matrix,
                mask_matrix=mask_matrix,
                ground_truth=final_prediction[prediction_tag]
            )
            mask_matrix = (disagreement<disagreement_cutoff)

            ## RETRAIN THESE ENSEMBLE WEIGHTS ON THE VALIDATION SET IF NEEDED!
            if ((last_mask_matrix!=mask_matrix).any()
               and np.sum(mask_matrix)!=0):

                if hash_mask(mask_matrix) in cache_dict:
                    (optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
                else:
                    expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
                        expert_mask=mask_matrix,
                        prediction_matrix=expert_predictions_matrix,
                        mask_matrix=masked_expert_predictions_matrix,
                        labels=valid_labels,
                        average_distribution=average_distribution,
                        )
                    cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
                reoptimized = True
                continue
            else:
                break
        try:
            test_if_valid_distribution(final_prediction[prediction_tag])
        except:
            final_prediction[prediction_tag] = make_monotone_distribution(final_prediction[prediction_tag])
            test_if_valid_distribution(final_prediction[prediction_tag])

        if reoptimized:
            print("    Weight:  Name:")
            for expert_name, weight in zip(expert_pkl_files, expert_weight):
                if weight>0.:
                    print(string.rjust("%.3f%%" % (100*weight), 12), end=' ')
                    print(expert_name.split('/')[-1])
예제 #12
0
def merge_all_prediction_files(prediction_file_location = INTERMEDIATE_PREDICTIONS_PATH,
                               redo_tta = True):

    submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()

    # calculate the average distribution
    regular_labels = _load_file(_TRAIN_LABELS_PATH)

    average_systole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,1]]), axis=0))
    average_diastole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,2]]), axis=0))

    ss_expert_pkl_files = sorted([]
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout_seqshift_96.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_big_leaky_after_seqshift.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_big.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_zoom_mask_leaky_after.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky_after.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_maxout.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_nomask.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc64small_360_gauss_longer.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_128mm.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_96mm.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc80_framemax.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_128mm_96.pkl")
        +glob.glob(prediction_file_location+"j6_4ch.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll.pkl")
        +glob.glob(prediction_file_location+"j6_4ch_32mm_specialist.pkl")
        +glob.glob(prediction_file_location+"j6_4ch_128mm_specialist.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc64_leaky_convroll.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc80small_360_gauss_longer_augzoombright.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll_augzoombright.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_128mm_zoom.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_128mm_skew.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc64small_360.pkl")
    )


    fp_expert_pkl_files = sorted([]
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout_seqshift_96.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_big_leaky_after_seqshift.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_big.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_zoom_mask_leaky_after.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky_after.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom.pkl")
        +glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered.pkl")
        +glob.glob(prediction_file_location+"je_os_fixedaggr_rellocframe.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_filtered.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_framemax_reg.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv_augzoombright_short.pkl")
        +glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered_discs.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale64small_filtered_longer.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright_betterdist.pkl")
        +glob.glob(prediction_file_location+"je_os_segmentandintegrate_smartsigma_dropout.pkl")
    )

    everything = sorted([]
        +glob.glob(prediction_file_location+"*.pkl")
    )
    expert_pkl_files = ss_expert_pkl_files + fp_expert_pkl_files

    print("found %d/44 files" % len(expert_pkl_files))
    """
    # filter expert_pkl_files
    for file in expert_pkl_files[:]:
        try:
            with open(file, 'r') as f:
                print "testing file",file.split('/')[-1]
                data = pickle.load(f)
                if 'predictions' not in data.keys():
                    expert_pkl_files.remove(file)
                    print "                -> removed"
        except:
            print sys.exc_info()[0]
            expert_pkl_files.remove(file)
            print "                -> removed"
    """
    NUM_EXPERTS = len(expert_pkl_files)
    NUM_VALIDATIONS = len(validation_patients_indices)
    NUM_TESTS = len(test_patients_indices)

    systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
    diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')

    systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
    diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')

    test_systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
    test_diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')

    test_systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
    test_diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')

    for i,file in enumerate(expert_pkl_files):
        with open(file, 'r') as f:
            print()
            print("loading file",file.split('/')[-1])
            predictions = pickle.load(f)['predictions']

        if redo_tta:
            best_average_method = normalav
            best_average_crps = utils.maxfloat
            for average_method in [geomav,
                                   normalav,
                                   prodav,
                                   weighted_geom_method
                                   ]:
                calculate_tta_average(predictions, average_method, average_systole, average_diastole)

                crps = get_validate_crps(predictions, regular_labels)
                print(string.rjust(average_method.__name__,25),"->",crps)
                if crps<best_average_crps:
                    best_average_method = average_method
                    best_average_crps = crps

            print(" I choose you,", best_average_method.__name__)
            calculate_tta_average(predictions, best_average_method, average_systole, average_diastole)
            print(" validation loss:", get_validate_crps(predictions, regular_labels))

        for j,patient in enumerate(validation_patients_indices):
            prediction = predictions[patient-1]
            # average distributions get zero weight later on
            if "systole_average" in prediction and prediction["systole_average"] is not None:
                systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
            else:
                systole_masked_expert_predictions_matrix[i,j] = False

            if "diastole_average" in prediction and prediction["diastole_average"] is not None:
                diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
            else:
                diastole_masked_expert_predictions_matrix[i,j] = False

        for j,patient in enumerate(test_patients_indices):
            prediction = predictions[patient-1]
            # average distributions get zero weight later on
            if "systole_average" in prediction and prediction["systole_average"] is not None:
                test_systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
            else:
                test_systole_masked_expert_predictions_matrix[i,j] = False

            if "diastole_average" in prediction and prediction["diastole_average"] is not None:
                test_diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
            else:
                test_diastole_masked_expert_predictions_matrix[i,j] = False

        del predictions  # can be LOADS of data



    cv = [id-1 for id in validation_patients_indices]
    systole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,1].flatten()])
    systole_expert_weight, first_pass_sys_loss, systole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
        expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
        prediction_matrix=systole_expert_predictions_matrix,
        mask_matrix=systole_masked_expert_predictions_matrix,
        labels=systole_valid_labels,
        average_distribution=average_systole,
        )

    cv = [id-1 for id in validation_patients_indices]
    diastole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,2].flatten()])
    diastole_expert_weight, first_pass_dia_loss, diastole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
        expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
        prediction_matrix=diastole_expert_predictions_matrix,
        mask_matrix=diastole_masked_expert_predictions_matrix,
        labels=diastole_valid_labels,
        average_distribution=average_diastole,
        )


    # print the final weight of every expert
    print("  Systole:  Diastole: Name:")
    for expert_name, systole_weight, diastole_weight in zip(expert_pkl_files, systole_expert_weight, diastole_expert_weight):
        print(string.rjust("%.3f%%" % (100*systole_weight), 10), end=' ')
        print(string.rjust("%.3f%%" % (100*diastole_weight), 10), end=' ')
        print(expert_name.split('/')[-1])

    print()
    print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2))
    print()

    print()
    print("Average the experts according to these weights to find the final distribution")
    final_predictions = [{
                            "patient": i+1,
                            "final_systole": None,
                            "final_diastole": None
                        } for i in range(NUM_PATIENTS)]


    generate_final_predictions(
        final_predictions=final_predictions,
        prediction_tag="final_systole",
        expert_predictions_matrix=systole_expert_predictions_matrix,
        masked_expert_predictions_matrix=systole_masked_expert_predictions_matrix,
        test_expert_predictions_matrix=test_systole_expert_predictions_matrix,
        test_masked_expert_predictions_matrix=test_systole_masked_expert_predictions_matrix,
        optimal_params=systole_optimal_params,
        average_distribution=average_systole,
        valid_labels=systole_valid_labels,
        expert_pkl_files=expert_pkl_files,
        expert_weight=systole_expert_weight,
        disagreement_cutoff=0.01 # 0.01
    )

    generate_final_predictions(
        final_predictions=final_predictions,
        prediction_tag="final_diastole",
        expert_predictions_matrix=diastole_expert_predictions_matrix,
        masked_expert_predictions_matrix=diastole_masked_expert_predictions_matrix,
        test_expert_predictions_matrix=test_diastole_expert_predictions_matrix,
        test_masked_expert_predictions_matrix=test_diastole_masked_expert_predictions_matrix,
        optimal_params=diastole_optimal_params,
        average_distribution=average_diastole,
        valid_labels=diastole_valid_labels,
        expert_pkl_files=expert_pkl_files,
        expert_weight=diastole_expert_weight,
        disagreement_cutoff=0.015  # diastole has about 50% more error
    )

    print()
    print("Calculating training and validation set scores for reference")


    validation_dict = {}
    for patient_ids, set_name in [(validation_patients_indices, "validation")]:
        errors = []
        for patient in patient_ids:
            prediction = final_predictions[patient-1]

            if "final_systole" in prediction:
                assert patient == regular_labels[patient-1, 0]
                error1 = utils.CRSP(prediction["final_systole"], regular_labels[patient-1, 1])
                errors.append(error1)
                prediction["systole_crps_error"] = error1
                error2 = utils.CRSP(prediction["final_diastole"], regular_labels[patient-1, 2])
                errors.append(error2)
                prediction["diastole_crps_error"] = error1
                prediction["average_crps_error"] = 0.5*error1 + 0.5*error2

        if len(errors)>0:
            errors = np.array(errors)
            estimated_CRSP = np.mean(errors)
            print("  %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP))
            validation_dict[set_name] = estimated_CRSP
        else:
            print("  %s kaggle loss: not calculated" % (string.rjust(set_name, 12)))
    print("WARNING: both of the previous are overfitted!")

    print()
    print("estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2))
    print()


    print("dumping submission file to %s" % submission_path)
    with open(submission_path, 'w') as csvfile:
        csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        csvwriter.writerow(['Id'] + ['P%d'%i for i in range(600)])
        for prediction in final_predictions:
            # the submission only has patients 501 to 700
            if prediction["patient"] in test_patients_indices:
                if "final_diastole" not in prediction or "final_systole" not in prediction:
                    raise Exception("Not all test-set patients were predicted")
                csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_diastole"].flatten()])
                csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_systole"].flatten()])
    print("submission file dumped")
예제 #13
0
def predict_model(expid, mfile=None):
    metadata_path = MODEL_PATH + "%s.pkl" % (expid if not mfile else mfile)
    prediction_path = INTERMEDIATE_PREDICTIONS_PATH + "%s.pkl" % expid
    submission_path = SUBMISSION_PATH + "%s.csv" % expid

    if theano.config.optimizer != "fast_run":
        print "WARNING: not running in fast mode!"

    print "Using"
    print "  %s" % metadata_path
    print "To generate"
    print "  %s" % prediction_path
    print "  %s" % submission_path

    print "Build model"
    interface_layers = config().build_model()

    output_layers = interface_layers["outputs"]
    input_layers = interface_layers["inputs"]
    top_layer = lasagne.layers.MergeLayer(
        incomings=output_layers.values()
    )
    all_layers = lasagne.layers.get_all_layers(top_layer)
    num_params = lasagne.layers.count_params(top_layer)
    print "  number of parameters: %d" % num_params
    print string.ljust("  layer output shapes:",36),
    print string.ljust("#params:",10),
    print "output shape:"
    for layer in all_layers[:-1]:
        name = string.ljust(layer.__class__.__name__, 32)
        num_param = sum([np.prod(p.get_value().shape) for p in layer.get_params()])
        num_param = string.ljust(num_param.__str__(), 10)
        print "    %s %s %s" % (name,  num_param, layer.output_shape)

    xs_shared = {
        key: lasagne.utils.shared_empty(dim=len(l_in.output_shape), dtype='float32') for (key, l_in) in input_layers.iteritems()
    }
    idx = T.lscalar('idx')

    givens = dict()

    for key in input_layers.keys():
        if key=="sunny":
            givens[input_layers[key].input_var] = xs_shared[key][idx*config().sunny_batch_size:(idx+1)*config().sunny_batch_size]
        else:
            givens[input_layers[key].input_var] = xs_shared[key][idx*config().batch_size:(idx+1)*config().batch_size]

    network_outputs = [
        lasagne.layers.helper.get_output(network_output_layer, deterministic=True)
        for network_output_layer in output_layers.values()
    ]

    iter_test = theano.function([idx], network_outputs + theano_printer.get_the_stuff_to_print(),
                                 givens=givens, on_unused_input="ignore",
                                 # mode=NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True)
                                 )

    print "Load model parameters for resuming"
    resume_metadata = np.load(metadata_path)
    lasagne.layers.set_all_param_values(top_layer, resume_metadata['param_values'])
    num_batches_chunk = config().batches_per_chunk
    num_batches = get_number_of_test_batches()
    num_chunks = int(np.ceil(num_batches / float(config().batches_per_chunk)))

    chunks_train_idcs = range(1, num_chunks+1)

    data_loader.filter_patient_folders()

    create_test_gen = partial(config().create_test_gen,
                              required_input_keys = xs_shared.keys(),
                              required_output_keys = ["patients", "classification_correction_function"],
                              )

    print "Generate predictions with this model"
    start_time = time.time()
    prev_time = start_time


    predictions = [{"patient": i+1,
                    "systole": np.zeros((0,600)),
                    "diastole": np.zeros((0,600))
                    } for i in xrange(NUM_PATIENTS)]


    for e, test_data in izip(itertools.count(start=1), buffering.buffered_gen_threaded(create_test_gen())):
        print "  load testing data onto GPU"

        for key in xs_shared:
            xs_shared[key].set_value(test_data["input"][key])


        patient_ids = test_data["output"]["patients"]
        classification_correction = test_data["output"]["classification_correction_function"]
        print "  patients:", " ".join(map(str, patient_ids))
        print "  chunk %d/%d" % (e, num_chunks)

        for b in xrange(num_batches_chunk):
            iter_result = iter_test(b)
            network_outputs = tuple(iter_result[:len(output_layers)])
            network_outputs_dict = {output_layers.keys()[i]: network_outputs[i] for i in xrange(len(output_layers))}
            kaggle_systoles, kaggle_diastoles = config().postprocess(network_outputs_dict)
            kaggle_systoles, kaggle_diastoles = kaggle_systoles.astype('float64'), kaggle_diastoles.astype('float64')
            for idx, patient_id in enumerate(patient_ids[b*config().batch_size:(b+1)*config().batch_size]):
                if patient_id != 0:
                    index = patient_id-1
                    patient_data = predictions[index]
                    assert patient_id==patient_data["patient"]

                    kaggle_systole = kaggle_systoles[idx:idx+1,:]
                    kaggle_diastole = kaggle_diastoles[idx:idx+1,:]
                    assert np.isfinite(kaggle_systole).all() and np.isfinite(kaggle_systole).all()
                    kaggle_systole = classification_correction[b*config().batch_size + idx](kaggle_systole)
                    kaggle_diastole = classification_correction[b*config().batch_size + idx](kaggle_diastole)
                    assert np.isfinite(kaggle_systole).all() and np.isfinite(kaggle_systole).all()
                    patient_data["systole"] =  np.concatenate((patient_data["systole"], kaggle_systole ),axis=0)
                    patient_data["diastole"] = np.concatenate((patient_data["diastole"], kaggle_diastole ),axis=0)

        now = time.time()
        time_since_start = now - start_time
        time_since_prev = now - prev_time
        prev_time = now
        est_time_left = time_since_start * (float(num_chunks - (e + 1)) / float(e + 1 - chunks_train_idcs[0]))
        eta = datetime.now() + timedelta(seconds=est_time_left)
        eta_str = eta.strftime("%c")
        print "  %s since start (%.2f s)" % (utils.hms(time_since_start), time_since_prev)
        print "  estimated %s to go (ETA: %s)" % (utils.hms(est_time_left), eta_str)
        print

    already_printed = False
    for prediction in predictions:
        if prediction["systole"].size>0 and prediction["diastole"].size>0:
            average_method =  getattr(config(), 'tta_average_method', partial(np.mean, axis=0))
            prediction["systole_average"] = average_method(prediction["systole"])
            prediction["diastole_average"] = average_method(prediction["diastole"])
            try:
                test_if_valid_distribution(prediction["systole_average"])
                test_if_valid_distribution(prediction["diastole_average"])
            except:
                if not already_printed:
                    print "WARNING: These distributions are not distributions"
                    already_printed = True
                prediction["systole_average"] = make_monotone_distribution(prediction["systole_average"])
                prediction["diastole_average"] = make_monotone_distribution(prediction["diastole_average"])
                test_if_valid_distribution(prediction["systole_average"])
                test_if_valid_distribution(prediction["diastole_average"])


    print "Calculating training and validation set scores for reference"

    validation_dict = {}
    for patient_ids, set_name in [(validation_patients_indices, "validation"),
                                      (train_patients_indices,  "train")]:
        errors = []
        for patient in patient_ids:
            prediction = predictions[patient-1]
            if "systole_average" in prediction:
                assert patient == regular_labels[patient-1, 0]
                error = CRSP(prediction["systole_average"], regular_labels[patient-1, 1])
                errors.append(error)
                error = CRSP(prediction["diastole_average"], regular_labels[patient-1, 2])
                errors.append(error)
        if len(errors)>0:
            errors = np.array(errors)
            estimated_CRSP = np.mean(errors)
            print "  %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP)
            validation_dict[set_name] = estimated_CRSP
        else:
            print "  %s kaggle loss: not calculated" % (string.rjust(set_name, 12))


    print "dumping prediction file to %s" % prediction_path
    with open(prediction_path, 'w') as f:
        pickle.dump({
                        'metadata_path': metadata_path,
                        'prediction_path': prediction_path,
                        'submission_path': submission_path,
                        'configuration_file': config().__name__,
                        'git_revision_hash': utils.get_git_revision_hash(),
                        'experiment_id': expid,
                        'time_since_start': time_since_start,
                        'param_values': lasagne.layers.get_all_param_values(top_layer),
                        'predictions': predictions,
                        'validation_errors': validation_dict,
                    }, f, pickle.HIGHEST_PROTOCOL)
    print "prediction file dumped"

    print "dumping submission file to %s" % submission_path
    with open(submission_path, 'w') as csvfile:
        csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        csvwriter.writerow(['Id'] + ['P%d'%i for i in xrange(600)])
        for prediction in predictions:
            # the submission only has patients 501 to 700
            if prediction["patient"] in data_loader.test_patients_indices:
                if "diastole_average" not in prediction or "systole_average" not in prediction:
                    raise Exception("Not all test-set patients were predicted")
                csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["diastole_average"].flatten()])
                csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["systole_average"].flatten()])
    print "submission file dumped"

    return
예제 #14
0
def generate_final_predictions(
        final_predictions,
        prediction_tag,
        expert_predictions_matrix,
        masked_expert_predictions_matrix,
        test_expert_predictions_matrix,
        test_masked_expert_predictions_matrix,
        optimal_params,
        average_distribution,
        valid_labels,
        expert_pkl_files,
        expert_weight,
        disagreement_cutoff):
    cache_dict = dict()

    for final_prediction in final_predictions:
        patient_id = final_prediction['patient']
        if patient_id in train_patients_indices:
            continue

        print "   final prediction of patient %d" % patient_id
        reoptimized = False
        # get me the data for this patient
        # (NUM_EXPERTS, 600)
        if patient_id in validation_patients_indices:
            idx = validation_patients_indices.index(patient_id)
            prediction_matrix = expert_predictions_matrix[:, idx, :]
            mask_matrix = masked_expert_predictions_matrix[:, idx]

        elif patient_id in test_patients_indices:
            idx = test_patients_indices.index(patient_id)
            prediction_matrix = test_expert_predictions_matrix[:, idx, :]
            mask_matrix = test_masked_expert_predictions_matrix[:, idx]

        else:
            raise "This patient is neither train, validation or test?"


        # Is there an expert in the set, which did not predict this patient?
        # if so, re-optimize our weights on the validation set!
        # so, if a model is unavailble, but should be available, re-optimize!
        if np.logical_and(np.logical_not(mask_matrix),  (expert_weight!=0.0)).any():
            if hash_mask(mask_matrix) in cache_dict:
                (optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
            else:
                expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
                    expert_mask=mask_matrix,
                    prediction_matrix=expert_predictions_matrix,
                    mask_matrix=masked_expert_predictions_matrix,
                    labels=valid_labels,
                    average_distribution=average_distribution,
                    )
                cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
            reoptimized = True
        else:
            optimal_params_for_this_patient = optimal_params



        while True: # do-while: break condition at the end
            last_mask_matrix = mask_matrix

            final_prediction[prediction_tag] = optimize_expert_weights(
                            expert_predictions=prediction_matrix[:,None,:],
                            mask_matrix=mask_matrix[:,None],
                            average_distribution=average_distribution,
                            do_optimization=False,
                            optimal_params=optimal_params_for_this_patient,
                            )

            ## find the disagreement between models
            # and remove the ones who disagree too much with the average
            disagreement = find_disagreement(
                expert_predictions=prediction_matrix,
                mask_matrix=mask_matrix,
                ground_truth=final_prediction[prediction_tag]
            )
            mask_matrix = (disagreement<disagreement_cutoff)

            ## RETRAIN THESE ENSEMBLE WEIGHTS ON THE VALIDATION SET IF NEEDED!
            if ((last_mask_matrix!=mask_matrix).any()
               and np.sum(mask_matrix)!=0):

                if hash_mask(mask_matrix) in cache_dict:
                    (optimal_params_for_this_patient, expert_weight) = cache_dict[hash_mask(mask_matrix)]
                else:
                    expert_weight, _, optimal_params_for_this_patient = get_optimal_ensemble_weights_for_these_experts(
                        expert_mask=mask_matrix,
                        prediction_matrix=expert_predictions_matrix,
                        mask_matrix=masked_expert_predictions_matrix,
                        labels=valid_labels,
                        average_distribution=average_distribution,
                        )
                    cache_dict[hash_mask(mask_matrix)] = (optimal_params_for_this_patient, expert_weight)
                reoptimized = True
                continue
            else:
                break
        try:
            test_if_valid_distribution(final_prediction[prediction_tag])
        except:
            final_prediction[prediction_tag] = make_monotone_distribution(final_prediction[prediction_tag])
            test_if_valid_distribution(final_prediction[prediction_tag])

        if reoptimized:
            print "    Weight:  Name:"
            for expert_name, weight in zip(expert_pkl_files, expert_weight):
                if weight>0.:
                    print string.rjust("%.3f%%" % (100*weight), 12),
                    print expert_name.split('/')[-1]
예제 #15
0
def merge_all_prediction_files(prediction_file_location = INTERMEDIATE_PREDICTIONS_PATH,
                               redo_tta = True):

    submission_path = SUBMISSION_PATH + "final_submission-%s.csv" % time.time()

    # calculate the average distribution
    regular_labels = _load_file(_TRAIN_LABELS_PATH)

    average_systole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,1]]), axis=0))
    average_diastole = make_monotone_distribution(np.mean(np.array([utils.cumulative_one_hot(v) for v in regular_labels[:,2]]), axis=0))

    ss_expert_pkl_files = sorted([]
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout_seqshift_96.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_big_leaky_after_seqshift.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_big.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_zoom_mask_leaky_after.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi10_maxout.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky_after.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_maxout.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.ch2_zoom_leaky_after_nomask.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom_mask_leaky.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.gauss_roi_zoom.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc64small_360_gauss_longer.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_128mm.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_96mm.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc80_framemax.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_128mm_96.pkl")
        +glob.glob(prediction_file_location+"j6_4ch.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll.pkl")
        +glob.glob(prediction_file_location+"j6_4ch_32mm_specialist.pkl")
        +glob.glob(prediction_file_location+"j6_4ch_128mm_specialist.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc64_leaky_convroll.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc80small_360_gauss_longer_augzoombright.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc80_leaky_convroll_augzoombright.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_128mm_zoom.pkl")
        +glob.glob(prediction_file_location+"j6_2ch_128mm_skew.pkl")
        +glob.glob(prediction_file_location+"je_ss_jonisc64small_360.pkl")
    )


    fp_expert_pkl_files = sorted([]
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout_seqshift_96.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_big_leaky_after_seqshift.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_big.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_zoom_mask_leaky_after.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi10_maxout.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky_after.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom_mask_leaky.pkl")
        +glob.glob(prediction_file_location+"ira_configurations.meta_gauss_roi_zoom.pkl")
        +glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered.pkl")
        +glob.glob(prediction_file_location+"je_os_fixedaggr_rellocframe.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_filtered.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_framemax_reg.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_jsc80leakyconv_augzoombright_short.pkl")
        +glob.glob(prediction_file_location+"je_os_fixedaggr_relloc_filtered_discs.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale64small_filtered_longer.pkl")
        +glob.glob(prediction_file_location+"je_meta_fixedaggr_joniscale80small_augzoombright_betterdist.pkl")
        +glob.glob(prediction_file_location+"je_os_segmentandintegrate_smartsigma_dropout.pkl")
    )

    everything = sorted([]
        +glob.glob(prediction_file_location+"*.pkl")
    )
    expert_pkl_files = ss_expert_pkl_files + fp_expert_pkl_files

    print "found %d/44 files" % len(expert_pkl_files)
    """
    # filter expert_pkl_files
    for file in expert_pkl_files[:]:
        try:
            with open(file, 'r') as f:
                print "testing file",file.split('/')[-1]
                data = pickle.load(f)
                if 'predictions' not in data.keys():
                    expert_pkl_files.remove(file)
                    print "                -> removed"
        except:
            print sys.exc_info()[0]
            expert_pkl_files.remove(file)
            print "                -> removed"
    """
    NUM_EXPERTS = len(expert_pkl_files)
    NUM_VALIDATIONS = len(validation_patients_indices)
    NUM_TESTS = len(test_patients_indices)

    systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')
    diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_VALIDATIONS, 600), dtype='float32')

    systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')
    diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_VALIDATIONS), dtype='bool')

    test_systole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')
    test_diastole_expert_predictions_matrix = np.zeros((NUM_EXPERTS, NUM_TESTS, 600), dtype='float32')

    test_systole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')
    test_diastole_masked_expert_predictions_matrix = np.ones((NUM_EXPERTS, NUM_TESTS), dtype='bool')

    for i,file in enumerate(expert_pkl_files):
        with open(file, 'r') as f:
            print
            print "loading file",file.split('/')[-1]
            predictions = pickle.load(f)['predictions']

        if redo_tta:
            best_average_method = normalav
            best_average_crps = utils.maxfloat
            for average_method in [geomav,
                                   normalav,
                                   prodav,
                                   weighted_geom_method
                                   ]:
                calculate_tta_average(predictions, average_method, average_systole, average_diastole)

                crps = get_validate_crps(predictions, regular_labels)
                print string.rjust(average_method.__name__,25),"->",crps
                if crps<best_average_crps:
                    best_average_method = average_method
                    best_average_crps = crps

            print " I choose you,", best_average_method.__name__
            calculate_tta_average(predictions, best_average_method, average_systole, average_diastole)
            print " validation loss:", get_validate_crps(predictions, regular_labels)

        for j,patient in enumerate(validation_patients_indices):
            prediction = predictions[patient-1]
            # average distributions get zero weight later on
            if "systole_average" in prediction and prediction["systole_average"] is not None:
                systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
            else:
                systole_masked_expert_predictions_matrix[i,j] = False

            if "diastole_average" in prediction and prediction["diastole_average"] is not None:
                diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
            else:
                diastole_masked_expert_predictions_matrix[i,j] = False

        for j,patient in enumerate(test_patients_indices):
            prediction = predictions[patient-1]
            # average distributions get zero weight later on
            if "systole_average" in prediction and prediction["systole_average"] is not None:
                test_systole_expert_predictions_matrix[i,j,:] = prediction["systole_average"]
            else:
                test_systole_masked_expert_predictions_matrix[i,j] = False

            if "diastole_average" in prediction and prediction["diastole_average"] is not None:
                test_diastole_expert_predictions_matrix[i,j,:] = prediction["diastole_average"]
            else:
                test_diastole_masked_expert_predictions_matrix[i,j] = False

        del predictions  # can be LOADS of data



    cv = [id-1 for id in validation_patients_indices]
    systole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,1].flatten()])
    systole_expert_weight, first_pass_sys_loss, systole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
        expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
        prediction_matrix=systole_expert_predictions_matrix,
        mask_matrix=systole_masked_expert_predictions_matrix,
        labels=systole_valid_labels,
        average_distribution=average_systole,
        )

    cv = [id-1 for id in validation_patients_indices]
    diastole_valid_labels = np.array([utils.cumulative_one_hot(v) for v in regular_labels[cv,2].flatten()])
    diastole_expert_weight, first_pass_dia_loss, diastole_optimal_params = get_optimal_ensemble_weights_for_these_experts(
        expert_mask=np.ones((NUM_EXPERTS,), dtype='bool'),
        prediction_matrix=diastole_expert_predictions_matrix,
        mask_matrix=diastole_masked_expert_predictions_matrix,
        labels=diastole_valid_labels,
        average_distribution=average_diastole,
        )


    # print the final weight of every expert
    print "  Systole:  Diastole: Name:"
    for expert_name, systole_weight, diastole_weight in zip(expert_pkl_files, systole_expert_weight, diastole_expert_weight):
        print string.rjust("%.3f%%" % (100*systole_weight), 10),
        print string.rjust("%.3f%%" % (100*diastole_weight), 10),
        print expert_name.split('/')[-1]

    print
    print "estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2)
    print

    print
    print "Average the experts according to these weights to find the final distribution"
    final_predictions = [{
                            "patient": i+1,
                            "final_systole": None,
                            "final_diastole": None
                        } for i in xrange(NUM_PATIENTS)]


    generate_final_predictions(
        final_predictions=final_predictions,
        prediction_tag="final_systole",
        expert_predictions_matrix=systole_expert_predictions_matrix,
        masked_expert_predictions_matrix=systole_masked_expert_predictions_matrix,
        test_expert_predictions_matrix=test_systole_expert_predictions_matrix,
        test_masked_expert_predictions_matrix=test_systole_masked_expert_predictions_matrix,
        optimal_params=systole_optimal_params,
        average_distribution=average_systole,
        valid_labels=systole_valid_labels,
        expert_pkl_files=expert_pkl_files,
        expert_weight=systole_expert_weight,
        disagreement_cutoff=0.01 # 0.01
    )

    generate_final_predictions(
        final_predictions=final_predictions,
        prediction_tag="final_diastole",
        expert_predictions_matrix=diastole_expert_predictions_matrix,
        masked_expert_predictions_matrix=diastole_masked_expert_predictions_matrix,
        test_expert_predictions_matrix=test_diastole_expert_predictions_matrix,
        test_masked_expert_predictions_matrix=test_diastole_masked_expert_predictions_matrix,
        optimal_params=diastole_optimal_params,
        average_distribution=average_diastole,
        valid_labels=diastole_valid_labels,
        expert_pkl_files=expert_pkl_files,
        expert_weight=diastole_expert_weight,
        disagreement_cutoff=0.015  # diastole has about 50% more error
    )

    print
    print "Calculating training and validation set scores for reference"


    validation_dict = {}
    for patient_ids, set_name in [(validation_patients_indices, "validation")]:
        errors = []
        for patient in patient_ids:
            prediction = final_predictions[patient-1]

            if "final_systole" in prediction:
                assert patient == regular_labels[patient-1, 0]
                error1 = utils.CRSP(prediction["final_systole"], regular_labels[patient-1, 1])
                errors.append(error1)
                prediction["systole_crps_error"] = error1
                error2 = utils.CRSP(prediction["final_diastole"], regular_labels[patient-1, 2])
                errors.append(error2)
                prediction["diastole_crps_error"] = error1
                prediction["average_crps_error"] = 0.5*error1 + 0.5*error2

        if len(errors)>0:
            errors = np.array(errors)
            estimated_CRSP = np.mean(errors)
            print "  %s kaggle loss: %f" % (string.rjust(set_name, 12), estimated_CRSP)
            validation_dict[set_name] = estimated_CRSP
        else:
            print "  %s kaggle loss: not calculated" % (string.rjust(set_name, 12))
    print "WARNING: both of the previous are overfitted!"

    print
    print "estimated leaderboard loss: %f" % ((first_pass_sys_loss + first_pass_dia_loss)/2)
    print


    print "dumping submission file to %s" % submission_path
    with open(submission_path, 'w') as csvfile:
        csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        csvwriter.writerow(['Id'] + ['P%d'%i for i in xrange(600)])
        for prediction in final_predictions:
            # the submission only has patients 501 to 700
            if prediction["patient"] in test_patients_indices:
                if "final_diastole" not in prediction or "final_systole" not in prediction:
                    raise Exception("Not all test-set patients were predicted")
                csvwriter.writerow(["%d_Diastole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_diastole"].flatten()])
                csvwriter.writerow(["%d_Systole" % prediction["patient"]] + ["%.18f" % p for p in prediction["final_systole"].flatten()])
    print "submission file dumped"