auc_std = group["auc_std"]
f1_std = group["f1_std"]
combined_scores = np.sqrt(np.array(auc * f1))
score = np.mean(combined_scores)
score_std = np.std(combined_scores)
config_row = group[hyperparameter_columns].iloc[0]
print(config_row)
print("--> AUC: %0.4f +/- %0.4f" % (
auc.mean(),
auc.std()))
print("--> F1: %0.4f +/- %0.4f" % (
f1.mean(),
f1.std()))
print("--> combined: %0.4f +/- %0.4f" % (
score,
score_std))
if score > max_score:
best_auc = auc.mean()
best_f1 = f1.mean()
max_score = score
best_config = config_row
print("Best score: %0.4f, AUC=%0.4f, f1=%0.4f, hyperparameters:\n%s" % (
max_score, best_auc, best_f1, best_config))
print("=== Score Distributions For Hyperparameters ===")
hyperparameter_performance(results)
print("\n\n=== Hyperparameter Value Comparisons ===")
results = hyperparameter_score_difference_hypothesis_tests(results)
print(generate_latex_table(results))
if __name__ == "__main__":
args = parser.parse_args()
configs = generate_all_model_configs(
activations=args.activation,
init_methods=args.init,
max_ic50_values=args.max_ic50,
dropout_values=args.dropout,
minibatch_sizes=args.minibatch_size,
embedding_sizes=args.embedding_size,
n_pretrain_epochs_values=args.pretrain_epochs,
n_training_epochs_values=args.training_epochs,
hidden_layer_sizes=args.hidden_layer_size,
learning_rates=args.learning_rate,
optimizers=args.optimizer)
print("Total # configurations = %d" % len(configs))
training_datasets, _ = load_data(
args.binding_data_csv_path,
max_ic50=args.max_ic50,
peptide_length=9,
binary_encoding=False)
combined_df = evaluate_model_configs(
configs=configs,
results_filename=args.output,
train_fn=lambda config: evaluate_model_config_by_cross_validation(
config,
training_datasets,
min_samples_per_allele=args.min_samples_per_allele,
cv_folds=args.cv_folds))
hyperparameter_performance(combined_df)