data['TEST'] = pd.read_csv(cfg['PATHS']['TEST_SET'])
# Custom Keras callback that logs all training and validation metrics after each epoch to the current Azure run
class LogRunMetrics(Callback):
def on_epoch_end(self, epoch, log):
for metric_name in log:
if 'val' in metric_name:
run.log('validation_' + metric_name.split('_')[-1], log[metric_name])
else:
run.log('training_' + metric_name, log[metric_name])
#run.log('validation_auc', log['val_auc'])
# Set model callbacks
callbacks = [EarlyStopping(monitor='val_loss', verbose=1, patience=cfg['TRAIN']['PATIENCE'], mode='min', restore_best_weights=True),
LogRunMetrics()]
# Train a model
start_time = datetime.datetime.now()
model, test_metrics, test_generator = train_model(cfg, data, callbacks)
print("TRAINING TIME = " + str((datetime.datetime.now() - start_time).total_seconds() / 60.0) + " min")
# Log test set performance metrics, ROC, confusion matrix in Azure run
test_predictions = model.predict_generator(test_generator, verbose=0)
test_labels = test_generator.labels
for metric_name in test_metrics:
run.log('test_' + metric_name, test_metrics[metric_name])
covid_idx = test_generator.class_indices['COVID-19']
roc_plt = plot_roc("Test set", test_generator.labels, test_predictions, class_id=covid_idx)
run.log_image("ROC", plot=roc_plt)
cm_plt = plot_confusion_matrix(test_generator.labels, test_predictions, class_id=covid_idx)
run.log_image("Confusion matrix", plot=cm_plt)
cfg, data, callbacks, base_log_dir=cfg['PATHS']['LOGS'])
print("TOTAL MULTI_TRAIN TIME = " +
str((datetime.datetime.now() - start_time).total_seconds() / 60.0) +
" min")
# Identify the logs from the best model
run.log("TensorBoard logs folder original name", best_logs_date)
shutil.copytree(cfg['PATHS']['LOGS'] + best_logs_date,
cfg['PATHS']['LOGS'] + "logs")
# Log test set performance metrics, ROC, confusion matrix in Azure run
test_predictions = model.predict(data['X_test'],
batch_size=cfg['TRAIN']['BATCH_SIZE'])
for metric_name in test_metrics:
run.log('best_test_' + metric_name, test_metrics[metric_name])
roc_plt = plot_roc("Test set", data['Y_test'], test_predictions)
run.log_image("ROC", plot=roc_plt)
cm_plt = plot_confusion_matrix(data['Y_test'], test_predictions)
run.log_image("Confusion matrix", plot=cm_plt)
# Log test set performance of all models and save to CSV
for i in range(len(test_metrics_dict['loss'])):
run.log_row("Test set metrics",
model=(i + 1),
f1score=test_metrics_dict['f1score'][i],
recall=test_metrics_dict['recall'][i],
precision=test_metrics_dict['precision'][i],
auc=test_metrics_dict['auc'][i],
loss=test_metrics_dict['loss'][i])
test_set_metrics_df = pd.DataFrame(test_metrics_dict)
test_set_metrics_df.to_csv(cfg['PATHS']['MULTI_TRAIN_TEST_METRICS'])