def train_model(cfg, data, callbacks, verbose=2):
'''
Train a and evaluate model on given data.
:param cfg: Project config (from config.yml)
:param data: dict of partitioned dataset
:param callbacks: list of callbacks for Keras model
:param verbose: Verbosity mode to pass to model.fit()
:return: Trained model and associated performance metrics on the test set
'''
# Apply class imbalance strategy
num_neg, num_pos = np.bincount(data['Y_train'].astype(int))
class_weight = None
if cfg['TRAIN']['IMB_STRATEGY'] == 'class_weight':
class_weight = get_class_weights(num_pos, num_neg, cfg['TRAIN']['POS_WEIGHT'])
elif cfg['TRAIN']['IMB_STRATEGY'] != 'none':
data['X_train'], data['Y_train'] = minority_oversample(data['X_train'], data['Y_train'],
algorithm=cfg['TRAIN']['IMB_STRATEGY'])
thresholds = cfg['TRAIN']['THRESHOLDS'] # Load classification thresholds
# List metrics
metrics = ['accuracy', BinaryAccuracy(name='accuracy'), Precision(name='precision', thresholds=thresholds),
Recall(name='recall', thresholds=thresholds), F1Score(name='f1score', thresholds=thresholds),
AUC(name='auc')]
# Compute output bias
num_neg, num_pos = np.bincount(data['Y_train'].astype(int))
output_bias = np.log([num_pos / num_neg])
# Build the model graph.
if cfg['TRAIN']['MODEL_DEF'] == 'hifis_rnn_mlp':
model_def = hifis_rnn_mlp
else:
model_def = hifis_mlp
model = model_def(cfg['NN'][cfg['TRAIN']['MODEL_DEF'].upper()], (data['X_train'].shape[-1],), metrics,
data['METADATA'], output_bias=output_bias)
# Train the model.
history = model.fit(data['X_train'], data['Y_train'], batch_size=cfg['TRAIN']['BATCH_SIZE'],
epochs=cfg['TRAIN']['EPOCHS'], validation_data=(data['X_val'], data['Y_val']),
callbacks=callbacks, class_weight=class_weight, verbose=verbose)
# Run the model on the test set and print the resulting performance metrics.
test_results = model.evaluate(data['X_test'], data['Y_test'])
test_metrics = {}
test_summary_str = [['**Metric**', '**Value**']]
for metric, value in zip(model.metrics_names, test_results):
test_metrics[metric] = value
print(metric, ' = ', value)
test_summary_str.append([metric, str(value)])
return model, test_metrics
def train_model(cfg, data, callbacks, verbose=1):
'''
Train a and evaluate model on given data.
:param cfg: Project config (from config.yml)
:param data: dict of partitioned dataset
:param callbacks: list of callbacks for Keras model
:param verbose: Verbosity mode to pass to model.fit_generator()
:return: Trained model and associated performance metrics on the test set
'''
# If set in config file, oversample the minority class
if cfg['TRAIN']['IMB_STRATEGY'] == 'random_oversample':
data['TRAIN'] = random_minority_oversample(data['TRAIN'])
# Create ImageDataGenerators
train_img_gen = ImageDataGenerator(rotation_range=10, preprocessing_function=remove_text,
samplewise_std_normalization=True, samplewise_center=True)
val_img_gen = ImageDataGenerator(preprocessing_function=remove_text,
samplewise_std_normalization=True, samplewise_center=True)
test_img_gen = ImageDataGenerator(preprocessing_function=remove_text,
samplewise_std_normalization=True, samplewise_center=True)
# Create DataFrameIterators
img_shape = tuple(cfg['DATA']['IMG_DIM'])
y_col = 'label_str'
class_mode = 'categorical'
train_generator = train_img_gen.flow_from_dataframe(dataframe=data['TRAIN'], directory=cfg['PATHS']['RAW_DATA'],
x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'],
class_mode=class_mode, validate_filenames=False)
val_generator = val_img_gen.flow_from_dataframe(dataframe=data['VAL'], directory=cfg['PATHS']['RAW_DATA'],
x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'],
class_mode=class_mode, validate_filenames=False)
test_generator = test_img_gen.flow_from_dataframe(dataframe=data['TEST'], directory=cfg['PATHS']['RAW_DATA'],
x_col="filename", y_col=y_col, target_size=img_shape, batch_size=cfg['TRAIN']['BATCH_SIZE'],
class_mode=class_mode, validate_filenames=False, shuffle=False)
# Save model's ordering of class indices
dill.dump(test_generator.class_indices, open(cfg['PATHS']['OUTPUT_CLASS_INDICES'], 'wb'))
# Apply class imbalance strategy. We have many more X-rays negative for COVID-19 than positive.
histogram = np.bincount(np.array(train_generator.labels).astype(int)) # Get class distribution
class_weight = None
if cfg['TRAIN']['IMB_STRATEGY'] == 'class_weight':
class_multiplier = cfg['TRAIN']['CLASS_MULTIPLIER']
class_multiplier = [class_multiplier[cfg['DATA']['CLASSES'].index(c)] for c in test_generator.class_indices]
class_weight = get_class_weights(histogram, class_multiplier)
# Define metrics.
covid_class_idx = test_generator.class_indices['COVID-19'] # Get index of COVID-19 class
thresholds = 1.0 / len(cfg['DATA']['CLASSES']) # Binary classification threshold for a class
metrics = ['accuracy', CategoricalAccuracy(name='accuracy'),
Precision(name='precision', thresholds=thresholds, class_id=covid_class_idx),
Recall(name='recall', thresholds=thresholds, class_id=covid_class_idx),
AUC(name='auc'),
F1Score(name='f1score', thresholds=thresholds, class_id=covid_class_idx)]
# Define the model.
print('Training distribution: ', ['Class ' + list(test_generator.class_indices.keys())[i] + ': ' + str(histogram[i]) + '. '
for i in range(len(histogram))])
input_shape = cfg['DATA']['IMG_DIM'] + [3]
num_gpus = cfg['TRAIN']['NUM_GPUS']
if cfg['TRAIN']['MODEL_DEF'] == 'dcnn_resnet':
model_def = dcnn_resnet
elif cfg['TRAIN']['MODEL_DEF'] == 'resnet50v2':
model_def = resnet50v2
else:
model_def = resnet101v2
if cfg['TRAIN']['CLASS_MODE'] == 'binary':
histogram = np.bincount(data['TRAIN']['label'].astype(int))
output_bias = np.log([histogram[i] / (np.sum(histogram) - histogram[i]) for i in range(histogram.shape[0])])
model = model_def(cfg['NN']['DCNN_BINARY'], input_shape, metrics, 2, output_bias=output_bias, gpus=num_gpus)
else:
n_classes = len(cfg['DATA']['CLASSES'])
histogram = np.bincount(data['TRAIN']['label'].astype(int))
output_bias = np.log([histogram[i] / (np.sum(histogram) - histogram[i]) for i in range(histogram.shape[0])])
model = model_def(cfg['NN']['DCNN_MULTICLASS'], input_shape, metrics, n_classes, output_bias=output_bias,
gpus=num_gpus)
# Train the model.
steps_per_epoch = ceil(train_generator.n / train_generator.batch_size)
val_steps = ceil(val_generator.n / val_generator.batch_size)
history = model.fit_generator(train_generator, steps_per_epoch=steps_per_epoch, epochs=cfg['TRAIN']['EPOCHS'],
validation_data=val_generator, validation_steps=val_steps, callbacks=callbacks,
verbose=verbose, class_weight=class_weight)
# Run the model on the test set and print the resulting performance metrics.
test_results = model.evaluate_generator(test_generator, verbose=1)
test_metrics = {}
test_summary_str = [['**Metric**', '**Value**']]
for metric, value in zip(model.metrics_names, test_results):
test_metrics[metric] = value
print(metric, ' = ', value)
test_summary_str.append([metric, str(value)])
return model, test_metrics, test_generator