def test_deepconvlstm_starts_with_batchnorm(self):
""" DeepConvLSTM models should always start with a batch normalization layer. """
model = modelgen.generate_DeepConvLSTM_model((None, 20, 3), 2,
[32, 32], [32, 32])
assert_equal(
str(type(model.layers[0])),
"<class 'keras.layers.normalization.BatchNormalization'>",
'Wrong layer type.')
def test_deepconvlstm_enough_batchnorm(self):
"""LSTM model should contain as many batch norm layers as it has activations layers"""
model = modelgen.generate_DeepConvLSTM_model(
(None, 20, 3), 2, [32, 32, 32], [32, 32, 32])
batch_norm_layers = len(
[l for l in model.layers if 'BatchNormalization' in str(l)])
activation_layers = len(
[l for l in model.layers if 'Activation' in str(l)])
assert_equal(batch_norm_layers, activation_layers)
def test_deepconvlstm_batchnorm_dim(self):
"""The output shape of the batchnorm should be (None, nr_filters, nr_timesteps, nr_channels)"""
model = modelgen.generate_DeepConvLSTM_model((None, 20, 3), 2,
[32, 32], [32, 32])
batchnormlay = model.layers[3]
assert_equal(batchnormlay.output_shape, (None, 32, 20, 3))
def main(argv):
indir = argv[0]
mode = argv[1] # binary or multiclass
outdir = argv[2]
if mode == 'multiclass':
sleep_states = [
'Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM', 'Nonwear', 'Wake_ext'
]
else:
sleep_states = ['Wake', 'Sleep', 'Nonwear', 'Wake_ext']
collate_sleep = ['NREM 1', 'NREM 2', 'NREM 3', 'REM']
valid_sleep_states = [
state for state in sleep_states if state != 'Wake_ext'
]
num_classes = len(valid_sleep_states)
if not os.path.exists(outdir):
os.makedirs(outdir)
resultdir = os.path.join(outdir, mode, 'models')
if not os.path.exists(resultdir):
os.makedirs(resultdir)
# Read data from disk
data = pd.read_csv(os.path.join(indir, 'labels.txt'), sep='\t')
files = []
labels = []
users = []
for idx, row in data.iterrows():
files.append(os.path.join(indir, row['filename']) + '.npy')
labels.append(row['labels'])
users.append(row['user'])
if mode == 'binary':
labels = ['Sleep' if lbl in collate_sleep else lbl for lbl in labels]
early_stopping = EarlyStopping(monitor='val_macro_f1',
mode='max',
verbose=1,
patience=2)
seqlen, n_channels = np.load(files[0]).shape
batch_size = 32
# Use nested cross-validation based on users
# Outer CV
unique_users = list(set(users))
random.shuffle(unique_users)
out_cv_splits = 5
in_cv_splits = 5
out_fold_nusers = len(unique_users) // out_cv_splits
out_n_epochs = 10
in_n_epochs = 1
predictions = []
wake_idx = sleep_states.index('Wake')
wake_ext_idx = sleep_states.index('Wake_ext')
for out_fold in range(out_cv_splits):
print('Evaluating fold %d' % (out_fold + 1))
test_users = unique_users[out_fold * out_fold_nusers:(out_fold + 1) *
out_fold_nusers]
trainval_users = [
user for user in unique_users if user not in test_users
]
train_users = trainval_users[:int(0.8 * len(trainval_users))]
val_users = trainval_users[len(train_users):]
out_train_fnames, out_train_labels, out_train_users = get_partition(files, labels, users, train_users,\
sleep_states, is_train=True)
out_val_fnames, out_val_labels, out_val_users = get_partition(
files, labels, users, val_users, sleep_states)
out_test_fnames, out_test_labels, out_test_users = get_partition(
files, labels, users, test_users, sleep_states)
out_train_gen = DataGenerator(out_train_fnames, out_train_labels, valid_sleep_states, partition='out_train',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, shuffle=True, augment=True, aug_factor=0.75, balance=True)
print(
'Fold {}: Computing mean and standard deviation'.format(out_fold +
1))
mean, std = out_train_gen.fit()
#mean = None; std = None
out_val_gen = DataGenerator(out_val_fnames, out_val_labels, valid_sleep_states, partition='out_val',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, mean=mean, std=std)
out_test_gen = DataGenerator(out_test_fnames, out_test_labels, valid_sleep_states, partition='out_test',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, mean=mean, std=std)
# Get class weights
out_class_wts = class_weight.compute_class_weight(
'balanced', np.unique(out_train_labels), out_train_labels)
# Inner CV
val_acc = []
models = []
in_fold_nusers = len(trainval_users) // in_cv_splits
for in_fold in range(in_cv_splits):
in_val_users = trainval_users[in_fold *
in_fold_nusers:(in_fold + 1) *
in_fold_nusers]
in_train_users = [
user for user in trainval_users if user not in in_val_users
]
in_train_fnames, in_train_labels, in_train_users = get_partition(files, labels, users, in_train_users,\
sleep_states, is_train=True)
in_val_fnames, in_val_labels, in_val_users = get_partition(
files, labels, users, in_val_users, sleep_states)
in_train_gen = DataGenerator(in_train_fnames, in_train_labels, valid_sleep_states, partition='in_train',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, shuffle=True, augment=True, aug_factor=0.75, balance=True,\
mean=mean, std=std)
in_val_gen = DataGenerator(in_val_fnames, in_val_labels, valid_sleep_states, partition='in_val',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, mean=mean, std=std)
# Generate candidate architectures
model = modelgen.generate_models((None, seqlen, n_channels), \
number_of_classes=num_classes, \
number_of_models=1, metrics=[macro_f1])#, model_type='CNN')
# Compare generated architectures on a subset of data for few epochs
outfile = os.path.join(resultdir, 'model_comparison.json')
hist, acc, loss = find_architecture.train_models_on_samples(in_train_gen, in_val_gen,
model, nr_epochs=in_n_epochs, n_steps=1000, class_weight=out_class_wts, \
verbose=True, outputfile=outfile, metric='macro_f1')
val_acc.append(acc[0])
models.append(model[0])
# Choose best model and evaluate values on validation data
print('Evaluating on best model for fold %d' % out_fold)
best_model_index = np.argmax(val_acc)
best_model, best_params, best_model_type = models[best_model_index]
print('Best model type and parameters:')
print(best_model_type)
print(best_params)
if best_model_type == 'CNN':
best_model = modelgen.generate_CNN_model((None, seqlen, n_channels), num_classes, filters=best_params['filters'], \
fc_hidden_nodes=best_params['fc_hidden_nodes'], \
learning_rate=best_params['learning_rate'], \
regularization_rate=best_params['regularization_rate'], \
metrics=[macro_f1])
else:
best_model = modelgen.generate_DeepConvLSTM_model((None, seqlen, n_channels), num_classes,\
filters=best_params['filters'], \
lstm_dims=best_params['lstm_dims'], \
learning_rate=best_params['learning_rate'], \
regularization_rate=best_params['regularization_rate'], \
metrics=[macro_f1])
# Use early stopping and model checkpoints to handle overfitting and save best model
model_checkpt = ModelCheckpoint(os.path.join(resultdir,'best_model_fold'+str(out_fold+1)+'.h5'), monitor='val_macro_f1',\
mode='max', save_best_only=True)
history = F1scoreHistory()
hist = best_model.fit_generator(out_train_gen, epochs=out_n_epochs, \
validation_data=out_val_gen, class_weight=out_class_wts,\
callbacks=[early_stopping, model_checkpt])
# Plot training history
# plt.Figure()
# plt.plot(history.mean_f1score['train'])
# #plt.plot(history.mean_f1score['val'])
# plt.title('Model F1-score')
# plt.ylabel('F1-score')
# plt.xlabel('Batch')
# #plt.legend(['Train', 'Test'], loc='upper left')
# plt.savefig(os.path.join(resultdir,'Fold'+str(fold)+'_performance_curve.jpg'))
# plt.clf()
#
## # Save model
## best_model.save(os.path.join(resultdir,'best_model_fold'+str(fold)+'.h5'))
# Predict probability on validation data
probs = best_model.predict_generator(out_test_gen)
y_pred = probs.argmax(axis=1)
y_true = out_test_labels
predictions.append((out_test_users, y_true, y_pred))
# Save user report
if mode == 'binary':
save_user_report(
predictions, valid_sleep_states,
os.path.join(
resultdir, 'fold' + str(out_fold + 1) +
'_deeplearning_binary_results.csv'))
else:
save_user_report(
predictions, valid_sleep_states,
os.path.join(
resultdir, 'fold' + str(out_fold + 1) +
'_deeplearning_multiclass_results.csv'))
get_classification_report(predictions, valid_sleep_states)
# Save user report
if mode == 'binary':
save_user_report(
predictions, valid_sleep_states,
os.path.join(resultdir, 'deeplearning_binary_results.csv'))
else:
save_user_report(
predictions, valid_sleep_states,
os.path.join(resultdir, 'deeplearning_multiclass_results.csv'))