def test_integration(self):
"""Does most of the operations in the tutorial and uses many of mcfly's functionalities consecutively."""
X_train, X_val, y_train, y_val = self.generate_random_data_sets()
num_classes = y_train.shape[1]
metric = 'accuracy'
models = modelgen.generate_models(
X_train.shape,
number_of_classes=num_classes,
number_of_models=2,
metrics=[metric],
model_type='CNN') # Because CNNs are quick to train.
histories, val_accuracies, _ = find_architecture.train_models_on_samples(
X_train,
y_train,
X_val,
y_val,
models,
nr_epochs=5,
subset_size=150,
verbose=True,
outputfile=self.outputfile)
best_model_index = np.argmax(val_accuracies[metric])
best_model, _, _ = models[best_model_index]
_ = best_model.fit(X_train[:200, :, :],
y_train[:200, :],
epochs=2,
validation_data=(X_val, y_val))
best_model.save(self.modelfile)
model_reloaded = load_model(self.modelfile)
assert model_reloaded is not None, "Expected model" #TODO: check if it's a real model
assert len(histories) == 2, "Expected two models in histories"
assert os.path.exists(self.outputfile)
assert os.path.exists(self.modelfile)
def test_integration(self):
"""Does most of the operations in the tutorial and uses many of mcfly's functionalities consecutively."""
X_train, X_val, y_train, y_val = self.generate_random_data_sets()
num_classes = y_train.shape[1]
models = modelgen.generate_models(
X_train.shape,
number_of_classes=num_classes,
number_of_models=2,
model_type='CNN') # Because CNNs are quick to train.
histories, val_accuracies, val_losses = find_architecture.train_models_on_samples(
X_train,
y_train,
X_val,
y_val,
models,
nr_epochs=5,
subset_size=150,
verbose=True,
outputfile=self.outputfile)
best_model_index = np.argmax(val_accuracies)
best_model, best_params, best_model_types = models[best_model_index]
history = best_model.fit(X_train[:200, :, :],
y_train[:200, :],
epochs=2,
validation_data=(X_val, y_val))
best_model.save(self.modelfile)
model_reloaded = load_model(self.modelfile)
assert os.path.exists(self.outputfile)
assert os.path.exists(self.modelfile)
def test_generate_models_metrics(self):
""" "Test if correct number of models is generated and if metrics is correct. """
models = modelgen.generate_models((None, 20, 3), 2, 5)
model, hyperparams, modeltype = models[0]
metrics = [m.name for m in model.metrics]
assert metrics == ['accuracy'] or metrics == ['acc']
assert len(models) == 5
def create_dummy_model():
np.random.seed(123)
num_time_steps = 100
num_channels = 2
num_samples_train = 5
model, _parameters, _type = modelgen.generate_models(
(num_samples_train, num_time_steps, num_channels), 5, 1)[0]
return model
def test_generate_models_exception(self):
""" Test expected generate_models exception."""
x_shape = (None, 20, 10)
nr_classes = 2
n_models = 2
with pytest.raises(NameError, match="Unknown model name, 'wrong_entry'."):
_ = modelgen.generate_models(x_shape, nr_classes, n_models,
model_types=['CNN', "wrong_entry"])
def test_generate_models_pass_model_object(self):
""" Test if model class can be passed as model_types input."""
x_shape = (None, 20, 10)
nr_classes = 2
n_models = 4
models = modelgen.generate_models(x_shape, nr_classes, n_models,
model_types=['CNN', ResNet])
created_model_names = list({x[2] for x in models})
created_model_names.sort()
assert len(models) == 4, "Expected number of models to be 4"
assert created_model_names == ["CNN", "ResNet"], "Expected different model names."
for model in models:
assert isinstance(model[0], Model), "Expected keras model."
def test_generate_models_metrics(self):
""" Test if correct number of models is generated and if metrics is correct. """
x_shape = (None, 20, 10)
nr_classes = 2
X_train, y_train = generate_train_data(x_shape, nr_classes)
n_models = 5
models = modelgen.generate_models(x_shape, nr_classes, n_models)
for model in models:
model[0].fit(X_train, y_train, epochs = 1)
model, _, modeltype = models[0]
model_metrics = [m.name for m in model.metrics]
assert "accuracy" in model_metrics, "Not found accuracy for model {}. Found {}".format(
modeltype, model_metrics)
assert len(models) == n_models, "Expecting {} models, found {} models".format(
n_models, len(models))
def create_model(X, y_binary, lr, drp1, drp2):
models = modelgen.generate_models(X.shape,
y_binary.shape[1],
number_of_models=1,
model_type='CNN',
cnn_min_layers=2,
cnn_max_layers=2,
cnn_min_filters=4,
cnn_max_filters=4,
cnn_min_fc_nodes=6,
cnn_max_fc_nodes=6,
low_lr=lr,
high_lr=lr,
low_reg=1,
high_reg=1,
kernel_size=7,
max_drp_out1=drp1,
max_drp_out2=drp2,
min_drp_out1=drp1,
min_drp_out2=drp2)
# models = modelgen.generate_models(X.shape,
# y_binary.shape[1],
# number_of_models = 1,
# model_type = 'CNN',
# cnn_min_layers=4,
# cnn_max_layers=4,
# cnn_min_filters = 6,
# cnn_max_filters = 6,
# cnn_min_fc_nodes=12,
# cnn_max_fc_nodes=12,
# low_lr=2, high_lr=2,
# low_reg=1, high_reg=1,
# kernel_size = 7)
# i = 0
# for model, params, model_types in models:
# logging.info('model ' + str(i))
# i = i + 1
# logging.info(params)
# model.summary()
return models
y_val = np.load(os.path.join(preprocessed_path, 'y_val.npy'))
y_test = np.load(os.path.join(preprocessed_path, 'y_test.npy'))
with open(os.path.join(preprocessed_path, 'labels.json')) as f:
labels = json.load(f)
# ## Generate models
num_classes = y_train.shape[1]
models = modelgen.generate_models(X_train.shape,
number_of_classes=num_classes,
number_of_models = 15)
#what is the fraction of classes in the validation set?
pd.Series(y_val.mean(axis=0), index=labels)
if not os.path.exists(result_path):
os.makedirs(result_path)
histories, val_accuracies, val_losses = find_architecture.train_models_on_samples(X_train, y_train,
def main(argv):
infile = argv[0]
outdir = argv[1]
sleep_states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM']
if not os.path.exists(outdir):
os.makedirs(outdir)
resultdir = os.path.join(outdir, 'models')
if not os.path.exists(resultdir):
os.makedirs(resultdir)
all_data = np.load(infile)
X = all_data['data']
y = all_data['labels']
users = all_data['user']
dataset = all_data['dataset']
X = X[dataset == 'UPenn']
y = y[dataset == 'UPenn']
num_classes = y.shape[1]
# Shuffle data
shuf_idx = np.arange(X.shape[0])
np.random.shuffle(shuf_idx)
X = X[shuf_idx]
y = y[shuf_idx]
users = [users[i] for i in shuf_idx]
# Get small subset
#idx = np.random.randint(X.shape[0],size=10000)
#X = X[idx]; y = y[idx]; users = [users[i] for i in idx]
y_lbl = y.argmax(axis=1)
y_lbl = [sleep_states[i] for i in y_lbl]
# Use nested cross-validation based on users
# Outer CV
outer_cv_splits = 5
inner_cv_splits = 3
group_kfold = GroupKFold(n_splits=outer_cv_splits)
fold = 0
predictions = []
for train_indices, test_indices in group_kfold.split(X, y, users):
fold += 1
print('Evaluating fold %d' % fold)
out_X_train = X[train_indices]
out_y_train = y[train_indices]
naug_samp = augment(out_X_train,
out_y_train,
sleep_states,
fold=fold,
aug_factor=1.5)
out_X_train = np.memmap('tmp/X_aug_fold'+str(fold)+'.np', dtype='float32', mode='r', \
shape=(naug_samp,out_X_train.shape[1],out_X_train.shape[2]))
out_y_train = np.memmap('tmp/y_aug_fold' + str(fold) + '.np',
dtype='int32',
mode='r',
shape=(naug_samp, out_y_train.shape[1]))
out_X_test = X[test_indices]
out_y_test = y[test_indices]
out_lbl = out_y_train.argmax(axis=1)
# Inner CV
val_acc = []
models = []
strat_kfold = StratifiedKFold(n_splits=inner_cv_splits,
random_state=0,
shuffle=False)
for grp_train_indices, grp_test_indices in strat_kfold.split(
out_X_train, out_lbl):
grp_train_indices = sample(list(grp_train_indices),
len(grp_train_indices))
in_X_train = out_X_train[grp_train_indices]
in_y_train = out_y_train[grp_train_indices]
grp_test_indices = sample(list(grp_test_indices), 1000)
in_X_test = out_X_train[grp_test_indices]
in_y_test = out_y_train[grp_test_indices]
#print(Counter(in_y_train[:1000].argmax(axis=1))); continue
limit_mem()
# Generate candidate architectures
model = modelgen.generate_models(in_X_train.shape, \
number_of_classes=num_classes, \
number_of_models=1, metrics=[macro_f1], model_type='CNN')
# Compare generated architectures on a subset of data for few epochs
outfile = os.path.join(resultdir, 'model_comparison.json')
hist, acc, loss = find_architecture.train_models_on_samples(in_X_train, \
in_y_train, in_X_test, in_y_test, model, nr_epochs=5, \
subset_size=5000, verbose=True, batch_size=50, \
outputfile=outfile, metric='macro_f1')
val_acc.append(acc[0])
models.append(model[0])
# Choose best model and evaluate values on validation data
print('Evaluating on best model for fold %d' % fold)
best_model_index = np.argmax(val_acc)
best_model, best_params, best_model_type = models[best_model_index]
print('Best model type and parameters:')
print(best_model_type)
print(best_params)
nr_epochs = 5
ntrain = out_X_train.shape[0]
nval = ntrain // 5
val_idx = np.random.randint(ntrain, size=nval)
train_idx = [
i for i in range(out_X_train.shape[0]) if i not in val_idx
]
trainX = out_X_train[train_idx]
trainY = out_y_train[train_idx]
valX = out_X_train[val_idx]
valY = out_y_train[val_idx]
limit_mem()
best_model = modelgen.generate_CNN_model(trainX.shape, num_classes, filters=best_params['filters'], \
fc_hidden_nodes=best_params['fc_hidden_nodes'], \
learning_rate=best_params['learning_rate'], \
regularization_rate=best_params['regularization_rate'], \
metrics=[macro_f1])
history = best_model.fit(trainX, trainY, epochs=nr_epochs, batch_size=50, \
validation_data=(valX, valY))
# Save model
best_model.save(
os.path.join(resultdir, 'best_model_fold' + str(fold) + '.h5'))
# Predict probability on validation data
probs = best_model.predict_proba(out_X_test, batch_size=1)
y_pred = probs.argmax(axis=1)
y_true = out_y_test.argmax(axis=1)
predictions.append((y_true, y_pred))
get_classification_report(predictions, sleep_states)
def test_generate_models_metrics(self):
models = modelgen.generate_models((None, 20, 3), 2)
model, hyperparams, modeltype = models[0]
assert_equal(model.metrics, ['accuracy'])
def main(argv):
indir = argv[0]
mode = argv[1] # binary or multiclass
outdir = argv[2]
if mode == 'multiclass':
sleep_states = [
'Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM', 'Nonwear', 'Wake_ext'
]
else:
sleep_states = ['Wake', 'Sleep', 'Nonwear', 'Wake_ext']
collate_sleep = ['NREM 1', 'NREM 2', 'NREM 3', 'REM']
valid_sleep_states = [
state for state in sleep_states if state != 'Wake_ext'
]
num_classes = len(valid_sleep_states)
if not os.path.exists(outdir):
os.makedirs(outdir)
resultdir = os.path.join(outdir, mode, 'models')
if not os.path.exists(resultdir):
os.makedirs(resultdir)
# Read data from disk
data = pd.read_csv(os.path.join(indir, 'labels.txt'), sep='\t')
files = []
labels = []
users = []
for idx, row in data.iterrows():
files.append(os.path.join(indir, row['filename']) + '.npy')
labels.append(row['labels'])
users.append(row['user'])
if mode == 'binary':
labels = ['Sleep' if lbl in collate_sleep else lbl for lbl in labels]
early_stopping = EarlyStopping(monitor='val_macro_f1',
mode='max',
verbose=1,
patience=2)
seqlen, n_channels = np.load(files[0]).shape
batch_size = 32
# Use nested cross-validation based on users
# Outer CV
unique_users = list(set(users))
random.shuffle(unique_users)
out_cv_splits = 5
in_cv_splits = 5
out_fold_nusers = len(unique_users) // out_cv_splits
out_n_epochs = 10
in_n_epochs = 1
predictions = []
wake_idx = sleep_states.index('Wake')
wake_ext_idx = sleep_states.index('Wake_ext')
for out_fold in range(out_cv_splits):
print('Evaluating fold %d' % (out_fold + 1))
test_users = unique_users[out_fold * out_fold_nusers:(out_fold + 1) *
out_fold_nusers]
trainval_users = [
user for user in unique_users if user not in test_users
]
train_users = trainval_users[:int(0.8 * len(trainval_users))]
val_users = trainval_users[len(train_users):]
out_train_fnames, out_train_labels, out_train_users = get_partition(files, labels, users, train_users,\
sleep_states, is_train=True)
out_val_fnames, out_val_labels, out_val_users = get_partition(
files, labels, users, val_users, sleep_states)
out_test_fnames, out_test_labels, out_test_users = get_partition(
files, labels, users, test_users, sleep_states)
out_train_gen = DataGenerator(out_train_fnames, out_train_labels, valid_sleep_states, partition='out_train',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, shuffle=True, augment=True, aug_factor=0.75, balance=True)
print(
'Fold {}: Computing mean and standard deviation'.format(out_fold +
1))
mean, std = out_train_gen.fit()
#mean = None; std = None
out_val_gen = DataGenerator(out_val_fnames, out_val_labels, valid_sleep_states, partition='out_val',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, mean=mean, std=std)
out_test_gen = DataGenerator(out_test_fnames, out_test_labels, valid_sleep_states, partition='out_test',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, mean=mean, std=std)
# Get class weights
out_class_wts = class_weight.compute_class_weight(
'balanced', np.unique(out_train_labels), out_train_labels)
# Inner CV
val_acc = []
models = []
in_fold_nusers = len(trainval_users) // in_cv_splits
for in_fold in range(in_cv_splits):
in_val_users = trainval_users[in_fold *
in_fold_nusers:(in_fold + 1) *
in_fold_nusers]
in_train_users = [
user for user in trainval_users if user not in in_val_users
]
in_train_fnames, in_train_labels, in_train_users = get_partition(files, labels, users, in_train_users,\
sleep_states, is_train=True)
in_val_fnames, in_val_labels, in_val_users = get_partition(
files, labels, users, in_val_users, sleep_states)
in_train_gen = DataGenerator(in_train_fnames, in_train_labels, valid_sleep_states, partition='in_train',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, shuffle=True, augment=True, aug_factor=0.75, balance=True,\
mean=mean, std=std)
in_val_gen = DataGenerator(in_val_fnames, in_val_labels, valid_sleep_states, partition='in_val',\
batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\
n_classes=num_classes, mean=mean, std=std)
# Generate candidate architectures
model = modelgen.generate_models((None, seqlen, n_channels), \
number_of_classes=num_classes, \
number_of_models=1, metrics=[macro_f1])#, model_type='CNN')
# Compare generated architectures on a subset of data for few epochs
outfile = os.path.join(resultdir, 'model_comparison.json')
hist, acc, loss = find_architecture.train_models_on_samples(in_train_gen, in_val_gen,
model, nr_epochs=in_n_epochs, n_steps=1000, class_weight=out_class_wts, \
verbose=True, outputfile=outfile, metric='macro_f1')
val_acc.append(acc[0])
models.append(model[0])
# Choose best model and evaluate values on validation data
print('Evaluating on best model for fold %d' % out_fold)
best_model_index = np.argmax(val_acc)
best_model, best_params, best_model_type = models[best_model_index]
print('Best model type and parameters:')
print(best_model_type)
print(best_params)
if best_model_type == 'CNN':
best_model = modelgen.generate_CNN_model((None, seqlen, n_channels), num_classes, filters=best_params['filters'], \
fc_hidden_nodes=best_params['fc_hidden_nodes'], \
learning_rate=best_params['learning_rate'], \
regularization_rate=best_params['regularization_rate'], \
metrics=[macro_f1])
else:
best_model = modelgen.generate_DeepConvLSTM_model((None, seqlen, n_channels), num_classes,\
filters=best_params['filters'], \
lstm_dims=best_params['lstm_dims'], \
learning_rate=best_params['learning_rate'], \
regularization_rate=best_params['regularization_rate'], \
metrics=[macro_f1])
# Use early stopping and model checkpoints to handle overfitting and save best model
model_checkpt = ModelCheckpoint(os.path.join(resultdir,'best_model_fold'+str(out_fold+1)+'.h5'), monitor='val_macro_f1',\
mode='max', save_best_only=True)
history = F1scoreHistory()
hist = best_model.fit_generator(out_train_gen, epochs=out_n_epochs, \
validation_data=out_val_gen, class_weight=out_class_wts,\
callbacks=[early_stopping, model_checkpt])
# Plot training history
# plt.Figure()
# plt.plot(history.mean_f1score['train'])
# #plt.plot(history.mean_f1score['val'])
# plt.title('Model F1-score')
# plt.ylabel('F1-score')
# plt.xlabel('Batch')
# #plt.legend(['Train', 'Test'], loc='upper left')
# plt.savefig(os.path.join(resultdir,'Fold'+str(fold)+'_performance_curve.jpg'))
# plt.clf()
#
## # Save model
## best_model.save(os.path.join(resultdir,'best_model_fold'+str(fold)+'.h5'))
# Predict probability on validation data
probs = best_model.predict_generator(out_test_gen)
y_pred = probs.argmax(axis=1)
y_true = out_test_labels
predictions.append((out_test_users, y_true, y_pred))
# Save user report
if mode == 'binary':
save_user_report(
predictions, valid_sleep_states,
os.path.join(
resultdir, 'fold' + str(out_fold + 1) +
'_deeplearning_binary_results.csv'))
else:
save_user_report(
predictions, valid_sleep_states,
os.path.join(
resultdir, 'fold' + str(out_fold + 1) +
'_deeplearning_multiclass_results.csv'))
get_classification_report(predictions, valid_sleep_states)
# Save user report
if mode == 'binary':
save_user_report(
predictions, valid_sleep_states,
os.path.join(resultdir, 'deeplearning_binary_results.csv'))
else:
save_user_report(
predictions, valid_sleep_states,
os.path.join(resultdir, 'deeplearning_multiclass_results.csv'))
y_val = y_train[porc:,:]
X_train = X_train[:porc,:]
y_train = y_train[:porc,:]
print(y_train.shape)
print(y_val.shape)
print(y_test.shape)
if(args.epochs!=None):
epocas = args.epochs
if(args.load_model==None):
models = modelgen.generate_models(X_train.shape,
number_of_classes=2,
number_of_models =5,
model_type='DeepConvLSTM',
low_lr=2, high_lr=5
)
models_to_print = range(len(models))
for i, item in enumerate(models):
if i in models_to_print:
model, params, model_types = item
print("-------------------------------------------------------------------------------------------------------")
print("Model " + str(i))
print(" ")
print("Hyperparameters:")
print(params)
print(" ")
print("Model description:")
model.summary()
def test_generate_models_metrics(self):
models = modelgen.generate_models((None, 20, 3), 2)
model, hyperparams, modeltype = models[0]
metrics = [m.name for m in model.metrics]
assert metrics == ['accuracy'] or metrics == ['acc']
y_train = to_categorical(y_train_old,2)
print("Pre Treino")
porc = int((X_train.shape[0])*0.8)
X_val = X_train[porc:,:]
y_val = y_train[porc:,:]
X_train = X_train[:porc,:]
y_train = y_train[:porc,:]
print(X_train.shape)
print(X_val.shape)
models = modelgen.generate_models(X_train.shape,
number_of_classes=2,
number_of_models =5,
# metrics = {'f':fmeasure,'p':precision,'r':recall},
#metrics = {'acc':'accuracy','f':fmeasure,'p':precision,'r':recall},
model_type='DeepConvLSTM',
low_lr=2, high_lr=5
)
models_to_print = range(len(models))
for i, item in enumerate(models):
if i in models_to_print:
model, params, model_types = item
print("-------------------------------------------------------------------------------------------------------")
print("Model " + str(i))
print(" ")
print("Hyperparameters:")
print(params)
print(" ")
ys.append(np.load(os.path.join(datapath, 'y_' + str(i) + ext)))
# In[4]:
print(Xs[0].shape, ys[0].shape)
# ## Generate models
# First step is to create a model architecture. As we do not know what architecture is best for our data we will create a set of models to investigate which architecture is most suitable for our data and classification task. You will need to specificy how many models you want to create with argument 'number_of_models', the type of model which can been 'CNN' or 'DeepConvLSTM', and maximum number of layers per modeltype. See for a full overview of the optional arguments the function documentation of modelgen.generate_models
# In[16]:
num_classes = ys[0].shape[1]
np.random.seed(123)
models = modelgen.generate_models(Xs[0].shape,
number_of_classes=num_classes,
number_of_models=nr_models)
# In[ ]:
# In[19]:
for i, (model, params, model_type) in enumerate(models):
storage.savemodel(model, resultpath, "model_" + str(i))
# ## Compare models
# Now that the model architectures have been generated it is time to compare the models by training them in a subset of the training data and evaluating the models in the validation subset. This will help us to choose the best candidate model. Performance results are stored in a json file.
# In[25]:
