def test_deepconvlstm_starts_with_batchnorm(self): """ DeepConvLSTM models should always start with a batch normalization layer. """ model = modelgen.generate_DeepConvLSTM_model((None, 20, 3), 2, [32, 32], [32, 32]) assert_equal( str(type(model.layers[0])), "<class 'keras.layers.normalization.BatchNormalization'>", 'Wrong layer type.')
def test_deepconvlstm_enough_batchnorm(self): """LSTM model should contain as many batch norm layers as it has activations layers""" model = modelgen.generate_DeepConvLSTM_model( (None, 20, 3), 2, [32, 32, 32], [32, 32, 32]) batch_norm_layers = len( [l for l in model.layers if 'BatchNormalization' in str(l)]) activation_layers = len( [l for l in model.layers if 'Activation' in str(l)]) assert_equal(batch_norm_layers, activation_layers)
def test_deepconvlstm_batchnorm_dim(self): """The output shape of the batchnorm should be (None, nr_filters, nr_timesteps, nr_channels)""" model = modelgen.generate_DeepConvLSTM_model((None, 20, 3), 2, [32, 32], [32, 32]) batchnormlay = model.layers[3] assert_equal(batchnormlay.output_shape, (None, 32, 20, 3))
def main(argv): indir = argv[0] mode = argv[1] # binary or multiclass outdir = argv[2] if mode == 'multiclass': sleep_states = [ 'Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM', 'Nonwear', 'Wake_ext' ] else: sleep_states = ['Wake', 'Sleep', 'Nonwear', 'Wake_ext'] collate_sleep = ['NREM 1', 'NREM 2', 'NREM 3', 'REM'] valid_sleep_states = [ state for state in sleep_states if state != 'Wake_ext' ] num_classes = len(valid_sleep_states) if not os.path.exists(outdir): os.makedirs(outdir) resultdir = os.path.join(outdir, mode, 'models') if not os.path.exists(resultdir): os.makedirs(resultdir) # Read data from disk data = pd.read_csv(os.path.join(indir, 'labels.txt'), sep='\t') files = [] labels = [] users = [] for idx, row in data.iterrows(): files.append(os.path.join(indir, row['filename']) + '.npy') labels.append(row['labels']) users.append(row['user']) if mode == 'binary': labels = ['Sleep' if lbl in collate_sleep else lbl for lbl in labels] early_stopping = EarlyStopping(monitor='val_macro_f1', mode='max', verbose=1, patience=2) seqlen, n_channels = np.load(files[0]).shape batch_size = 32 # Use nested cross-validation based on users # Outer CV unique_users = list(set(users)) random.shuffle(unique_users) out_cv_splits = 5 in_cv_splits = 5 out_fold_nusers = len(unique_users) // out_cv_splits out_n_epochs = 10 in_n_epochs = 1 predictions = [] wake_idx = sleep_states.index('Wake') wake_ext_idx = sleep_states.index('Wake_ext') for out_fold in range(out_cv_splits): print('Evaluating fold %d' % (out_fold + 1)) test_users = unique_users[out_fold * out_fold_nusers:(out_fold + 1) * out_fold_nusers] trainval_users = [ user for user in unique_users if user not in test_users ] train_users = trainval_users[:int(0.8 * len(trainval_users))] val_users = trainval_users[len(train_users):] out_train_fnames, out_train_labels, out_train_users = get_partition(files, labels, users, train_users,\ sleep_states, is_train=True) out_val_fnames, out_val_labels, out_val_users = get_partition( files, labels, users, val_users, sleep_states) out_test_fnames, out_test_labels, out_test_users = get_partition( files, labels, users, test_users, sleep_states) out_train_gen = DataGenerator(out_train_fnames, out_train_labels, valid_sleep_states, partition='out_train',\ batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\ n_classes=num_classes, shuffle=True, augment=True, aug_factor=0.75, balance=True) print( 'Fold {}: Computing mean and standard deviation'.format(out_fold + 1)) mean, std = out_train_gen.fit() #mean = None; std = None out_val_gen = DataGenerator(out_val_fnames, out_val_labels, valid_sleep_states, partition='out_val',\ batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\ n_classes=num_classes, mean=mean, std=std) out_test_gen = DataGenerator(out_test_fnames, out_test_labels, valid_sleep_states, partition='out_test',\ batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\ n_classes=num_classes, mean=mean, std=std) # Get class weights out_class_wts = class_weight.compute_class_weight( 'balanced', np.unique(out_train_labels), out_train_labels) # Inner CV val_acc = [] models = [] in_fold_nusers = len(trainval_users) // in_cv_splits for in_fold in range(in_cv_splits): in_val_users = trainval_users[in_fold * in_fold_nusers:(in_fold + 1) * in_fold_nusers] in_train_users = [ user for user in trainval_users if user not in in_val_users ] in_train_fnames, in_train_labels, in_train_users = get_partition(files, labels, users, in_train_users,\ sleep_states, is_train=True) in_val_fnames, in_val_labels, in_val_users = get_partition( files, labels, users, in_val_users, sleep_states) in_train_gen = DataGenerator(in_train_fnames, in_train_labels, valid_sleep_states, partition='in_train',\ batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\ n_classes=num_classes, shuffle=True, augment=True, aug_factor=0.75, balance=True,\ mean=mean, std=std) in_val_gen = DataGenerator(in_val_fnames, in_val_labels, valid_sleep_states, partition='in_val',\ batch_size=batch_size, seqlen=seqlen, n_channels=n_channels,\ n_classes=num_classes, mean=mean, std=std) # Generate candidate architectures model = modelgen.generate_models((None, seqlen, n_channels), \ number_of_classes=num_classes, \ number_of_models=1, metrics=[macro_f1])#, model_type='CNN') # Compare generated architectures on a subset of data for few epochs outfile = os.path.join(resultdir, 'model_comparison.json') hist, acc, loss = find_architecture.train_models_on_samples(in_train_gen, in_val_gen, model, nr_epochs=in_n_epochs, n_steps=1000, class_weight=out_class_wts, \ verbose=True, outputfile=outfile, metric='macro_f1') val_acc.append(acc[0]) models.append(model[0]) # Choose best model and evaluate values on validation data print('Evaluating on best model for fold %d' % out_fold) best_model_index = np.argmax(val_acc) best_model, best_params, best_model_type = models[best_model_index] print('Best model type and parameters:') print(best_model_type) print(best_params) if best_model_type == 'CNN': best_model = modelgen.generate_CNN_model((None, seqlen, n_channels), num_classes, filters=best_params['filters'], \ fc_hidden_nodes=best_params['fc_hidden_nodes'], \ learning_rate=best_params['learning_rate'], \ regularization_rate=best_params['regularization_rate'], \ metrics=[macro_f1]) else: best_model = modelgen.generate_DeepConvLSTM_model((None, seqlen, n_channels), num_classes,\ filters=best_params['filters'], \ lstm_dims=best_params['lstm_dims'], \ learning_rate=best_params['learning_rate'], \ regularization_rate=best_params['regularization_rate'], \ metrics=[macro_f1]) # Use early stopping and model checkpoints to handle overfitting and save best model model_checkpt = ModelCheckpoint(os.path.join(resultdir,'best_model_fold'+str(out_fold+1)+'.h5'), monitor='val_macro_f1',\ mode='max', save_best_only=True) history = F1scoreHistory() hist = best_model.fit_generator(out_train_gen, epochs=out_n_epochs, \ validation_data=out_val_gen, class_weight=out_class_wts,\ callbacks=[early_stopping, model_checkpt]) # Plot training history # plt.Figure() # plt.plot(history.mean_f1score['train']) # #plt.plot(history.mean_f1score['val']) # plt.title('Model F1-score') # plt.ylabel('F1-score') # plt.xlabel('Batch') # #plt.legend(['Train', 'Test'], loc='upper left') # plt.savefig(os.path.join(resultdir,'Fold'+str(fold)+'_performance_curve.jpg')) # plt.clf() # ## # Save model ## best_model.save(os.path.join(resultdir,'best_model_fold'+str(fold)+'.h5')) # Predict probability on validation data probs = best_model.predict_generator(out_test_gen) y_pred = probs.argmax(axis=1) y_true = out_test_labels predictions.append((out_test_users, y_true, y_pred)) # Save user report if mode == 'binary': save_user_report( predictions, valid_sleep_states, os.path.join( resultdir, 'fold' + str(out_fold + 1) + '_deeplearning_binary_results.csv')) else: save_user_report( predictions, valid_sleep_states, os.path.join( resultdir, 'fold' + str(out_fold + 1) + '_deeplearning_multiclass_results.csv')) get_classification_report(predictions, valid_sleep_states) # Save user report if mode == 'binary': save_user_report( predictions, valid_sleep_states, os.path.join(resultdir, 'deeplearning_binary_results.csv')) else: save_user_report( predictions, valid_sleep_states, os.path.join(resultdir, 'deeplearning_multiclass_results.csv'))