def main(argv): infile = argv[0] outdir = argv[1] sleep_states = ['Wake', 'Sleep'] df = pd.read_csv(infile) df = df[(df['label'] != 'Wake_ext') & (df['label'] != 'Nonwear')].reset_index() df['binary_label'] = df['label'] df.loc[df['label'] == 'NREM 1', 'binary_label'] = 'Sleep' df.loc[df['label'] == 'NREM 2', 'binary_label'] = 'Sleep' df.loc[df['label'] == 'NREM 3', 'binary_label'] = 'Sleep' df.loc[df['label'] == 'REM', 'binary_label'] = 'Sleep' y_true = np.array([sleep_states.index(val) for val in df['binary_label']]) y_pred = np.array([sleep_states.index(val) for val in df['heuristic']]) y_pred_onehot = np.zeros( (len(y_pred), len(sleep_states))) # convert to one-hot representation y_pred_onehot[np.arange(len(y_pred)), y_pred] = 1 predictions = [(df['user'], df['timestamp'], df['filename'], y_true, y_pred_onehot)] cv_save_classification_result( predictions, sleep_states, os.path.join(outdir, 'heuristic_classification.csv'))
def main(argv): infile = argv[0] mode = argv[1] # binary or multiclass or nonwear dataset = argv[2] outdir = argv[3] resultdir = os.path.join(outdir, 'models') if not os.path.exists(resultdir): os.makedirs(resultdir) # Read data file and retain data only corresponding to 5 sleep states or nonwear df = pd.read_csv(infile, dtype={ 'label': object, 'user': object, 'position': object, 'dataset': object }) if mode == 'binary': states = ['Wake', 'Sleep'] collate_states = ['NREM 1', 'NREM 2', 'NREM 3', 'REM'] df.loc[df['label'].isin(collate_states), 'label'] = 'Sleep' elif mode == 'nonwear': states = ['Wear', 'Nonwear'] collate_states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] df.loc[df['label'].isin(collate_states), 'label'] = 'Wear' else: states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] df = df[df['label'].isin(states)].reset_index() print('... Number of data samples: %d' % len(df)) ctr = Counter(df['label']) for cls in ctr: print('%s: %d (%0.2f%%)' % (cls, ctr[cls], ctr[cls] * 100.0 / len(df))) feat_cols = [ 'ENMO_mean', 'ENMO_std', 'ENMO_min', 'ENMO_max', 'ENMO_mad', 'ENMO_entropy1', 'ENMO_entropy2', 'ENMO_prevdiff', 'ENMO_nextdiff', 'angz_mean', 'angz_std', 'angz_min', 'angz_max', 'angz_mad', 'angz_entropy1', 'angz_entropy2', 'angz_prevdiff', 'angz_nextdiff', 'LIDS_mean', 'LIDS_std', 'LIDS_min', 'LIDS_max', 'LIDS_mad', 'LIDS_entropy1', 'LIDS_entropy2', 'LIDS_prevdiff', 'LIDS_nextdiff' ] ######################## Partition the datasets ####################### # Nested cross-validation - outer CV for estimating model performance # Inner CV for estimating model hyperparameters # Split data based on users, not on samples, for outer CV # Use Stratified CV for inner CV to ensure similar label distribution ts = df['timestamp'] X = df[feat_cols].values y = df['label'] y = np.array([states.index(i) for i in y]) groups = df['user'] fnames = df['filename'] feat_len = X.shape[1] # Outer CV imbalanced_pred = [] imbalanced_imp = [] balanced_pred = [] balanced_imp = [] outer_cv_splits = 5 inner_cv_splits = 5 group_kfold = GroupKFold(n_splits=outer_cv_splits) out_fold = 0 for train_indices, test_indices in group_kfold.split(X, y, groups): out_fold += 1 out_fold_X_train = X[train_indices, :] out_fold_X_test = X[test_indices, :] out_fold_y_train = y[train_indices] out_fold_y_test = y[test_indices] out_fold_users_test = groups[test_indices] out_fold_ts_test = ts[test_indices] out_fold_fnames_test = fnames[test_indices] # Inner CV strat_kfold = StratifiedKFold(n_splits=inner_cv_splits, random_state=0, shuffle=True) # #################### Without balancing ####################### # # custom_cv_indices = [] # for grp_train_idx, grp_test_idx in \ # strat_kfold.split(out_fold_X_train,out_fold_y_train): # custom_cv_indices.append((grp_train_idx, grp_test_idx)) # # pipe = Pipeline([('scl', StandardScaler()), # ('clf', RandomForestClassifier(class_weight='balanced', # random_state=0))]) # # print('Fold'+str(out_fold)+' - Imbalanced: Hyperparameter search') # search_params = {'clf__n_estimators':[50,100,200,300,500], # 'clf__max_depth': [5,10,None]} # cv_clf = RandomizedSearchCV(estimator=pipe, param_distributions=search_params, # cv=custom_cv_indices, scoring='f1_macro', n_iter=5, # n_jobs=-1, verbose=2) # cv_clf.fit(out_fold_X_train, out_fold_y_train) # pickle.dump(cv_clf, open(os.path.join(resultdir,\ # 'fold'+str(out_fold)+'_'+ mode + '_imbalanced_RF.sav'),'wb')) # out_fold_y_test_pred = cv_clf.predict_proba(out_fold_X_test) # print('Fold'+str(out_fold)+' - Imbalanced', cv_clf.best_params_) # # imbalanced_pred.append((out_fold_users_test, out_fold_ts_test, out_fold_fnames_test, # out_fold_y_test, out_fold_y_test_pred)) # imbalanced_imp.append(cv_clf.best_estimator_.named_steps['clf'].feature_importances_) ################## Balancing with SMOTE ################### scaler = StandardScaler() scaler.fit(out_fold_X_train) out_fold_X_train_sc = scaler.transform(out_fold_X_train) out_fold_X_test_sc = scaler.transform(out_fold_X_test) # Resample training data print('Fold' + str(out_fold) + ' - Balanced: SMOTE') # Imblearn - Undersampling techniques ENN and Tomek are too slow and # difficult to parallelize # So stick only with oversampling techniques smote = SMOTE(random_state=0, n_jobs=-1, sampling_strategy='all') out_fold_X_train_resamp, out_fold_y_train_resamp = \ smote.fit_resample(out_fold_X_train_sc, out_fold_y_train) custom_resamp_cv_indices = [] for grp_train_idx, grp_test_idx in \ strat_kfold.split(out_fold_X_train_resamp,out_fold_y_train_resamp): custom_resamp_cv_indices.append((grp_train_idx, grp_test_idx)) # Note: imblearn Pipeline is slow and sklearn pipeline yields poor results clf = RandomForestClassifier(class_weight='balanced', max_depth=None, random_state=0) print('Fold' + str(out_fold) + ' - Balanced: Hyperparameter search') search_params = { 'n_estimators': [50, 100, 200, 300, 500], 'max_depth': [5, 10, None] } cv_clf = RandomizedSearchCV(estimator=clf, param_distributions=search_params, cv=custom_resamp_cv_indices, scoring='f1_macro', n_iter=5, n_jobs=-1, verbose=2) cv_clf.fit(out_fold_X_train_resamp, out_fold_y_train_resamp) pickle.dump([scaler,cv_clf], open(os.path.join(resultdir,\ 'fold'+str(out_fold)+'_'+ mode + '_balanced_RF.sav'),'wb')) out_fold_y_test_pred = cv_clf.predict_proba(out_fold_X_test_sc) print('Fold' + str(out_fold) + ' - Balanced', cv_clf.best_params_) balanced_pred.append( (out_fold_users_test, out_fold_ts_test, out_fold_fnames_test, out_fold_y_test, out_fold_y_test_pred)) balanced_imp.append(cv_clf.best_estimator_.feature_importances_) # print('############## Imbalanced classification ##############') # # Save imbalanced classification reports # cv_save_feat_importances_result(imbalanced_imp, feat_cols, # os.path.join(outdir, mode + '_imbalanced_feat_imp.csv')) # cv_save_classification_result(imbalanced_pred, states, # os.path.join(outdir, mode + '_imbalanced_classification.csv')) print('############## Balanced classification ##############') # Save balanced classification reports cv_save_feat_importances_result( balanced_imp, feat_cols, os.path.join(outdir, mode + '_balanced_feat_imp.csv')) cv_save_classification_result( balanced_pred, states, os.path.join(outdir, mode + '_balanced_classification.csv'))
def main(argv): indir = args.indir mode = args.mode # binary or multiclass or nonwear outdir = args.outdir if mode == 'multiclass': states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM', 'Wake_ext'] elif mode == 'binary': states = ['Wake', 'Sleep', 'Wake_ext'] collate_states = ['NREM 1', 'NREM 2', 'NREM 3', 'REM'] elif mode == 'nonwear': states = ['Wear', 'Nonwear'] collate_states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] valid_states = [state for state in states if state != 'Wake_ext'] num_classes = len(valid_states) if not os.path.exists(outdir): os.makedirs(outdir) resultdir = os.path.join(outdir, mode, 'models') if not os.path.exists(resultdir): os.makedirs(resultdir) # Read data from disk data = pd.read_csv(os.path.join(indir, 'features_30.0s.csv')) labels = data['label'].values users = data['user'].values if mode == 'binary': labels = np.array( ['Sleep' if lbl in collate_states else lbl for lbl in labels]) elif mode == 'nonwear': labels = np.array( ['Wear' if lbl in collate_states else lbl for lbl in labels]) # Read raw data shape_df = pd.read_csv(os.path.join(indir, 'datashape_30.0s.csv')) num_samples = shape_df['num_samples'].values[0] seqlen = shape_df['num_timesteps'].values[0] n_channels = shape_df['num_channels'].values[0] raw_data = np.memmap(os.path.join(indir, 'rawdata_30.0s.npz'), dtype='float32', mode='r', shape=(num_samples, seqlen, n_channels)) # Hyperparameters lr = args.lr # learning rate num_epochs = args.num_epochs batch_size = args.batchsize max_seqlen = 1504 num_channels = args.num_channels # number of raw data channels feat_channels = args.feat_channels # Add ENMO, z-angle and LIDS as additional channels # Use nested cross-validation based on users # Outer CV unique_users = list(set(users)) random.shuffle(unique_users) cv_splits = 5 user_cnt = Counter(users[np.isin(labels, valid_states)]).most_common() samp_per_fold = len(users) // cv_splits # Get users to be used in test for each fold such that each fold has similar # number of samples fold_users = [[] for i in range(cv_splits)] fold_cnt = [[] for i in range(cv_splits)] for user, cnt in user_cnt: idx = -1 maxdiff = 0 for j in range(cv_splits): if (samp_per_fold - sum(fold_cnt[j])) > maxdiff: maxdiff = samp_per_fold - sum(fold_cnt[j]) idx = j fold_users[idx].append(user) fold_cnt[idx].append(cnt) predictions = [] if mode != 'nonwear': wake_idx = states.index('Wake') wake_ext_idx = states.index('Wake_ext') for fold in range(cv_splits): print('Evaluating fold %d' % (fold + 1)) test_users = fold_users[fold] trainval_users = [(key, val) for key, val in user_cnt if key not in test_users] random.shuffle(trainval_users) # validation data is approximately 10% of total samples val_samp = 0.1 * sum([tup[1] for tup in user_cnt]) nval = 0 val_sum = 0 while (val_sum < val_samp): val_sum += trainval_users[nval][1] nval += 1 val_users = [key for key, val in trainval_users[:nval]] train_users = [key for key, val in trainval_users[nval:]] print('#users: Train = {:d}, Val = {:d}, Test = {:d}'.format( len(train_users), len(val_users), len(test_users))) # Create partitions # make a copy to change wake_ext for this fold fold_labels = np.array( [states.index(lbl) if lbl in states else -1 for lbl in labels]) train_indices = get_partition(raw_data, fold_labels, users, train_users, states, mode, is_train=True) val_indices = get_partition(raw_data, fold_labels, users, val_users, states, mode) test_indices = get_partition(raw_data, fold_labels, users, test_users, states, mode) nsamples = len(train_indices) + len(val_indices) + len(test_indices) print('Train: {:0.2f}%, Val: {:0.2f}%, Test: {:0.2f}%'\ .format(len(train_indices)*100.0/nsamples, len(val_indices)*100.0/nsamples,\ len(test_indices)*100.0/nsamples)) if mode != 'nonwear': chosen_indices = train_indices[ fold_labels[train_indices] != wake_ext_idx] else: chosen_indices = train_indices class_wts = class_weight.compute_class_weight( class_weight='balanced', classes=np.unique(fold_labels[chosen_indices]), y=fold_labels[chosen_indices]) # Rename wake_ext as wake for training samples if mode != 'nonwear': rename_indices = train_indices[fold_labels[train_indices] == wake_ext_idx] fold_labels[rename_indices] = wake_idx print('Train', Counter(np.array(fold_labels)[train_indices])) print('Val', Counter(np.array(fold_labels)[val_indices])) print('Test', Counter(np.array(fold_labels)[test_indices])) # Data generators for computing statistics stat_gen = DataGenerator(train_indices, raw_data, fold_labels, valid_states, partition='stat',\ batch_size=batch_size, seqlen=seqlen, n_channels=num_channels, feat_channels=feat_channels,\ n_classes=num_classes, shuffle=True) mean, std = stat_gen.fit() np.savez(os.path.join(resultdir, 'Fold' + str(fold + 1) + '_stats'), mean=mean, std=std) # Data generators for train/val/test train_gen = DataGenerator(train_indices, raw_data, fold_labels, valid_states, partition='train',\ batch_size=batch_size, seqlen=seqlen, n_channels=num_channels, feat_channels=feat_channels,\ n_classes=num_classes, shuffle=True, augment=True, aug_factor=0.75, balance=True, mean=mean, std=std) val_gen = DataGenerator(val_indices, raw_data, fold_labels, valid_states, partition='val',\ batch_size=batch_size, seqlen=seqlen, n_channels=num_channels, feat_channels=feat_channels,\ n_classes=num_classes, mean=mean, std=std) test_gen = DataGenerator(test_indices, raw_data, fold_labels, valid_states, partition='test',\ batch_size=batch_size, seqlen=seqlen, n_channels=num_channels, feat_channels=feat_channels,\ n_classes=num_classes, mean=mean, std=std) # Create model # Use batchnorm as first step since computing mean and std # across entire dataset is time-consuming model = FCN(input_shape=(seqlen, num_channels + feat_channels), max_seqlen=max_seqlen, num_classes=len(valid_states), norm_max=args.maxnorm) #print(model.summary()) model.compile(optimizer=Adam(lr=lr), loss=focal_loss(), metrics=['accuracy', macro_f1]) # Train model # Use callback to compute F-scores over entire validation data metrics_cb = Metrics(val_data=val_gen, batch_size=batch_size) # Use early stopping and model checkpoints to handle overfitting and save best model model_checkpt = ModelCheckpoint(os.path.join(resultdir,'fold'+str(fold+1)+'_'+mode+'-{epoch:02d}-{val_f1:.4f}.h5'),\ monitor='val_f1',\ mode='max', save_best_only=True) batch_renorm_cb = BatchRenormScheduler(len(train_gen)) history = model.fit( train_gen, epochs=num_epochs, validation_data=val_gen, verbose=1, shuffle=False, callbacks=[batch_renorm_cb, metrics_cb, model_checkpt], workers=2, max_queue_size=20, use_multiprocessing=False) # Plot training history plot_results(fold+1, history.history['loss'], history.history['val_loss'],\ os.path.join(resultdir,'Fold'+str(fold+1)+'_'+mode+'_loss.jpg'), metric='Loss') plot_results(fold+1, history.history['accuracy'], history.history['val_accuracy'],\ os.path.join(resultdir,'Fold'+str(fold+1)+'_'+mode+'_accuracy.jpg'), metric='Accuracy') plot_results(fold+1, history.history['macro_f1'], metrics_cb.val_f1,\ os.path.join(resultdir,'Fold'+str(fold+1)+'_'+mode+'_macro_f1.jpg'), metric='Macro F1') # Predict probability on validation data using best model best_model_file, epoch, val_f1 = get_best_model(resultdir, fold + 1) print('Predicting with model saved at Epoch={:d} with val_f1={:0.4f}'. format(epoch, val_f1)) model.load_weights(os.path.join(resultdir, best_model_file)) probs = model.predict(test_gen) y_pred = probs.argmax(axis=1) y_true = fold_labels[test_indices] predictions.append( (users[test_indices], data.iloc[test_indices]['timestamp'], data.iloc[test_indices]['filename'], test_indices, y_true, probs)) # Save user report cv_save_classification_result( predictions, valid_states, os.path.join( resultdir, 'fold' + str(fold + 1) + '_deeplearning_' + mode + '_results.csv'), method='dl') cv_get_classification_report(predictions, mode, valid_states, method='dl') cv_get_classification_report(predictions, mode, valid_states, method='dl') # Save user report cv_save_classification_result(predictions, valid_states, os.path.join( resultdir, 'deeplearning_' + mode + '_results.csv'), method='dl')
def main(args): if not os.path.exists(os.path.join(args.outdir, 'models')): os.makedirs(os.path.join(args.outdir, 'models')) if args.mode == 'binary': sleep_states = ['Wake', 'Sleep', 'Nonwear'] else: sleep_states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM', 'Nonwear'] feat_cols = [ 'ENMO_mean', 'ENMO_std', 'ENMO_min', 'ENMO_max', 'ENMO_mad', 'ENMO_entropy1', 'ENMO_entropy2', 'ENMO_prevdiff', 'ENMO_nextdiff', 'angz_mean', 'angz_std', 'angz_min', 'angz_max', 'angz_mad', 'angz_entropy1', 'angz_entropy2', 'angz_prevdiff', 'angz_nextdiff', 'LIDS_mean', 'LIDS_std', 'LIDS_min', 'LIDS_max', 'LIDS_mad', 'LIDS_entropy1', 'LIDS_entropy2', 'LIDS_prevdiff', 'LIDS_nextdiff' ] X_test, y_test, users_test, ts_test, fnames_test = get_data(args.test, feat_cols, sleep_states, mode=args.mode) predictions = [] feat_imp = [] if args.testmode == 'pretrain': # 'pretrained' - use pretrained models on test data model_files = os.listdir(args.modeldir) for i, fname in enumerate(model_files): scaler, clf = pickle.load( open(os.path.join(args.modeldir, fname), 'rb')) X_test_sc = scaler.transform(X_test) y_pred = clf.predict_proba(X_test_sc) predictions.append( (users_test, ts_test, fnames_test, y_test, y_pred)) feat_imp.append(clf.best_estimator_.feature_importances_) else: resultdir = os.path.join(args.outdir, 'models', 'tranfer', args.mode) if not os.path.exists(resultdir): os.makedirs(resultdir) # 'finetune'- use models tuned using validation data from same distribution as test data X_train, y_train, users_train, _, _ = get_data(args.train, feat_cols, sleep_states, mode=args.mode) X_val, y_val, _, _, _ = get_data(args.val, feat_cols, sleep_states, mode=args.mode) outer_cv_splits = 5 inner_cv_splits = 5 group_kfold = GroupKFold(n_splits=outer_cv_splits) fold = 0 for train_indices, test_indices in group_kfold.split( X_train, y_train, users_train): fold += 1 fold_X_train = X_train[train_indices, :] fold_y_train = y_train[train_indices] # Scale features scaler = StandardScaler() scaler.fit(fold_X_train) X_train_sc = scaler.transform(fold_X_train) X_val_sc = scaler.transform(X_val) X_test_sc = scaler.transform(X_test) # Balance training samples using SMOTE smote = SMOTE(random_state=0, n_jobs=-1, sampling_strategy='all') X_train_resamp, y_train_resamp = smote.fit_resample( X_train_sc, fold_y_train) X_concat = np.concatenate((X_train_resamp, X_val_sc), axis=0) y_concat = np.concatenate((y_train_resamp, y_val), axis=0) # Get suitable parameters using validation data clf = RandomForestClassifier(class_weight='balanced', max_depth=None, random_state=0) search_params = { 'n_estimators': [50, 100, 200, 300, 500], 'max_depth': [5, 10, None] } cv_indices = [(np.arange(X_train_resamp.shape[0]), np.arange(X_train_resamp.shape[0], X_concat.shape[0]))] cv_clf = RandomizedSearchCV(estimator=clf, param_distributions=search_params, cv=cv_indices, scoring='f1_macro', n_iter=5, n_jobs=-1, verbose=2) cv_clf.fit(X_concat, y_concat) pickle.dump( cv_clf, open( os.path.join( resultdir, 'fold_' + str(fold) + '_' + args.testmode + '_RF.sav'), 'wb')) y_pred = cv_clf.predict_proba(X_test_sc) predictions.append( (users_test, ts_test, fnames_test, y_test, y_pred)) feat_imp.append(cv_clf.best_estimator_.feature_importances_) cv_save_feat_importances_result( feat_imp, feat_cols, os.path.join( args.outdir, 'transfer_' + args.mode + '_' + args.testmode + '_feat_imp.csv')) cv_save_classification_result( predictions, sleep_states, os.path.join( args.outdir, 'transfer_' + args.mode + '_' + args.testmode + '_classification.csv'))
def main(argv): infile = argv[0] mode = argv[1] # binary or multiclass or nonwear dataset = argv[2] outdir = argv[3] resultdir = os.path.join(outdir,'models') if not os.path.exists(resultdir): os.makedirs(resultdir) # Read data file and retain data only corresponding to 5 sleep states or nonwear df = pd.read_csv(infile, dtype={'label':object, 'user':object, 'position':object, 'dataset':object}) if mode == 'binary': states = ['Wake', 'Sleep'] collate_states = ['NREM 1', 'NREM 2', 'NREM 3', 'REM'] df.loc[df['label'].isin(collate_states), 'label'] = 'Sleep' elif mode == 'nonwear': states = ['Wear', 'Nonwear'] collate_states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] df.loc[df['label'].isin(collate_states), 'label'] = 'Wear' else: states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] df = df[df['label'].isin(states)].reset_index() print('... Number of data samples: %d' % len(df)) ctr = Counter(df['label']) for cls in ctr: print('%s: %d (%0.2f%%)' % (cls,ctr[cls],ctr[cls]*100.0/len(df))) feat_cols = ['ENMO_mean','ENMO_std','ENMO_range','ENMO_mad', 'ENMO_entropy1','ENMO_entropy2', 'ENMO_prev30diff', 'ENMO_next30diff', 'ENMO_prev60diff', 'ENMO_next60diff', 'ENMO_prev120diff', 'ENMO_next120diff', 'angz_mean','angz_std','angz_range','angz_mad', 'angz_entropy1','angz_entropy2', 'angz_prev30diff', 'angz_next30diff', 'angz_prev60diff', 'angz_next60diff', 'angz_prev120diff', 'angz_next120diff', 'LIDS_mean','LIDS_std','LIDS_range','LIDS_mad', 'LIDS_entropy1','LIDS_entropy2', 'LIDS_prev30diff', 'LIDS_next30diff', 'LIDS_prev60diff', 'LIDS_next60diff', 'LIDS_prev120diff', 'LIDS_next120diff'] ######################## Partition the datasets ####################### # Nested cross-validation - outer CV for estimating model performance # Inner CV for estimating model hyperparameters # Split data based on users, not on samples, for outer CV # Use Stratified CV for inner CV to ensure similar label distribution ts = df['timestamp'] X = df[feat_cols].values y = df['label'] y = np.array([states.index(i) for i in y]) groups = df['user'] fnames = df['filename'] feat_len = X.shape[1] encoder = OneHotEncoder() scorer = make_scorer(average_precision_score, average='macro') # Outer CV imbalanced_pred = []; imbalanced_imp = [] balanced_pred = []; balanced_imp = [] outer_cv_splits = 5; inner_cv_splits = 5 outer_group_kfold = GroupKFold(n_splits=outer_cv_splits) out_fold = 0 for train_indices, test_indices in outer_group_kfold.split(X,y,groups): out_fold += 1 out_fold_X_train = X[train_indices,:]; out_fold_X_test = X[test_indices,:] out_fold_y_train = y[train_indices]; out_fold_y_test = y[test_indices] out_fold_users_train = groups[train_indices]; out_fold_users_test = groups[test_indices] out_fold_ts_test = ts[test_indices] out_fold_fnames_test = fnames[test_indices] if mode != 'multiclass': class_wt = compute_class_weight('balanced', np.unique(out_fold_y_train), out_fold_y_train) class_wt = {i:val for i,val in enumerate(class_wt)} else: class_wt = [] for cls in range(len(states)): class_train = (out_fold_y_train == cls).astype(np.int32) cls_wt = compute_class_weight('balanced', np.unique(class_train), class_train) cls_wt = {i:val for i,val in enumerate(cls_wt)} class_wt.append(cls_wt) # Inner CV ################## Balancing with SMOTE ################### scaler = StandardScaler() scaler.fit(out_fold_X_train) out_fold_X_train_sc = scaler.transform(out_fold_X_train) out_fold_X_test_sc = scaler.transform(out_fold_X_test) # Imblearn - Undersampling techniques ENN and Tomek are too slow and # difficult to parallelize # So stick only with oversampling techniques print('Fold'+str(out_fold)+' - Balanced: SMOTE') smote = SMOTE(random_state=0, n_jobs=-1, sampling_strategy='all') # Resample training data for each user train_users = list(set(out_fold_users_train)) out_fold_X_train_resamp, out_fold_y_train_resamp, out_fold_users_train_resamp = None, None, None for i,user in enumerate(train_users): #print('%d/%d - %s' % (i+1,len(train_users),user)) user_X = out_fold_X_train_sc[out_fold_users_train == user] user_y = out_fold_y_train[out_fold_users_train == user] if len(set(user_y)) == 1: print('%d/%d: %s has only one class' % (i+1,len(train_users),user)) print(Counter(user_y)) continue try: user_X_resamp, user_y_resamp = smote.fit_resample(user_X, user_y) except: print('%d/%d: %s failed to fit' % (i+1,len(train_users),user)) print(Counter(user_y)) continue user_y_resamp = user_y_resamp.reshape(-1,1) user_resamp = np.array([user] * len(user_X_resamp)).reshape(-1,1) if out_fold_X_train_resamp is None: out_fold_X_train_resamp = user_X_resamp out_fold_y_train_resamp = user_y_resamp out_fold_users_train_resamp = user_resamp else: out_fold_X_train_resamp = np.vstack((out_fold_X_train_resamp, user_X_resamp)) out_fold_y_train_resamp = np.vstack((out_fold_y_train_resamp, user_y_resamp)) out_fold_users_train_resamp = np.vstack((out_fold_users_train_resamp, user_resamp)) # Shuffle resampled data resamp_indices = np.arange(len(out_fold_X_train_resamp)) np.random.shuffle(resamp_indices) out_fold_X_train_resamp = out_fold_X_train_resamp[resamp_indices] out_fold_y_train_resamp = out_fold_y_train_resamp[resamp_indices].reshape(-1) out_fold_users_train_resamp = out_fold_users_train_resamp[resamp_indices].reshape(-1) inner_group_kfold = GroupKFold(n_splits=inner_cv_splits) custom_resamp_cv_indices = [] for grp_train_idx, grp_test_idx in \ inner_group_kfold.split(out_fold_X_train_resamp, out_fold_y_train_resamp, out_fold_users_train_resamp): custom_resamp_cv_indices.append((grp_train_idx, grp_test_idx)) grp_train_users = out_fold_users_train_resamp[grp_train_idx] grp_test_users = out_fold_users_train_resamp[grp_test_idx] # Note: imblearn Pipeline is slow and sklearn pipeline yields poor results clf = RandomForestClassifier(class_weight=class_wt, max_depth=None, random_state=0) print('Fold'+str(out_fold)+' - Balanced: Hyperparameter search') search_params = {'n_estimators':[100,150,200,300,400,500], 'max_depth': [5,10,15,20,None]} cv_clf = RandomizedSearchCV(estimator=clf, param_distributions=search_params, cv=custom_resamp_cv_indices, scoring=scorer, n_iter=10, n_jobs=-1, verbose=2) if mode == 'multiclass': out_fold_y_train_resamp = encoder.fit_transform(out_fold_y_train_resamp.reshape(-1,1)).todense() cv_clf.fit(out_fold_X_train_resamp, out_fold_y_train_resamp) print(cv_clf.best_estimator_) joblib.dump([scaler,cv_clf], os.path.join(resultdir,\ 'fold'+str(out_fold)+'_'+ mode + '_balanced_RF.sav')) out_fold_y_test_pred = cv_clf.predict_proba(out_fold_X_test_sc) if mode == 'multiclass': # collect probabilities from each binary classification multiclass_y_pred = None for cls in range(len(out_fold_y_test_pred)): if cls == 0: multiclass_y_pred = out_fold_y_test_pred[cls][:,1].reshape(-1,1) else: multiclass_y_pred = np.hstack((multiclass_y_pred, out_fold_y_test_pred[cls][:,1].reshape(-1,1))) out_fold_y_test_pred = multiclass_y_pred print('Fold'+str(out_fold)+' - Balanced', cv_clf.best_params_) balanced_pred.append((out_fold_users_test, out_fold_ts_test, out_fold_fnames_test, out_fold_y_test, out_fold_y_test_pred)) balanced_imp.append(cv_clf.best_estimator_.feature_importances_) print('############## Balanced classification ##############') # Save balanced classification reports cv_save_feat_importances_result(balanced_imp, feat_cols, os.path.join(outdir, mode + '_balanced_feat_imp.csv')) cv_save_classification_result(balanced_pred, states, os.path.join(outdir, mode + '_balanced_classification.csv'))
def main(argv): indir = args.indir mode = args.mode # binary or multiclass or nonwear modeldir = args.modeldir outdir = args.outdir if mode == 'multiclass': states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] elif mode == 'binary': states = ['Wake', 'Sleep'] collate_states = ['NREM 1', 'NREM 2', 'NREM 3', 'REM'] elif mode == 'nonwear': states = ['Wear', 'Nonwear'] collate_states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] num_classes = len(states) if not os.path.exists(outdir): os.makedirs(outdir) resultdir = os.path.join(outdir, mode, 'models') if not os.path.exists(resultdir): os.makedirs(resultdir) # Load pretrained model pretrained_model = load_model(os.path.join(modeldir, 'pretrained_model.h5')) pretrained_renet_weights = None for layer in pretrained_model.layers: if layer.name == "model": pretrained_resnet_weights = layer.get_weights() # Hyperparameters num_epochs = args.num_epochs num_channels = args.num_channels # number of raw data channels feat_channels = args.feat_channels # Add ENMO, z-angle and LIDS as additional channels batchsize = args.batchsize # Batchsize hp_epochs = args.hp_epochs # No. of hyperparameter validation epochs lr = args.lr # Learning rate batchsize = args.batchsize # Batch size resultdir = os.path.join(outdir, mode, 'lr-{:4f}_batchsize-{:d}'.format(lr, batchsize)) if not os.path.exists(resultdir): os.makedirs(resultdir) model_hyperparam = {} model_hyperparam['maxnorm'] = [0.5, 1.0, 2.0, 3.0] model_hyperparam['dense_units'] = {'min': 100, 'max': 700, 'step': 50} model_hyperparam['dropout'] = [0.1, 0.2, 0.3] model_hyperparam['lr'] = [1e-3, 1e-4] # Read data from disk data = pd.read_csv(os.path.join(indir, 'all_train_features_30.0s.csv')) ts = data['timestamp'] fnames = data['filename'] labels = data['label'] users = data['user'].astype(str).values if mode == 'binary': labels = np.array( ['Sleep' if lbl in collate_states else lbl for lbl in labels]) elif mode == 'nonwear': labels = np.array( ['Wear' if lbl in collate_states else lbl for lbl in labels]) # unique_users = list(set(users)) # random.shuffle(unique_users) # unique_users = unique_users[:10] num_classes = len(states) # Get valid values for CV split valid_indices = data[labels != -1].index.values # dummy values for partition as raw data cannot be loaded to memory X = data[['ENMO_mean', 'ENMO_std', 'ENMO_mad']].values[valid_indices] y = labels[valid_indices] groups = users[valid_indices] # Read raw data fp = h5py.File(os.path.join(indir, 'all_train_rawdata_30.0s.h5'), 'r') raw_data = fp['data'] [num_samples, seqlen, n_channels] = raw_data.shape # Read raw data statistics stats = np.load(os.path.join(modeldir, "stats.npz")) mean = stats['mean'] std = stats['std'] # Use nested cross-validation based on users # Outer CV outer_cv_splits = 5 inner_cv_splits = 5 out_fold = 0 predictions = [] outer_group_kfold = GroupKFold(n_splits=outer_cv_splits) for train_indices, test_indices in outer_group_kfold.split(X, y, groups): if os.path.exists('untitled_project'): shutil.rmtree('untitled_project') out_fold += 1 print('Evaluating outer fold %d' % (out_fold)) out_fold_train_indices = valid_indices[train_indices] out_fold_test_indices = valid_indices[test_indices] out_fold_y_train = labels[out_fold_train_indices] out_fold_y_test = labels[out_fold_test_indices] out_fold_users_train = users[out_fold_train_indices] out_fold_users_test = users[out_fold_test_indices] out_fold_ts_test = ts[out_fold_test_indices] out_fold_fnames_test = fnames[out_fold_test_indices] # Build a hypermodel for hyperparam search hyperModel = ResnetHyperModel(hyperparam=model_hyperparam, seqlen=seqlen,\ channels=num_channels+feat_channels,\ pretrained_wts=pretrained_resnet_weights,\ num_classes=num_classes) tuner = CVTuner(hypermodel=hyperModel, oracle=kerastuner.oracles.Hyperband( objective='val_loss', max_epochs=3), cv=inner_cv_splits, states=states, num_classes=num_classes, seqlen=seqlen, num_channels=num_channels, feat_channels=feat_channels, mean=mean, std=std) # Use a subset of training data for hyperparam search # train_users = list(set(out_fold_users_train)) # random.shuffle(train_users) # nsubtrain_users = int(0.5*len(train_users)) # sub_train_users = np.array(train_users[:nsubtrain_users]) # sub_train_indices = out_fold_train_indices[np.isin(out_fold_users_train, sub_train_users)] tuner.search(data=raw_data, labels=labels, users=users, indices=out_fold_train_indices, batch_size=batchsize) # Train fold with best best hyperparameters best_hp = tuner.get_best_hyperparameters()[0] with open( os.path.join(resultdir, 'fold' + str(out_fold) + '_hyperparameters.txt'), "w") as fp: fp.write("Maxnorm = {:.2f}\n".format(best_hp.values['maxnorm'])) fp.write("Learning rate = {:.4f}\n".format(best_hp.values['lr'])) fp.write("Preclassification layer units = {:d}\n".format( best_hp.values['preclassification'])) fp.write("Dropout = {:.2f}\n".format(best_hp.values['dropout'])) print(best_hp.values) model = tuner.hypermodel.build(best_hp) for layer in model.layers: if layer.name == "model": layer.set_weights(pretrained_resnet_weights) print(model.summary()) # Data generators # Split train data into train and validation data based on users trainval_users = list(set(out_fold_users_train)) random.shuffle(trainval_users) ntrainval_users = len(trainval_users) nval_users = int(0.1 * ntrainval_users) ntrain_users = ntrainval_users - nval_users train_users = np.array(trainval_users[:ntrain_users]) val_users = np.array(trainval_users[ntrain_users:]) out_fold_val_indices = out_fold_train_indices[np.isin( out_fold_users_train, val_users)] out_fold_train_indices = out_fold_train_indices[np.isin( out_fold_users_train, train_users)] train_gen = DataGenerator(out_fold_train_indices, raw_data, labels, states, partition='train',\ batch_size=batchsize, seqlen=seqlen, n_channels=num_channels, feat_channels=feat_channels,\ n_classes=num_classes, shuffle=True, augment=True, aug_factor=0.75, balance=True, mean=mean, std=std) val_gen = DataGenerator(out_fold_val_indices, raw_data, labels, states, partition='val',\ batch_size=batchsize, seqlen=seqlen, n_channels=num_channels, feat_channels=feat_channels,\ n_classes=num_classes, mean=mean, std=std) test_gen = DataGenerator(out_fold_test_indices, raw_data, labels, states, partition='test',\ batch_size=batchsize, seqlen=seqlen, n_channels=num_channels, feat_channels=feat_channels,\ n_classes=num_classes, mean=mean, std=std) # Use callback to compute F-scores over entire validation data metrics_cb = Metrics(val_data=val_gen, batch_size=batchsize) # Use early stopping and model checkpoints to handle overfitting and save best model model_checkpt = ModelCheckpoint(os.path.join(resultdir,'fold'+str(out_fold)+'_'+mode+'-{epoch:02d}-{val_f1:.4f}.h5'),\ monitor='val_f1',\ mode='max', save_best_only=True) batch_renorm_cb = BatchRenormScheduler(len(train_gen)) history = model.fit( train_gen, epochs=num_epochs, validation_data=val_gen, verbose=1, shuffle=False, callbacks=[batch_renorm_cb, metrics_cb, model_checkpt], workers=2, max_queue_size=20, use_multiprocessing=False) # Plot training history plot_results(out_fold+1, history.history['loss'], history.history['val_loss'],\ os.path.join(resultdir,'Fold'+str(out_fold)+'_'+mode+'_loss.jpg'), metric='Loss') plot_results(out_fold+1, history.history['accuracy'], history.history['val_accuracy'],\ os.path.join(resultdir,'Fold'+str(out_fold)+'_'+mode+'_accuracy.jpg'), metric='Accuracy') plot_results(out_fold+1, history.history['macro_f1'], metrics_cb.val_f1,\ os.path.join(resultdir,'Fold'+str(out_fold)+'_'+mode+'_macro_f1.jpg'), metric='Macro F1') # Predict probability on validation data using best model best_model_file, epoch, val_f1 = get_best_model(resultdir, out_fold) print('Predicting with model saved at Epoch={:d} with val_f1={:0.4f}'. format(epoch, val_f1)) model.load_weights(os.path.join(resultdir, best_model_file)) probs = model.predict(test_gen) y_pred = probs.argmax(axis=1) y_true = out_fold_y_test predictions.append( (users[test_indices], data.iloc[test_indices]['timestamp'], data.iloc[test_indices]['filename'], test_indices, y_true, probs)) # Save user report cv_save_classification_result( predictions, states, os.path.join( resultdir, 'fold' + str(out_fold) + '_deeplearning_' + mode + '_results.csv'), method='dl') cv_get_classification_report(predictions, mode, states, method='dl') cv_get_classification_report(predictions, mode, states, method='dl') # Save user report cv_save_classification_result(predictions, states, os.path.join( resultdir, 'deeplearning_' + mode + '_results.csv'), method='dl')
def main(argv): infile = argv[0] modeldir = argv[1] mode = argv[2] ensemble = int(argv[3]) # 0 - use best model, 1 - use ensemble outdir = argv[4] df = pd.read_csv(infile) method = 'feat_eng' if mode == 'binary': states = ['Wake', 'Sleep'] collate_states = ['NREM 1', 'NREM 2', 'NREM 3', 'REM'] df.loc[df['label'].isin(collate_states), 'label'] = 'Sleep' elif mode == 'nonwear': states = ['Wear', 'Nonwear'] collate_states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] df.loc[df['label'].isin(collate_states), 'label'] = 'Wear' elif mode == 'multiclass': states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM'] elif mode == 'hierarchical': method = 'hierarchical' states = ['Wake', 'NREM 1', 'NREM 2', 'NREM 3', 'REM', 'Nonwear'] # Class hierarchy for sleep stages class_hierarchy = { ROOT: {"Wear", "Nonwear"}, "Wear": {"Wake", "Sleep"}, "Sleep": {"NREM", "REM"}, "NREM": {"Light", "NREM 3"}, "Light": {"NREM 1", "NREM 2"} } graph = DiGraph(class_hierarchy) classes = [node for node in graph.nodes if node != ROOT] df = df[df['label'].isin(states)].reset_index() feat_cols = [ 'ENMO_mean', 'ENMO_std', 'ENMO_range', 'ENMO_mad', 'ENMO_entropy1', 'ENMO_entropy2', 'ENMO_prev30diff', 'ENMO_next30diff', 'ENMO_prev60diff', 'ENMO_next60diff', 'ENMO_prev120diff', 'ENMO_next120diff', 'angz_mean', 'angz_std', 'angz_range', 'angz_mad', 'angz_entropy1', 'angz_entropy2', 'angz_prev30diff', 'angz_next30diff', 'angz_prev60diff', 'angz_next60diff', 'angz_prev120diff', 'angz_next120diff', 'LIDS_mean', 'LIDS_std', 'LIDS_range', 'LIDS_mad', 'LIDS_entropy1', 'LIDS_entropy2', 'LIDS_prev30diff', 'LIDS_next30diff', 'LIDS_prev60diff', 'LIDS_next60diff', 'LIDS_prev120diff', 'LIDS_next120diff' ] ts_test = df['timestamp'] x_test = df[feat_cols].values y_test = df['label'] if mode != 'hierarchical': y_test = np.array([states.index(i) for i in y_test]) users_test = df['user'] fnames_test = df['filename'] N = x_test.shape[0] if ensemble: model_fnames = os.listdir(modeldir) model_fnames = [fname for fname in model_fnames if mode in fname] nfolds = len(model_fnames) for fold, fname in enumerate(model_fnames): print('Processing fold ' + str(fold + 1)) if mode != 'hierarchical': scaler, cv_clf = joblib.load( open(os.path.join(modeldir, fname), 'rb')) x_test_sc = scaler.transform(x_test) fold_y_pred = cv_clf.predict_proba(x_test_sc) if mode == 'multiclass': fold_y_pred_collated = np.zeros( (fold_y_pred[0].shape[0], len(fold_y_pred))) for cls in range(len(fold_y_pred)): fold_y_pred_collated[:, cls] = fold_y_pred[cls][:, 1] fold_y_pred = fold_y_pred_collated else: cv_clf = pickle.load(open(os.path.join(modeldir, fname), 'rb')) cv_clf = cv_clf.best_estimator_ fold_y_pred = cv_clf.predict(x_test) fold_y_pred_prob = cv_clf.predict_proba(x_test) with multi_labeled(y_test, fold_y_pred, cv_clf.named_steps['clf'].graph_) \ as (y_test_, y_pred_, graph_, classes_): states = classes_ y_test_ = fill_ancestors(y_test_, graph=graph_) fold_y_pred_ = np.zeros(fold_y_pred_prob.shape) for new_idx, label in enumerate(classes_): old_idx = classes.index(label) fold_y_pred_[:, new_idx] = fold_y_pred_prob[:, old_idx] fold_y_pred = fold_y_pred_ # Accumulate prediction probabilities if fold == 0: y_pred = np.zeros((N, len(states))) y_pred += fold_y_pred # Get average predictions y_pred = y_pred / float(nfolds) if mode == 'hierarchical': y_test = y_test_ else: if mode != 'hierarchical': model_fnames = os.listdir(modeldir) model_fname = [fname for fname in model_fnames if mode in fname][0] scaler, clf = joblib.load( open(os.path.join(modeldir, model_fname), 'rb')) x_test_sc = scaler.transform(x_test) y_pred = clf.predict_proba(x_test_sc) # Save test results y_pred = [(users_test, ts_test, fnames_test, y_test, y_pred)] cv_save_classification_result(y_pred, states, os.path.join( outdir, mode + '_test_classification.csv'), method=method)
def main(argv): infile = argv[0] dataset = argv[1] outdir = argv[2] resultdir = os.path.join(outdir, 'models') if not os.path.exists(resultdir): os.makedirs(resultdir) # Read data file and retain data only corresponding to 5 sleep states df = pd.read_csv(infile, dtype={'label':object, 'user':object,\ 'position':object, 'dataset':object}) states = ['Wake','NREM 1','NREM 2','NREM 3','REM','Nonwear'] df = df[df['label'].isin(states)].reset_index() print('... Number of data samples: %d' % len(df)) ctr = Counter(df['label']) for cls in ctr: print('%s: %d (%0.2f%%)' % (cls,ctr[cls],ctr[cls]*100.0/len(df))) feat_cols = ['ENMO_mean','ENMO_std','ENMO_range','ENMO_mad', 'ENMO_entropy1','ENMO_entropy2', 'ENMO_prev30diff', 'ENMO_next30diff', 'ENMO_prev60diff', 'ENMO_next60diff', 'ENMO_prev120diff', 'ENMO_next120diff', 'angz_mean','angz_std','angz_range','angz_mad', 'angz_entropy1','angz_entropy2', 'angz_prev30diff', 'angz_next30diff', 'angz_prev60diff', 'angz_next60diff', 'angz_prev120diff', 'angz_next120diff', 'LIDS_mean','LIDS_std','LIDS_range','LIDS_mad', 'LIDS_entropy1','LIDS_entropy2', 'LIDS_prev30diff', 'LIDS_next30diff', 'LIDS_prev60diff', 'LIDS_next60diff', 'LIDS_prev120diff', 'LIDS_next120diff'] ts = df['timestamp'] X = df[feat_cols].values y = df['label'] #y = np.array([states.index(i) for i in y]) groups = df['user'] fnames = df['filename'] feat_len = X.shape[1] # Class hierarchy for sleep stages class_hierarchy = { ROOT : {"Wear", "Nonwear"}, "Wear" : {"Wake", "Sleep"}, "Sleep" : {"NREM", "REM"}, "NREM" : {"Light", "NREM 3"}, "Light" : {"NREM 1", "NREM 2"} } graph = DiGraph(class_hierarchy) classes = [node for node in graph.nodes if node != ROOT] outer_cv_splits = 5; inner_cv_splits = 5 factor = 10.0 # Outer CV group_kfold = GroupKFold(n_splits=outer_cv_splits) out_fold = 0 hierarchical_pred = [] for train_indices, test_indices in group_kfold.split(X,y,groups): out_fold += 1 print('Processing fold ' + str(out_fold)) out_fold_X_train = X[train_indices,:]; out_fold_X_test = X[test_indices,:] out_fold_y_train = y[train_indices]; out_fold_y_test = y[test_indices] out_fold_users_test = groups[test_indices] out_fold_ts_test = ts[test_indices] out_fold_fnames_test = fnames[test_indices] # Create a pipeline with scaler and hierarchical classifier pipe = Pipeline([('scaler', StandardScaler()), ('clf', HierarchicalClassifier( base_estimator=RandomForestClassifier(random_state=0, n_estimators=100, n_jobs=-1), class_hierarchy=class_hierarchy, prediction_depth='mlnp', progress_wrapper=tqdm, #stopping_criteria=0.7 )) ]) # Inner CV strat_kfold = StratifiedKFold(n_splits=inner_cv_splits,\ random_state=0, shuffle=True) custom_cv_indices = [] for grp_train_idx, grp_test_idx in strat_kfold.split(out_fold_X_train,out_fold_y_train): custom_cv_indices.append((grp_train_idx, grp_test_idx)) print('Training') search_params = {'clf__base_estimator__n_estimators':[50,100,200,300,500,700], \ 'clf__base_estimator__max_depth': [5,10,15,None]} cv_clf = RandomizedSearchCV(estimator=pipe, param_distributions=search_params, \ cv=custom_cv_indices, scoring=make_scorer(custom_h_fbeta,graph=graph), n_iter=5, \ n_jobs=-1, verbose=1) cv_clf.fit(out_fold_X_train, out_fold_y_train) joblib.dump(cv_clf, os.path.join(resultdir,\ 'fold'+str(out_fold)+'_hierarchical_RF.sav')) print('Predicting') out_fold_y_pred = cv_clf.predict(out_fold_X_test) out_fold_y_pred_prob = cv_clf.predict_proba(out_fold_X_test) best_clf = cv_clf.best_estimator_ # Demonstrate using our hierarchical metrics module with MLB wrapper with multi_labeled(out_fold_y_test, out_fold_y_pred, best_clf.named_steps['clf'].graph_) \ as (y_test_, y_pred_, graph_, classes_): states = classes_ y_test_ = fill_ancestors(y_test_, graph=graph_) y_pred_ = fill_ancestors(y_pred_, graph=graph_) y_pred_prob_ = np.zeros(out_fold_y_pred_prob.shape) for new_idx, label in enumerate(classes_): old_idx = classes.index(label) y_pred_prob_[:,new_idx] = out_fold_y_pred_prob[:,old_idx] hierarchical_pred.append((out_fold_users_test, out_fold_ts_test, out_fold_fnames_test, y_test_, y_pred_prob_)) cv_save_classification_result(hierarchical_pred, states, os.path.join(outdir, 'hierarchical_classification_results.csv'), method = 'hierarchical')