def _model_build(self, *arg): self._prepare_test_data() model = KerasClassifier( build_fn=self.create_model, verbose=0) optimizers = [ 'adam'] init = [ 'normal', 'uniform'] epochs = [ 100, 150] batches = [ 5, 10] param_grid = dict( optimizer=optimizers, nb_epoch=epochs, batch_size=batches, init=init) grid = GridSearchCV( estimator=model, param_grid=param_grid, cv=5) grid_result = grid.fit( self.x_train, self.y_train) print("Best: %f using %s" % ( grid_result.best_score_, grid_result.best_params_)) # means = grid_result.cv_results_[ # 'mean_test_score'] # stds = grid_result.cv_results_[ # 'std_test_score'] # params = grid_result.cv_results_[ # 'params'] # for mean, stdev, param in zip(means, stds, params): # print("%f (%f) with: %r" % ( # mean, # stdev, # param)) # Training # with Best # Parameter model = Sequential() model.add(Dense( 12, input_dim=8, init=grid_result.best_params_['init'], activation='relu')) model.add(Dense( 8, init=grid_result.best_params_['init'], activation='relu')) model.add(Dense( 1, init=grid_result.best_params_['init'], activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=grid_result.best_params_['optimizer'], metrics=['accuracy']) # Compile # model model.fit( self.x_train, self.y_train, nb_epoch=grid_result.best_params_['nb_epoch'], batch_size=grid_result.best_params_['batch_size']) yy_pred = model.predict( self.x_test) self.y_pred = [np.round( x) for x in yy_pred] self.y_true = self.y_test self.prob = model.predict_proba( self.x_test) self._analyse_result()
print("%f (%f) with: %r" % (mean, stdev, param)) # # # RUN MODEL # # model = Sequential() model.add(Dense(20, activation='relu', input_dim=len(elo_data.columns) - 1)) model.add(Dropout(0.5)) model.add(Dense(1, activation='sigmoid')) model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy']) model.fit(X_train_elo, Y_train_elo, epochs=10, batch_size=50, validation_split=0.2, verbose=1) model.test_on_batch(X_test_elo, Y_test_elo, sample_weight=None) model.evaluate(X_test_elo, Y_test_elo, verbose=1) pred = model.predict_classes(X_test_elo, verbose=1) plot_model(model, to_file='model.png', show_shapes=True) SVG(model_to_dot(model).create(prog='dot', format='svg'))
print("-------------------------------------------------------------") print("Best Parameters selected:",grid_result.best_params_) """**So we have got the best hyper paramters that we will use to build our final model**""" model=keras.Sequential() # Adding the input layer and the first hidden layer model.add(Dense(128,activation='relu',kernel_initializer = 'he_uniform',input_dim=784)) # Adding the second hidden layer model.add(Dense(units = 64, kernel_initializer = 'he_uniform',activation='relu')) # Adding the output layer model.add(Dense(units= 10, kernel_initializer = 'glorot_uniform', activation = 'softmax')) # Compiling the ANN model.compile(optimizer = 'Adamax', loss = 'categorical_crossentropy', metrics = ['accuracy']) from keras.utils import to_categorical # Fitting the ANN to the Training set model_history=model.fit(train_images, to_categorical(train_labels) ,batch_size = 10, epochs = 20) """We have got an accuracy of **97.19** on our training dataset""" # summarize history for loss from matplotlib import pyplot as plt plt.plot(model_history.history['loss']) plt.title('model loss')
if resp.lower().startswith("s"): print("- CONTINUANDO Programa") valid = True elif resp.lower().startswith("n"): valid = True print("- FINALIZANDO Programa") exit() classifier = build_classifier() print(("- COMPILANDO Rede neural artificial (epochs = {0} - batch_size = {1})" ).format(epoch, batch_size)) print( "- DEFININDO Compilador: optimizer = adam, loss = categorical_crossentropy, metrics = [accuracy]" ) classifier.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) print("- TREINANDO Rede neural artificial (fitting)") classifier.fit(X_train, Y_train, batch_size=batch_size, epochs=epoch) # ---------------------- PREDICTION ---------------------- print("\n- COMECANDO Fase de testes (previsoes)") def prepare4pred(X): Y = np.array([]) for line in X: if line[0] > line[1] and line[0] > line[2]: Y = np.append(Y, 0) elif line[1] > line[0] and line[1] > line[2]: Y = np.append(Y, 1)
x = layers.Dense(300, activation='relu')(inputs) x = layers.Dropout(0.3)(x) x = layers.BatchNormalization()(x) x = layers.Dense(300, activation='relu')(x) x = layers.Dropout(0.3)(x) x = layers.BatchNormalization()(x) y = layers.Dense(1, activation='sigmoid')(x) model = Model(inputs=inputs, outputs=y) return model model = create_model_1() learning_rate = 1e-3 label_smoothing = 0.0 model.compile(optimizer=tfa.optimizers.SWA( tf.keras.optimizers.Adam(learning_rate=learning_rate)), loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=['accuracy']) es = callbacks.EarlyStopping(monitor='val_accuracy', min_delta=0.000001, patience=10, verbose=Verbose, mode='max', baseline=None, restore_best_weights=True) sb = callbacks.ModelCheckpoint('./nn_model.w8', save_weights_only=True, save_best_only=True, verbose=Verbose, monitor='val_accuracy', mode='max') plateau = callbacks.ReduceLROnPlateau(monitor='val_accuracy',
grid.fit(sequence_matrix, y_train) print( grid.best_score_, grid.best_params_) #returns the best cross validation score and parameters # gave the result 0.9844460397224216 {'batch_size': 128, 'epochs': 4} #train the model with the best epoch and and batch size inputs = Input(name='inputs', shape=[maximum_length]) layer = Embedding(maximum_word, 50, input_length=maximum_length)(inputs) layer = LSTM(64)(layer) layer = Dense(units=256, activation='relu')(layer) layer = Dropout(0.5)(layer) layer = Dense(units=1, activation='sigmoid')(layer) model = Model(inputs=inputs, outputs=layer) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) #run the model with batch_size:128 and epochs 4 model.fit(sequence_matrix, y_train, batch_size=128, epochs=4) #test performance on test set test_sequences = token.texts_to_sequences(x_test) test_sequences_matrix = sequence.pad_sequences(test_sequences, maxlen=maximum_length) loss_and_accuracy = model.evaluate(test_sequences_matrix, y_test) print(loss_and_accuracy)
# model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=1, validation_data=(x_test, y_test)) # score = model.evaluate(x_test, y_test, verbose=0) # print('Test loss: ', score[0]) # print('Test accuracy: ', score[1]) # 将定义好的model传入sklearn进行cross_validation model = KerasClassifier(build_fn=create_model, nb_epoch=20, batch_size=128) kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=41) # results = cross_val_score(model, X, Y, cv=kfold, scoring='accuracy') scores_history = [] for train_ind, test_ind in kfold.split(X, Y0): model = Sequential() model.add(Dense(units=512, input_dim=784, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(units=512, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(units=num_classes, activation='softmax')) model.summary() model.compile(optimizer=RMSprop(), loss='categorical_crossentropy', metrics=['accuracy']) model.fit(X[train_ind], Y[train_ind], epochs=epochs, batch_size=batch_size, verbose=0) scores = model.evaluate(X[test_ind], Y[test_ind], verbose=0) scores_history.append(scores)
seed = 7 np.random.seed(seed) classifier = Sequential() classifier.add( Dense(units=66, kernel_initializer='uniform', activation='relu', input_dim=22)) classifier.add(Dropout(rate=0.1)) classifier.add(Dense(units=44, kernel_initializer='uniform', activation='relu')) classifier.add(Dropout(rate=0.3)) classifier.add( Dense(units=1, kernel_initializer='uniform', activation='sigmoid')) classifier.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) hist = classifier.fit(X_train, y_train, batch_size=1, epochs=100) y_pred = classifier.predict(X_test) y_pred = (y_pred >= 0.5) cm = confusion_matrix(y_test, y_pred) cm ########################### # Tuning the ANN (GridSearch) from keras.wrappers.scikit_learn import KerasClassifier from sklearn.model_selection import GridSearchCV from time import gmtime, strftime
model.add(layers.Conv2D(32, (3, 3), activation='relu', padding='same', input_shape=(100, 800, 3))) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(64, (3, 3), activation='relu', padding='same')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(128, (3, 3), activation='relu', padding='same')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Conv2D(128, (3, 3), activation='relu', padding='same')) model.add(layers.MaxPooling2D((2, 2))) model.add(layers.Flatten()) model.add(layers.Dense(512, activation='relu')) model.add(layers.Dense(3, activation='softmax')) model.summary() model.compile(loss='categorical_crossentropy', optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc']) history = model.fit(X_train, y_train, epochs=35, batch_size=150, validation_data=(X_val, y_val)) model.save('best_model_imag_3clases.h5') Y_pred_proba=model.predict(X_test) Y_pred=model.predict_classes(X_test) test_loss_trained_net, test_acc_trained_net = model.evaluate(X_test, y_test) print('test_acc:', test_acc_trained_net) #Resultado 75% de acuraccy sobre el test
## Adding the second hidden layer classifier.add(Dense(output_dim = 16, init = 'uniform', activation = 'relu')) classifier.add(Dense(output_dim = 16, init = 'uniform', activation = 'relu')) classifier.add(Dense(output_dim = 16, init = 'uniform', activation = 'relu')) ## Adding the output layer classifier.add(Dense(output_dim = 1, init = 'uniform', activation = 'tanh')) ## only one node(yes/no) and activation function changed to sigmoid ## if output variable is encoded then dim=3, activation = softmax ## Compiling the Artificial Neural Network - finding the best weight values with stochastic approach classifier.compile(optimizer = 'nadam', loss = 'binary_crossentropy', metrics = ['accuracy'] ) ## logarithmic loss ## Fitting the ANN to the Training set. Two additional arguments - batch size & number of epochs classifier.fit(X_train, y_train, batch_size=25, epochs=180) y_pred = classifier.predict(X_test) y_pred = (y_pred > 0.5) # Making the Confusion Matrix from sklearn.metrics import confusion_matrix cm = confusion_matrix(y_test, y_pred)
class NeuralNetwork: def __init__(self, ds: Dataset): self.ds = ds def create_model(self, typ): clear_session() if typ == 'Sequential': self.model = Sequential() def add_final_layer(self): if self.ds.y_train.nunique() == 2: self.model.add(layers.Dense(1, activation='sigmoid')) self.model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) def sequential(self, layers, activation='relu'): """Create a sequential Keras model""" self.create_model('Sequential') self.add_dense(layers, activation) self.add_final_layer() def add_dense(self, filters, activation='relu'): for x in filters: self.model.add(layers.Dense(x, activation=activation)) def add_embedding_layer(self): if self.ds.weights is not None: self.model.add( layers.Embedding(input_dim=self.ds.vocab_size, output_dim=self.ds.weights.shape[1], input_length=self.ds.X_train.shape[1], weights=[self.ds.weights])) else: self.model.add( layers.Embedding(input_dim=self.ds.vocab_size, output_dim=self.ds.weights.shape[1], input_length=self.ds.X_train.shape[1])) def embedding_to_sequential(self, filters, activation='relu'): self.create_model('Sequential') self.add_embedding_layer() self.model.add(layers.GlobalMaxPool1D()) self.add_dense(filters) self.add_final_layer() def cnn(self, cnn_filters, kernel_sizes, dense_filters, activation='relu'): self.create_model('Sequential') self.add_embedding_layer() self.add_cnn(cnn_filters, kernel_sizes) self.model.add(layers.GlobalMaxPool1D) self.add_dense(dense_filters) self.add_final_layer() def grid_search(self, param_grid, **kwargs): #kwargs: epoch=10, verbose=False #not sure if this works, might need to return model if 'kernel_size' in param_grid: self.model = KerasClassifier(build_fn=self.cnn, **kwargs) elif 'filters' in param_grid: self.model = KerasClassifier(build_fn=self.embedding_to_sequential, **kwargs) else: self.model = KerasClassifier(build_fn=self.sequential, **kwargs) grid = RandomizedSearchCV(estimator=self.model, param_distributions=param_grid, verbose=1) grid.fit(self.ds.X_train, self.ds.y_train) test_accuracy = grid.score(self.ds.X_test, self.ds.y_test) print(f'Test accuracy for best model is {test_accuracy}') return grid def add_cnn(self, filters, kernels): assert len(filters) == len(kernels) for i in range(len(filters)): self.model.add(layers.Conv1D(filters[i], kernels[i])) def train(self, **kwargs): self.model.fit(self.ds.X_train, self.ds.y_train, **kwargs) def evaluate(self): trainl, traina = self.model.evaluate(self.ds.X_train, self.ds.y_train, verbose=False) vall, vala = self.model.evaluate(self.ds.X_test, self.ds.y_test, verbose=False) print(f'Training loss {trainl}, training accuracy {traina}') print(f'Validation loss {vall}, validation accuracy {vala}') return vall, vala def plot_history(self, path): metrics = ['acc', 'val_acc', 'loss', 'val_loss'] metric_dict = { 'acc': ['Training Accuracy', 'b'], 'val_acc': ['Validation Accuracy', 'r'], 'loss': ['Training Loss', 'b'], 'val_loss': ['Validation Loss', 'r'] } plt.figure(figsize=(12, 5)) plt.subplot(1, 2, 1) plt.title('Training and validation accuracy') plt.legend() for metric in metrics: if 'loss' in metric: plt.subplot(1, 2, 2) met = self.model.history.history[metric] x = range(1, len(met) + 1) plt.plot(x, met, metric_dict[met][1], label=metric_dict[met][0]) plt.title('Training and validation loss') plt.legend() plt.savefig(path)
from sklearn.pipeline import Pipeline from sklearn.preprocessing import MinMaxScaler pipe = Pipeline([('minmax', MinMaxScaler()), ('model', model)]) search = RandomizedSearchCV(estimator=pipe, param_distributions=hyperparameters, n_iter=20, n_jobs=16, cv=5) search.fit(x_train, y_train) print(search.best_params_) print('score :', search.score(x_test, y_test)) # 모델 실행 # 결과 출력 # print('\n Accuracy: %.4f'%(model.evaluate(x, y)[1])) ''' inputs = Input(shape=(8, ), name='input') x = Dense(512, activation='relu', name='hidden1')(inputs) x = Dropout(keep_prob)(x) x = Dense(256, activation='relu', name='hidden2')(x) x = Dropout(keep_prob)(x) x = Dense(128, activation='relu', name='hidden3')(x) x = Dropout(keep_prob)(x) x = Dense(128, activation='relu', name='hidden4')(x) x = Dropout(keep_prob)(x) x = Dense(256, activation='relu', name='hidden5')(x) x = Dropout(keep_prob)(x) x = Dense(512, activation='relu', name='hidden6')(x) x = Dropout(keep_prob)(x) prediction = Dense(1, activation='sigmoid', name='output')(x) model = Model(inputs=inputs, outputs=prediction)
X_train = np.array(X_train) X_test= np.array(X_test) X_val= np.array(X_val) y_train=np.asarray(pd.get_dummies(y_train)) y_val=np.asarray(pd.get_dummies(y_val)) y_test=np.asarray(pd.get_dummies(y_test)) model = models.Sequential() model.add(layers.Dense(256, activation='relu', kernel_regularizer=regularizers.l2(0.01), input_shape=(32,))) model.add(layers.Dense(128, activation='relu', )) model.add(layers.Dense(64, activation='relu',)) model.add(layers.Dense(10, activation='softmax')) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) model.summary() history = model.fit(X_train, y_train, epochs=35, batch_size=120, validation_data=(X_val, y_val)) model.save('best_model_feat_10classes.h5') Y_pred_proba=model.predict(X_test) Y_pred=model.predict_classes(X_test) test_loss_trained_net, test_acc_trained_net = model.evaluate(X_test, y_test)
L = GRU(n_x, activation='tanh', kernel_regularizer=regularizers.l2(0.001))(L) print ('Post GRU is:', L) L = Dense(2, activation='softmax', kernel_regularizer=regularizers.l2(0.001))(L) print('Dense layer is:', L) model = Model(inputs=sequence_input, outputs=L) # Optimization and compile opt = Adam(lr=0.005, beta_1=0.9, beta_2=0.999, decay=0.01) print('Begin compiling...') model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) model.summary() # Begin training model.fit(data_train, Y_train, batch_size=batch_size, epochs=epochs, verbose=2, validation_data=(data_val, Y_val)) score = model.evaluate(data_test, Y_test, batch_size=batch_size) print ('The evaluation is: ', score) # Evaluate testing set test_accuracy = grid.score(X_test, y_test)
# adding Input layer and first hidden layer with dropout classifier.add( Dense(output_dim=6,init= "uniform",activation="relu",input_dim=11)) classifier.add(Dropout(p=0.1)) # adding second hidden layer classifier.add( Dense(output_dim=6,init= "uniform",activation="relu")) classifier.add(Dropout(p=0.1)) # adding output layer #if we have more than 1 output layer we use classifier.add( Dense(output_dim=1,init= "uniform",activation="sigmoid")) #compiling the ann #dependent variable has binary outcome we use loss function as binary_crossentropy #dependentvariable are more than two outcome then we use loss function as categorical_crossentropy classifier.compile(optimizer="adam",loss= "binary_crossentropy", metrics=["accuracy"]) #Tuning the ann from keras.wrappers.scikit_learn import KerasClassifier from sklearn.model_selection import GridSearchCV from keras.models import Sequential from keras.layers import Dense def build_classifier(optimizer): classifier = Sequential() classifier.add(Dense(units = 6, kernel_initializer = 'uniform', activation = 'relu', input_dim = 11)) classifier.add(Dense(units = 6, kernel_initializer = 'uniform', activation = 'relu')) classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid')) classifier.compile(optimizer = optimizer, loss = 'binary_crossentropy', metrics = ['accuracy']) return classifier classifier = KerasClassifier(build_fn = build_classifier)
test_X = X[split_value:, :] test_Y = Y[split_value:] train_X = X[:split_value, :] train_Y = Y[:split_value] opt = Adamax(lr=0.01) opt = Adam(lr=0.01) opt = SGD(lr=0.01, momentum=0.0) model = Sequential() model.add(LSTM(40, return_sequences=True)) model.add(Dropout(0.0)) model.add(Dense(1)) model.add(Flatten()) model.compile(loss='mean_absolute_error', optimizer=opt, metrics=['accuracy']) history = model.fit(train_X, train_Y, epochs=100, batch_size=40, validation_data=(test_X, test_Y), verbose=1) print(history.history.keys()) plt.plot(history.history['acc']) plt.plot(history.history['val_acc']) plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left')
def main(): ######################## ### Parse Input Args ### ######################## parser = argparse.ArgumentParser( description='Predicting Traits Using Convolutional Neural Networks. \ See README.md for more information about the pipeline usage.\n' 'Written by: Emily Bolger\nModified by: Christina Azodi', epilog='https://github.com/ShiuLab') ## Required req_group = parser.add_argument_group(title='REQUIRED INPUT') req_group.add_argument( '-x', help='Feature data. Format options available: ' ' 1) Matrix with one row for every instance. 2) Directory with one ' 'matrix for every instance with file name matching instance name ' 'provided in -y', required=True) req_group.add_argument('-y', help='Matrix with label to predict', required=True) req_group.add_argument('-test', help='List of instances to use as test set', required=True) req_group.add_argument('-save', help='Name for Output File', required=True) ## Optional inp_group = parser.add_argument_group(title='OPTIONAL INPUT') inp_group.add_argument('-feat', help="List of column names in -x to use", default=False) inp_group.add_argument( '-x_sort', help='Method to sort feature (-x) data ' 'by column. Options: (False, alpha, file_with_order, cluster)', default=False) inp_group.add_argument('-shape', help='Dimension of -x (e.g. for input ' 'with 4 rows and 6 columns: -shape 4,6)', default='default') inp_group.add_argument('-onehot', help='T/F to convert 1xX data into matrix' ' by one-hot encoding', default='F') inp_group.add_argument('-onehot_order', help='Order for 1-hot y axis. For ' 'example: -onehot_order 1,0,-1)', default=False) inp_group.add_argument('-y_name', help='Col name to predict', default='Y') inp_group.add_argument('-sep', help='Deliminator for X & Y', default='\t') # How to run CNN inp_group.add_argument('-run', help='T/F to run the final model. If F, will' 'only run the grid search if -gs T', default='f') inp_group.add_argument('-n', help='Num replicates of model... Different' 'train/validation split each replicate', type=int, default=10) inp_group.add_argument('-n_jobs', '-p', help='Number of processors for ' 'parallel computing (max for HPCC = 14)', type=int, default=1) inp_group.add_argument('-cv', help='Number of cross validation folds', type=int, default=5) # Parameter Selection inp_group.add_argument( '-params', help='How to select parameters. Options: ' 'grid search (gs), default, from XXX_GridSearch.txt (provide path).', default='default') inp_group.add_argument('-gs_reps', help='Number of combinations of ' 'parameters to test in the grid search', type=int, default=100) # Default CNN parameters inp_group.add_argument('-cnn_type', help='CNN architecture. Options: ' '(simple, DeepGS)', default="simple") inp_group.add_argument('-filters', help='Number of kernels/filters in each ' 'CNN layer', type=int, default=32) inp_group.add_argument('-kernel_l', help='Length of kernel (height ' 'defaults to the full height of the dataset)', type=int, default=16) inp_group.add_argument('-stride_len', help='Stride of Convolution kernels ' '(width defaults to 1)', type=int, default=1) inp_group.add_argument('-activation', help='Activation function in all but ' 'last dense layer, which is set to linear', type=str, default='relu') inp_group.add_argument('-pool_size', help='Size of max pooling layer filter ' '(first number only, second defaults to 1)', type=int, default=8) inp_group.add_argument('-optimizer', help='Optimization function to use)', type=str, default='Adam') inp_group.add_argument('-dropout', help='Value for Dropout Rate', type=float, default=0.5) inp_group.add_argument('-l2', help='Value for L2 regularization', type=float, default=0.2) inp_group.add_argument('-learn_rate', help='Value for Learning Rate', type=float, default=0.01) inp_group.add_argument('-clip_value', help='Clip Value', type=float, default=0.5) inp_group.add_argument('-patience', help='Patience for Early Stopping', type=int, default=10) inp_group.add_argument('-min_delta', help='Minimum Delta Value for Early ' 'Stopping', type=float, default=0) inp_group.add_argument('-num_epochs', help='Max number of Epochs', type=int, default=1000) inp_group.add_argument('-n_channels', help='Num channels', type=int, default=1) # Argument parsing args = parser.parse_args() if args.shape == 'default': tmp = pd.read_csv(args.x, sep=args.sep, index_col=0) shape_r, shape_c = 1, tmp.shape[1] else: shape_r, shape_c = args.shape.strip().split(',') shape_r = int(shape_r) shape_c = int(shape_c) ######################## ### Parse Input Data ### ######################## print("\n***** Loading Data ******\n") # Step 1: Read in x file, if feat file given only keep those features if os.path.isfile(args.x): x = pd.read_csv(args.x, sep=args.sep, index_col=0) x.index = x.index.astype('str') instance_order = list(x.index.values) with open(args.test) as test_file: test_instances = test_file.read().splitlines() test_instances = [str(i) for i in test_instances] train_val_instances = list( set(instance_order) - set(test_instances)) test_index = [x.index.get_loc(i) for i in test_instances] train_val_index = [x.index.get_loc(i) for i in train_val_instances] if args.feat: with open(args.feat) as f: features = f.read().strip().splitlines() x = x.loc[:, features] elif os.path.isdir(args.x): x = ANN.fun.Image2Features(args.x, shape_r, shape_c) n_instances = x.shape[0] n_feats = x.shape[1] print("Total number of instances: %i" % n_instances) print("Number of features used: %i" % n_feats) # Step 2: Sort x data if args.x_sort == 'alpha': print('Sorting feature data by column alpha numerically...') x = x.reindex(sorted(x.columns), axis=1) elif args.x_sort == 'cluster': print('Sorting feature data by column using clustering...') print('\n\nNOT IMPLEMENTED YET... PROGRESSING WITHOUT SORTING...\n\n') else: if not args.x_sort: print('Using -x in the order provided in -x or in -feat') else: with open(args.x_sort) as order: order_list = order.read().strip().splitlines() x = x.loc[:, order_list] print('\nSnapshot of input feature data:') print(x.head()) # Step 3: One-hot-encode X if required if args.onehot.lower() in ['t', 'true']: x_1hot_list = [] x = x.round(0) labels = pd.unique(x.values.ravel()) ohe = preprocessing.OneHotEncoder(categories='auto', sparse=False) for i in range(len(x)): x_row = np.array(x.iloc[i, ]).reshape(n_feats, -1) oh_matrix = ohe.fit_transform(x_row) if oh_matrix.shape[1] < len(labels): labels_present = pd.unique(x_row.ravel()) missing = list(set(labels) - set(labels_present)) print( "Instance in row %i is has no '%s', so adding by hand..." % (i, missing)) x_row = np.append(x_row, np.array([missing]), axis=0) oh_matrix = ohe.fit_transform(x_row) oh_matrix = oh_matrix[:-1, :] x_1hot_list.append(oh_matrix) x = np.swapaxes(np.array(x_1hot_list), 1, 2) data_height = x.shape[1] data_width = x.shape[2] x = x.reshape((n_instances, data_height, data_width, args.n_channels)) print("\nShape of feature data used for training/validation/testing:") print(x.shape) print("\nSnapshot of feature data for first instance in data set:") print(x[0, :, :, 0]) # Step 4: Read in Y data and make sure sorted as in -x y = pd.read_csv(args.y, sep=args.sep, index_col=0) y.index = y.index.astype('str') y = y[[args.y_name]] print("\nShape of Label Data:") print(y.shape) # Step 5: Remove testing data x_test = x[test_index, :, :, :] x_train = x[train_val_index, :, :, :] y_test = y.ix[test_instances] y_train = y.ix[train_val_instances] ################################ ### Define CNN architectures ### ################################ def tfp_pearson(y_true, y_pred): return tfp.stats.correlation(y_pred, y_true, event_axis=None) def make_cnn_model(cnn_type=args.cnn_type, learn_rate=args.learn_rate, filters=args.filters, pool_size=args.pool_size, kernel_l=args.kernel_l, kernel_h=data_height, activation=args.activation, optimizer=args.optimizer, units=1): if optimizer.lower() == 'adam': opt = tf.keras.optimizers.Adam(lr=learn_rate, clipvalue=args.clip_value) elif optimizer.lower() == 'nadam': opt = tf.keras.optimizers.Nadam(lr=learn_rate, clipvalue=args.clip_value) elif optimizer.lower() == 'rmsprop': opt = tf.keras.optimizers.RMSprop(lr=learn_rate, clipvalue=args.clip_value) if cnn_type.lower() == 'simple': K.clear_session() model = models.Sequential() model.add( layers.Conv2D( filters=filters, kernel_size=tuple([kernel_h, kernel_l]), kernel_regularizer=tf.keras.regularizers.l2(args.l2), strides=tuple([args.stride_len, 1]), activation=activation, kernel_initializer='glorot_normal', input_shape=(data_height, data_width, args.n_channels))) model.add(layers.MaxPooling2D(pool_size=tuple([1, pool_size]))) model.add(layers.Flatten()) model.add(layers.Dropout(args.dropout)) model.add(layers.Dense(24, activation=activation)) model.add(layers.BatchNormalization()) model.add(layers.Dense(units=units, activation='linear')) model.compile(optimizer=opt, loss='mean_squared_error') elif cnn_type.lower() == 'deepgs': K.clear_session() model = models.Sequential() model.add( layers.Conv2D(filters=filters, kernel_size=tuple([kernel_h, kernel_l]), strides=tuple([args.stride_len, 1]), activation=activation, kernel_initializer='glorot_normal', input_shape=(data_height, data_width, args.n_channels))) model.add(layers.MaxPooling2D(pool_size=tuple([1, pool_size]))) model.add( layers.Conv2D(filters=filters, kernel_size=tuple([1, kernel_l]), strides=tuple([args.stride_len, 1]), activation=activation)) model.add(layers.MaxPooling2D(pool_size=tuple([1, pool_size]))) model.add(layers.Dropout(args.dropout)) model.add(layers.Flatten()) model.add(layers.Dense(units=24, activation=activation)) model.add(layers.BatchNormalization()) model.add(layers.Dropout(args.dropout)) model.add(layers.Dense(units=units, activation='linear')) model.compile(optimizer=opt, loss='mean_squared_error') return model #################### ### Grid Search ### #################### if args.params.lower() == 'gs': print('\n***** Starting Random Search with %i reps using %i testing ' 'instances and %i fold cross-validation *****\n' % (args.gs_reps, x_train.shape[0], args.cv)) scoring = { 'neg_mse': 'neg_mean_squared_error', 'exp_var': 'explained_variance' } param_grid = dict(learn_rate=[1, 0.1, 0.01, 0.001, 0.0001, 0.00001], filters=[8, 16, 32], kernel_l=[8, 16, 32], pool_size=[4, 8, 16], activation=["relu", "selu", "elu"], optimizer=['RMSprop', 'Adam', 'nadam'], cnn_type=['simple', 'deepgs']) model = KerasClassifier(build_fn=make_cnn_model, batch_size=100, epochs=50, verbose=1) rand_search = RandomizedSearchCV(estimator=model, param_distributions=param_grid, cv=args.cv, n_iter=args.gs_reps, n_jobs=args.n_jobs, verbose=1, scoring=scoring, refit='neg_mse') gs_result = rand_search.fit(x_train, y_train) gs_result_df = pd.DataFrame.from_dict(gs_result.cv_results_) print("Saving Grid Search Results....") print(gs_result_df.head()) with open(args.save + "_GridSearch.txt", 'a') as out_gs: gs_result_df.to_csv(out_gs, header=out_gs.tell() == 0, sep='\t') ######################## ### Run Final Models ### ######################## if args.run.lower() in ['t', 'true']: # Step 1: Define the parameters from the Grid Search or use default if args.params.lower() != 'default': if args.params.lower() != 'gs': gs_result_df = pd.read_csv(args.params, sep='\t') gs_result_df.fillna(0, inplace=True) gs_mean = gs_result_df.groupby([ 'param_filters', 'param_optimizer', 'param_learn_rate', 'param_kernel_l', 'param_pool_size', 'param_cnn_type', 'param_activation' ]).agg({ 'mean_test_score': 'mean', 'std_test_score': 'mean' }).reset_index() gs_mean = gs_mean.sort_values(by='mean_test_score', ascending=False) print('\nSnapshot of grid search results:') print(gs_mean.head()) args.cnn_type = gs_mean['param_cnn_type'].iloc[0] args.pool_size = int(gs_mean['param_pool_size'].iloc[0]) args.learn_rate = float(gs_mean['param_learn_rate'].iloc[0]) args.kernel_l = int(gs_mean['param_kernel_l'].iloc[0]) args.filters = int(gs_mean['param_filters'].iloc[0]) args.activation = gs_mean['param_activation'].iloc[0] args.optimizer = gs_mean['param_optimizer'].iloc[0] print('\n***** Running CNN models ******') print('CNN Architecture: %s\nOptimizer: %s\nActivation function:' ' %s\nLearning Rate: %f\nNumber of kernels: ' '%i\nKernel shape: [%i, %i]\nPooling Size: [%i, 1]\n' % (args.cnn_type, args.optimizer, args.activation, args.learn_rate, args.filters, args.kernel_l, data_height, args.pool_size)) for i in range(args.n): print('Rep %i of %i' % (i, args.n)) run = True while run: # Step 2: Creating CNN model using Tensorflow model = make_cnn_model(cnn_type=args.cnn_type, learn_rate=args.learn_rate, optimizer=args.optimizer, filters=args.filters, pool_size=args.pool_size, kernel_l=args.kernel_l, kernel_h=data_height, activation=args.activation, units=1) #print(model.summary()) # Step 3: Split training into training2 and validation x_train2, x_val, y_train2, y_val = train_test_split( x_train, y_train, test_size=0.2) print('Train on %i, validate on %i, test on %i' % (x_train2.shape[0], x_val.shape[0], x_test.shape[0])) # Step 4: Define optimizer and early stopping criteria & train model.compile(optimizer=args.optimizer, loss='mean_squared_error', metrics=[tfp_pearson]) earlystop_callback = EarlyStopping(monitor='val_loss', mode='min', min_delta=args.min_delta, patience=args.patience, restore_best_weights=True, verbose=1) model.fit(x_train2, y_train2, batch_size=100, epochs=args.num_epochs, verbose=1, callbacks=[earlystop_callback], validation_data=(x_val, y_val)) # Step 5: Apply best model to train, val, test, and report results train_mse, train_pcc = model.evaluate(x_train2, y_train2) val_mse, val_pcc = model.evaluate(x_val, y_val) if val_pcc > 0: run = False else: print('\nPCC was negative on valid data.. retraining...') test_mse, test_pcc = model.evaluate(x_test, y_test) if np.isnan(test_pcc): # Still don't know why this happens, but this fixes it... print('Recalculating PCC using Numpy...') pred = model.predict(x_test).tolist() pred2 = [i for sublist in pred for i in sublist] test_pcc = np.corrcoef(pred2, y_test[args.y_name].values)[0, 1] print('PCC: train, val, and test: %3f, %3f, %3f' % (train_pcc, val_pcc, test_pcc)) if not os.path.isfile('RESULTS.txt'): out = open('RESULTS.txt', 'w') out.write( 'ID\tX\tY\ty_name\ttest_set\t' 'CNN_Type\tLearn_Rate\tMin_Delta\tPatience\tActivation\t' 'Optimizer\tKernel_num\tKernel_len\tPooling_Size\tDropout' '\tTrain_mse\tTrain_PCC\tVal_mse\tVal_PCC\tTest_mse\t' 'Test_PCC\n') out.close() out = open('RESULTS.txt', "a") out.write( '%s\t%s\t%s\t%s\t%s\t' '%s\t%f\t%f\t%i\t%s\t%s\t' '%i\t%i\t%i\t%f\t%f\t' '%f\t%f\t%f\t%f\t%f\n' % (args.save, args.x, args.y, args.y_name, args.test, args.cnn_type, args.learn_rate, args.min_delta, args.patience, args.activation, args.optimizer, args.filters, args.kernel_l, args.pool_size, args.dropout, train_mse, train_pcc, val_mse, val_pcc, test_mse, test_pcc)) out.close() print('\nDone!')
def main(): ######################## ### Parse Input Args ### ######################## parser = argparse.ArgumentParser( description='CNN classification code implemented using TensorFlow v2.0', epilog='https://github.com/azodichr') parser.add_argument('-x', help='Feature numpy dataset', required=True) parser.add_argument('-y', help='Class/Y numpy dataset', required=True) parser.add_argument('-run', help='T/F to run final models', default='t') parser.add_argument('-splits', help='Values for train/val/test', default='70,10,20') parser.add_argument('-y_name', help='Phenotype Trait') parser.add_argument('-f', help='Function: gs, run, full', default='full') parser.add_argument('-save', help='Name for Output File', default='test') parser.add_argument('-balance', help='t/f to downsample so balance classes', default='t') parser.add_argument('-n_channels', help='Number of channels', default=1, type=int) parser.add_argument('-cv', help='Number of cross validation folds', type=int, default=5) parser.add_argument('-n_jobs', '-p', help='Number of processors for ' 'parallel computing (max for HPCC = 14)', type=int, default=1) parser.add_argument('-save_model', help='T/F if want to save final models', type=str, default='f') parser.add_argument('-tag', help='Identifier String to add to RESULTS', type=str, default='cnn') parser.add_argument('-save_detailed', help='T/F Save detailed model performance', type=str, default='f') parser.add_argument('-original_df', help='DF fed into input_converter.py', type=str, default='') parser.add_argument('-imp_m', help='T/F to calculate importance of each motif', type=str, default='f') parser.add_argument('-imp_k', help='T/F to calculate importance of each kernel', type=str, default='f') # Default Hyperparameters parser.add_argument('-params', help='Output from -f gs (i.e. ' 'SAVE_GridSearch.txt)', default='default') parser.add_argument('-actfun', help='Activation function. (relu, sigmoid)', default='relu') parser.add_argument('-learn_rate', help='Learning Rate', default=0.01, type=float) parser.add_argument('-dropout', help='Dropout rate', default=0.25, type=float) parser.add_argument('-l2', help='Shrinkage parameter for L2 regularization', default=0.25, type=float) parser.add_argument('-filters', help='Number of Kernels/filters', default=8, type=int) parser.add_argument('-optimizer', help='Optimization function to use)', type=str, default='Adam') parser.add_argument('-dense', help='Number of nodes in dense layer', type=int, default=16) parser.add_argument('-activation', help='Activation function in all but ' 'last dense layer, which is set to linear', type=str, default='relu') parser.add_argument('-n_reps', '-n', help='Number of replicates (unique ' 'validation set/starting weights for each)', default=100, type=int) parser.add_argument('-clip_value', help='Clip Value', type=float, default=0.5) parser.add_argument('-patience', help='Patience for Early Stopping', type=int, default=5) parser.add_argument('-min_delta', help='Minimum Delta Value for Early ' 'Stopping', type=float, default=0) # Grid Search reps/space parser.add_argument('-gs_reps', '-gs_n', help='Number of Grid Search Reps' '(will append results if SAVE_GridSearch.csv exists)', type=int, default=10) parser.add_argument('-actfun_gs', help='Activation functions for Grid ' 'Search', nargs='*', default=['relu', 'selu', 'elu']) parser.add_argument('-dropout_gs', help='Dropout rates for Grid Search', nargs='*', type=float, default=[0.0, 0.1, 0.25]) parser.add_argument('-l2_gs', help='Shrinkage parameters for L2 for Grid ' 'Search', nargs='*', type=float, default=[0.01, 0.1, 0.25]) parser.add_argument('-lrate_gs', help='Learning Rate', nargs='*', type=float, default=[0.1, 0.01, 0.001, 0.0001]) parser.add_argument('-kernels_gs', help='Number of Kernels for Grid Search', default=[4, 8, 16, 24], type=int) args = parser.parse_args() k_height = 'tmp' args.k_len = 'tmp' def downsample(x, y): unique, counts = np.unique(y_all, return_counts=True) smaller_index = list(counts).index(min(counts)) bigger_index = list(counts).index(max(counts)) i_smaller = np.where(y_all == unique[smaller_index])[0] i_bigger = np.where(y_all == unique[bigger_index])[0] downsample_n = len(i_smaller) i_bigger_downsampled = np.random.choice(i_bigger, size=downsample_n, replace=False) i_keep = list(i_smaller) + list(i_bigger_downsampled) y = y_all[i_keep] x = x_all[i_keep] return x, y def make_cnn_model(learn_rate=args.learn_rate, filters=args.filters, dropout=args.dropout, dense=args.dense, l2=args.l2, activation=args.activation, optimizer=args.optimizer, units=1): if optimizer.lower() == 'adam': opt = tf.keras.optimizers.Adam(lr=learn_rate, clipvalue=args.clip_value) elif optimizer.lower() == 'nadam': opt = tf.keras.optimizers.Nadam(lr=learn_rate, clipvalue=args.clip_value) elif optimizer.lower() == 'rmsprop': opt = tf.keras.optimizers.RMSprop(lr=learn_rate, clipvalue=args.clip_value) elif optimizer.lower() == 'sgdm': opt = tf.keras.optimizers.SGD(lr=learn_rate, decay=1e-6, clipvalue=args.clip_value, momentum=0.9, nesterov=True) conv2d_layer = layers.Conv2D( filters=filters, kernel_size=tuple([k_height, 1]), kernel_regularizer=tf.keras.regularizers.l2(l2), activation=activation, kernel_initializer='glorot_normal', input_shape=(n_rows, n_columns, args.n_channels), name='conv2d_layer') K.clear_session() model = models.Sequential() model.add(conv2d_layer) model.add(layers.Flatten()) model.add(layers.Dense(dense, activation=activation)) model.add(layers.Dropout(dropout)) model.add(layers.Dense(units=1, activation='sigmoid')) model.compile(optimizer=opt, loss='binary_crossentropy') return model, conv2d_layer ########################## ### Data preprocessing ### ########################## x_all = np.load(args.x) y_all = np.load(args.y) x_all = x_all.reshape(x_all.shape + (args.n_channels, )) if args.balance.lower() in ['t', 'true']: x, y = downsample(x_all, y_all) print('Y shape (down-sampled): %s' % str(y.shape)) print('X shape (down-sampled): %s' % str(x.shape)) else: y = y_all x = x_all print("\nSnapshot of feature data for first instance in data set:") print(x[0, :, 0:5, 0]) n = y.shape[0] n_rows = x.shape[1] n_columns = x.shape[2] k_height = x.shape[1] args.k_len = 1 print('Kernel dimensions: ', k_height, args.k_len) ################### ### Grid Search ### ################### if args.params.lower() == 'gs': print('\n***** Starting Random Search with %i reps using %i testing ' 'instances and %i fold cross-validation *****\n' % (args.gs_reps, x.shape[0], args.cv)) scoring = {'acc': 'accuracy', 'f1': 'f1'} param_grid = dict( learn_rate=[0.1, 0.01, 0.001], filters=[8, 16], dense=[8, 16, 32], l2=[0.1, 0.25], #, 0.5], dropout=[0.1, 0.25], #, 0.5], activation=["relu"], #, 'selu', 'elu'], optimizer=['RMSprop', 'Adam', 'nadam']) model, conv2d_layer = KerasClassifier(build_fn=make_cnn_model, batch_size=100, epochs=30, verbose=0) rand_search = RandomizedSearchCV(estimator=model, param_distributions=param_grid, cv=args.cv, n_iter=args.gs_reps, n_jobs=args.n_jobs, scoring=scoring, refit='acc', verbose=0) gs_result = rand_search.fit(x, y) gs_result_df = pd.DataFrame.from_dict(gs_result.cv_results_) print("Saving Grid Search Results....") print(gs_result_df.head()) with open(args.save + "_GridSearch.txt", 'a') as out_gs: gs_result_df.to_csv(out_gs, header=out_gs.tell() == 0, sep='\t') print('\n\n Grid Search results saved to: %s_GridSearch.txt\n' % args.save) ################ ### Run final model ################ if args.run.lower() in ['t', 'true']: print('####### Running Final Model(s) ###########') # Step 1: Define the parameters from the Grid Search or use default if args.params.lower() != 'default': if args.params.lower() != 'gs': gs_result_df = pd.read_csv(args.params, sep='\t') gs_result_df.fillna(0, inplace=True) gs_mean = gs_result_df.groupby([ 'param_filters', 'param_optimizer', 'param_learn_rate', 'param_dropout', 'param_l2', 'param_dense', 'param_activation' ]).agg({ 'mean_test_acc': 'mean', 'std_test_acc': 'mean', 'mean_fit_time': 'count' }).reset_index() print('Parameter Search Coverage: \nMin: %i\nMean: %3f\nMax:%i' % (gs_mean['mean_fit_time'].min(), gs_mean['mean_fit_time'].mean(), gs_mean['mean_fit_time'].max())) if gs_mean['mean_fit_time'].min() == 1: print('Dropping parameter combinations with < 2 replicates...') gs_mean = gs_mean[gs_mean['mean_fit_time'] >= 2] gs_mean = gs_mean.sort_values(by='mean_test_acc', ascending=False) print('\nSnapshot of grid search results:') print(gs_mean.head()) args.learn_rate = float(gs_mean['param_learn_rate'].iloc[0]) args.l2 = float(gs_mean['param_l2'].iloc[0]) args.dropout = float(gs_mean['param_dropout'].iloc[0]) args.filters = int(gs_mean['param_filters'].iloc[0]) args.dense = int(gs_mean['param_dense'].iloc[0]) args.activation = gs_mean['param_activation'].iloc[0] args.optimizer = gs_mean['param_optimizer'].iloc[0] print('\n***** Running CNN models ******') print('Optimizer: %s\nActivation function:' ' %s\nLearning Rate: %4f\nNumber of kernels: ' '%i\nL2: %4f\nDropout: %4f\nDense nodes: %s\n' % (args.optimizer, args.activation, args.learn_rate, args.filters, args.l2, args.dropout, args.dense)) final_results = pd.DataFrame() motif_imps = pd.DataFrame() kern_imp = [] for n in range(args.n_reps): print("\nReplicate %i/%i" % (n, args.n_reps)) x, y = downsample(x_all, y_all) print(x.shape) model, conv2d_layer = make_cnn_model(learn_rate=args.learn_rate, optimizer='sgdm', filters=args.filters, dense=args.dense, l2=args.l2, dropout=args.dropout, activation=args.activation) #print(model.summary()) # Step 3: Split training into training2 and validation x_train, x_test, y_train, y_test = train_test_split(x, y, stratify=y, test_size=0.1) x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, stratify=y_train, test_size=0.111) print('Train on %i, validate on %i, test on %i' % (x_train.shape[0], x_val.shape[0], x_test.shape[0])) # Step 4: Define optimizer and early stopping criteria & train model.compile(optimizer=args.optimizer, loss='binary_crossentropy', metrics=['accuracy']) earlystop_callback = EarlyStopping(monitor='val_loss', mode='min', min_delta=args.min_delta, patience=args.patience, restore_best_weights=True, verbose=0) model.fit(x_train, y_train, batch_size=50, epochs=1000, verbose=0, callbacks=[earlystop_callback], validation_data=(x_val, y_val)) train_loss, train_acc = model.evaluate(x_train, y_train) val_loss, val_acc = model.evaluate(x_val, y_val) test_loss, test_acc = model.evaluate(x_test, y_test) val_yhat = model.predict(x_val) max_f1 = 0 best_thresh = 0 for thr in np.arange(0.01, 1, 0.01): thr_pred = val_yhat.copy() thr_pred[thr_pred >= thr] = 1 thr_pred[thr_pred < thr] = 0 if sum( thr_pred ) > 1: # Eliminates cases where all predictions are negative and the f1 and auROC are undefined f1 = f1_score(y_val, thr_pred, pos_label=1) # Returns F1 for positive class if f1 >= max_f1: max_f1 = f1 best_thresh = thr print('Threshold for F1 measure: %3f' % best_thresh) # Calculate AUC-ROC and F-measure from train, val, and test. yhat_train = model.predict(x_train) train_auroc = roc_auc_score(y_train, yhat_train) yhat_train[yhat_train >= best_thresh] = 1 yhat_train[yhat_train < best_thresh] = 0 train_f1 = f1_score(y_train, yhat_train, pos_label=1) yhat_val = model.predict(x_val) val_auroc = roc_auc_score(y_val, yhat_val) yhat_val[yhat_val >= best_thresh] = 1 yhat_val[yhat_val < best_thresh] = 0 val_f1 = f1_score(y_val, yhat_val, pos_label=1) yhat_test = model.predict(x_test) test_auroc = roc_auc_score(y_test, yhat_test) yhat_test[yhat_test >= best_thresh] = 1 yhat_test[yhat_test < best_thresh] = 0 test_f1 = f1_score(y_test, yhat_test, pos_label=1) if args.save_model.lower() in ['t', 'true']: model.save(args.save + '_model_' + str(n) + '.h5') final_results = final_results.append( { 'ID': args.save, 'Tag': args.tag, 'Rep': n, 'X_file': args.x, 'Y_file': args.y, 'ActFun': args.activation, 'dropout': args.dropout, 'L2': args.l2, 'LearnRate': args.learn_rate, 'Optimizer': args.optimizer, 'n_Kernels': args.filters, 'F1_threshold': best_thresh, 'n_Dense': args.dense, 'Acc_train': train_acc, 'Loss_train': train_loss, 'auROC_train': train_auroc, 'F1_train': train_f1, 'Acc_val': val_acc, 'Loss_val': val_loss, 'auROC_val': val_auroc, 'F1_val': val_f1, 'Acc_test': test_acc, 'Loss_test': test_loss, 'auROC_test': test_auroc, 'F1_test': test_f1 }, ignore_index=True) ########################## ## Model Interpretation ## ########################## if (args.imp_m.lower() in ['t', 'true'] or args.imp_k.lower() in ['t', 'true']): # Step 1: Read in x data meta data key = pd.read_csv( args.original_df, sep='\t', index_col=0, ) key_index_list = key.columns.str.split('_', expand=True).values key.columns = pd.MultiIndex.from_tuples([ (x[1], x[0]) for x in key_index_list ]) key = key.sort_index(axis=1) motifs = key.columns.levels[0].values omic_stack = list(key[list(key.columns.levels[0])[0]]) omic_stack.append('PA') # Calculate Motif importance (zero-out-each-feature) if args.imp_m.lower() in ['t', 'true']: motif_imp = np.empty((0, 2)) model_mot_imp = model for mx in range(0, x_test.shape[2] - 1): x_test_tmp = np.copy(x_test) x_test_tmp[:, ..., mx, :] = 0 yhat_m_imp = model_mot_imp.predict(x_test_tmp) auroc_m_imp = roc_auc_score(y_test, yhat_m_imp) imp_m_auc = test_auroc - auroc_m_imp motif_imp = np.vstack( (motif_imp, np.array([motifs[mx], imp_m_auc]))) motif_imp = pd.DataFrame( motif_imp, columns=['motif', 'auROC_test_decrease']) if n == 0: motif_imps = motif_imp else: motif_imps = pd.merge(motif_imps, motif_imp, on='motif') # Calculate Kernel Importance (zero-out-weights) if args.imp_k.lower() in ['t', 'true']: all_weights = model.get_weights() all_weights_2 = all_weights.copy() print( 'Performing Leave-One-Kernel-Out importance analysis...' ) for kx in range(0, args.filters): orig_weights = all_weights[0][:, :, 0, kx].copy() orig_weights = orig_weights.tolist() orig_weights = [i for l in orig_weights for i in l] conv2d_drop = copy.deepcopy(all_weights) conv2d_drop[0][:, :, 0, kx] = 0.0 print(conv2d_drop[0][1, :, 0, 0:10]) model_LOKO = tf.keras.models.clone_model(model) model_LOKO.set_weights(weights=conv2d_drop) yhat_k_imp = model_LOKO.predict(x_test) auroc_k_imp = roc_auc_score(y_test, yhat_k_imp) imp_k_auc = test_auroc - auroc_k_imp old = roc_auc_score(y_test, model.predict(x_test)) print(old, imp_k_auc) kern_imp.append([n, imp_k_auc, orig_weights]) if args.imp_m.lower() in ['t', 'true']: print('Snapshor ot motif importance scores...') motif_imps = motif_imps.set_index('motif') motif_imps = motif_imps.apply(pd.to_numeric, errors='coerce') motif_imps['mean_imp'] = motif_imps.mean(axis=1) motif_imps = motif_imps.sort_values('mean_imp', 0, ascending=False) print(motif_imps['mean_imp'].head()) motif_imps['mean_imp'].to_csv(args.save + "_Motif_imp", sep="\t", index=True) if args.imp_k.lower() in ['t', 'true']: print('\nSnapshot of kernel importance scores:') kern_imp = pd.DataFrame( kern_imp, columns=['rep', 'auROC_test_decrease', 'kernel']) print(kern_imp.head()) kern_imp.to_csv(args.save + "_Kernel_imp", sep="\t", index=True) final_results.to_csv(args.save + "_results.txt", header=True, sep='\t') # Save summary of results to RESULTS.txt calc_cols = [ 'F1_threshold', 'Acc_train', 'Acc_val', 'Acc_test', 'Loss_train', 'Loss_val', 'Loss_test', 'auROC_train', 'auROC_val', 'auROC_test', 'F1_train', 'F1_val', 'F1_test' ] final_results = final_results.drop(['Rep'], axis=1) std = final_results[calc_cols].std(axis=0, skipna=True) std = std.add_suffix('_std') mean = final_results[calc_cols].mean(axis=0, skipna=True) mean = mean.add_suffix('_mean') str_cols = final_results.drop(calc_cols, axis=1).iloc[0] str_cols = str_cols.append(pd.Series([args.n_reps], index=['Reps'])) summary = pd.concat([str_cols, mean, std]) #summary.set_index('index', inplace=True) print('\n### Summary of results on test set ###') print(summary.filter(like='test_mean', axis=0)) with open("RESULTS.txt", 'a') as f: summary.to_frame().transpose().to_csv(f, header=f.tell() == 0, sep='\t') print('Done!')
best = grid_result.best_params_ model = Sequential() model.add(Dense(best['neurons'], input_dim=1000, activation=best['activation'])) model.add(Dropout(best['dropout_rate'])) if best['hidden_layers'] == 0: pass else: for i in range(best['hidden_layers']): model.add(Dense(best['neurons'], activation=best['activation'])) model.add(Dropout(best['dropout_rate'])) model.add(Dense(2, kernel_initializer='uniform', activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=best['optimizer'], metrics=['accuracy']) hist = model.fit(x_train, y_train, batch_size=best['batch_size'], epochs=best['epochs'], validation_data=(x_test, y_test), verbose=2) ## print the final results from unseen data score = model.evaluate(x_test, y_test, verbose=0) print("Accuracy: ", score[1]) f.write(20*'-' + '\nAccuracy of best model on unseen data: ' + str(score[1]) + \ '\n\nParameters used: ' + str(best)) f.close()