def run(self): model = load_model(input()['model'].path, custom_objects=ak.CUSTOM_OBJECTS) if self.data_type == 'csv': dataset = np.loadtxt(input().path) X = dataset[:, :-1] y = dataset[:, -1] else: if self.data_type == 'image': dataset = keras.preprocessing.image_dataset_from_directory( input().path, batch_size=64, image_size=(self.image_height, self.image_widht)) else: dataset = keras.preprocessing.text_dataset_from_directory( input().path, batch_size=64) X = np.array() y = np.array() for data, labels in dataset: X = np.vstack((X, data)) y = np.vstack((y, labels)) prediction = model.predict(X) if self.metric == 'accuracy': metric = Accuracy() else: metric = MeanSquaredError() result = metric(y, prediction) print(f'{self.metric}:', result)
def setCNNModel(n_steps, n_features=1, filters=64, kernel_size=2, pool_size=2, f=None): """ :param n_steps: - number of time steps :param n_features: -number features, 1 for time series :param filters:-For convolution level :param kernel_size: :param pool_size: :param f: :return: """ # define model model = Sequential() model.add( Conv1D(filters=64, kernel_size=2, activation='relu', input_shape=(n_steps, n_features))) model.add(MaxPooling1D(pool_size=2)) model.add(Flatten()) model.add(Dense(50, activation='relu')) model.add(Dense(1)) model.compile(optimizer='adam', loss='mse', metrics=[MeanSquaredError()]) model.summary() model.summary(print_fn=lambda x: f.write(x + '\n')) return model
def LSTM(self, input_shape=None, dropout_p=0.2, n_output=2, learning_rate=1e-3): log.log("LSTM Model Initilaizing...") if input_shape != None: self.input_shape = input_shape self.dropout_p = dropout_p self.n_output = n_output self.learning_rate = learning_rate self.model.add( LSTM(16, dropout=self.dropout_p, input_shape=self.input_shape, return_sequences=True)) self.model.add( LSTM(32, dropout=self.dropout_p, return_sequences=True)) self.model.add( LSTM(64, dropout=self.dropout_p, return_sequences=True)) self.model.add(Flatten()) self.model.add(Dense(units=32, activation='relu')) self.model.add(Dropout(self.dropout_p)) self.model.add(Dense(units=16, activation='relu')) self.model.add(Dropout(self.dropout_p)) self.model.add(Dense(units=8, activation='relu')) self.model.add(Dropout(self.dropout_p)) self.model.add(Dense(units=self.n_output)) #print(self.model.summary()) self.model.compile( optimizer=Adam(learning_rate=self.learning_rate), loss=MeanAbsoluteError(), metrics=[MeanSquaredError()]) log.log("LSTM Model Initialized!")
def lstm_model(self, infer: bool = False): input_data = Input(shape=(self.sequence_len, self.input_dim), dtype=tf_float32, batch_size=None) BiLSTM_layer = Bidirectional( LSTM(self.rnn_size, return_sequences=True, return_state=False, stateful=False, dropout=self.dropout)) lstm_output = BiLSTM_layer(input_data) ''' dense = Dense(self.output_dim, activation='linear') ''' output_data = TimeDistributed( Dense(self.output_dim, activation='linear'))(lstm_output) model = Model(inputs=input_data, outputs=output_data, name=self.model_name) optimizer = Adam(learning_rate=self.learning_rate, clipnorm=self.gradient_clip) model.compile(optimizer, loss='mse', metrics=[MeanSquaredError()]) model.summary() return model
def _eval_metrics_fn(): return { "acc": Accuracy(), "mse": MeanSquaredError(), "acc_fn": lambda labels, outputs: tf.equal( tf.cast(outputs, tf.int32), tf.cast(labels, tf.int32) ), }
def setLSTMModel(units, type_LSTM, possible_types, n_steps, n_features, f=None): """ :param units: :param type_LSTM: :param possible_types: :param n_steps: :param n_features: :param f: :return: """ # define model model = Sequential() code_LSTM = possible_types.get(type_LSTM) if code_LSTM == 0: # Vanilla LSTM model.add( LSTM(units, activation='relu', input_shape=(n_steps, n_features))) pass elif code_LSTM == 1: # stacked LSTM model.add( LSTM(units, activation='relu', return_sequences=True, input_shape=(n_steps, n_features))) model.add(LSTM(units, activation='relu')) elif code_LSTM == 2: # Bidirectional LSTM model.add( Bidirectional(LSTM(units, activation='relu'), input_shape=(n_steps, n_features))) elif code_LSTM == 3: # CNN LSTM model.add( TimeDistributed(Conv1D(filters=64, kernel_size=1, activation='relu'), input_shape=(None, n_steps / 2, n_features))) model.add(TimeDistributed(MaxPooling1D(pool_size=2))) model.add(TimeDistributed(Flatten())) model.add(LSTM(units, activation='relu')) else: model.add( LSTM(units, activation='relu', input_shape=(n_steps, n_features))) model.add(Dense(1)) model.compile(optimizer='adam', loss='mse', metrics=[MeanSquaredError()]) model.summary() model.summary(print_fn=lambda x: f.write(x + '\n')) return model
def createModel(): """ Initialization of the model 3 hidden layer activation relu, loss mean squared error """ inp = Input(shape=(3,)) x = Dense(4,activation='relu',use_bias=False)(inp) x = Dense(4,activation='relu',use_bias=False)(x) x = Dense(1,activation='relu',use_bias=False)(x) model = Model(inp,x) model.compile(loss="mean_squared_error",optimizer='sgd',metrics=[MeanSquaredError()]) return model
def lstm_model(self, infer: bool = False): model = Sequential() model.add( Input(shape=(self.sequence_len, self.input_dim), dtype=tf_float32, batch_size=None)) for i in range(self.num_of_layers): model.add( LSTM(self.rnn_size, return_sequences=True, return_state=False, stateful=False)) model.add(TimeDistributed(Dense(self.output_dim, activation='linear'))) optimizer = Adam(learning_rate=self.learning_rate, clipnorm=self.gradient_clip) model.compile(optimizer, loss='mse', metrics=[MeanSquaredError()]) model.summary() return model
def __init__(self, hparams, name, log_dir): self.univariate = hparams.get('UNIVARIATE', True) self.batch_size = int(hparams.get('BATCH_SIZE', 32)) self.epochs = int(hparams.get('EPOCHS', 500)) self.patience = int(hparams.get('PATIENCE', 15)) self.val_frac = hparams.get('VAL_FRAC', 0.15) self.T_x = int(hparams.get('T_X', 32)) self.metrics = [ MeanSquaredError(name='mse'), RootMeanSquaredError(name='rmse'), MeanAbsoluteError(name='mae'), MeanAbsolutePercentageError(name='mape') ] self.standard_scaler = StandardScaler() self.forecast_start = datetime.datetime.today() model = None super(NNModel, self).__init__(model, self.univariate, name, log_dir=log_dir)
def __init__(self, rnn_layer_sizes=[128], layer_normalize=[True], dropouts=[0.1], show_summary=True, patience=3, epochs=1000, batch_size=128, lr=0.001, loss='MSE', max_seq_len=128, embedding_size=200, monitor_loss='val_loss', metrics=[ MeanSquaredError(name='MSE'), MeanAbsoluteError(name='MAE'), MeanSquaredLogarithmicError(name='MSLE'), ]): self.lr = lr self.batch_size = batch_size self.rnn_layer_sizes = rnn_layer_sizes self.layer_normalize = layer_normalize self.dropouts = dropouts self.max_seq_len = max_seq_len self.show_summary = show_summary self.patience = patience self.epochs = epochs self.loss = loss self.embedding_size = embedding_size self.monitor_loss = monitor_loss self.metrics = metrics self.earlystop = tensorflow.keras.callbacks.EarlyStopping( monitor=self.monitor_loss, patience=self.patience, verbose=1, restore_best_weights=True, mode='min') self.unk_token = '[unk]' self.pad_token = '[pad]'
def train_model(self, train_timeseries, val_timeseries, features): adam = Adam(lr=0.001, decay=1e-3) model = Sequential() model.add( LSTM(32, activation='relu', input_shape=(self.past_days, features), return_sequences=True)) model.add(Dropout(0.2)) model.add(LSTM(16, activation='relu', return_sequences=True)) model.add(Dropout(0.2)) model.add(LSTM(8, activation='relu', return_sequences=False)) model.add(Dense(1)) print(model.summary()) model.compile(loss="mean_squared_error", optimizer=adam, metrics=[MeanSquaredError(), RootMeanSquaredError()]) reduce_lr_plateau = ReduceLROnPlateau(monitor='val_loss', factor=0.4, patience=3, verbose=1) early_stopping = EarlyStopping(monitor='val_loss', patience=4, verbose=1, mode='min', min_delta=0.0001) history = model.fit(train_timeseries, validation_data=val_timeseries, epochs=self.epochs, callbacks=[early_stopping, reduce_lr_plateau]) return model, history
def build_model(): #main input is the length of the amino acid in the protein sequence (700,) main_input = Input(shape=(700, ), dtype='float32', name='main_input') #Embedding Layer used as input to the neural network embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) #secondary input is the protein profile features auxiliary_input = Input(shape=(700, 21), name='aux_input') #auxiliary_input = Masking(mask_value=0)(auxiliary_input) #get shape of input layers print("Protein Sequence shape: ", main_input.get_shape()) print("Protein Profile shape: ", auxiliary_input.get_shape()) #concatenate 2 input layers concat = Concatenate(axis=-1)([embed, auxiliary_input]) ######## Recurrent Bi-Directional Long-Short-Term-Memory Layers ######## lstm_f1 = Bidirectional( LSTM(400, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=0.5, recurrent_dropout=0.5))(conv_features) lstm_f2 = Bidirectional( LSTM(300, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=0.5, recurrent_dropout=0.5))(lstm_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)([lstm_f1, lstm_f2, conv_features]) concat_features = Dropout(0.4)(concat_features) #Dense Fully-Connected DNN layers dense_1 = Dense(300, activation='relu')(conv_features) dense_1_dropout = Dropout(dense_dropout)(dense_1) dense_2 = Dense(100, activation='relu')(dense_1_dropout) dense_2_dropout = Dropout(dense_dropout)(dense_2) dense_3 = Dense(50, activation='relu')(dense_2_dropout) dense_3_dropout = Dropout(dense_dropout)(dense_3) dense_4 = Dense(16, activation='relu')(dense_3_dropout) dense_4_dropout = Dropout(dense_dropout)(dense_4) #Final Dense layer with 8 nodes for the 8 output classifications main_output = Dense(8, activation='softmax', name='main_output')(dense_4_dropout) #create model from inputs and outputs model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output]) #use Adam optimizer adam = Adam(lr=0.0003) #Adam is fast, but tends to over-fit #SGD is low but gives great results, sometimes RMSProp works best, SWA can easily improve quality, AdaTune #compile model using adam optimizer and the cateogorical crossentropy loss function model.compile(optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, metrics=[ 'accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision() ]) model.summary() #set earlyStopping and checkpoint callback earlyStopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1, mode='min') checkpoint_path = "/blstm_3x1Dconv_dnn_" + str( datetime.date(datetime.now())) + ".h5" checkpointer = ModelCheckpoint(filepath=checkpoint_path, verbose=1, save_best_only=True, monitor='val_acc', mode='max') return model
def build_model(): """ Description: Building DCBGRU model Args: None Returns: None """ #main input is the length of the amino acid in the protein sequence (700,) main_input = Input(shape=(700, ), dtype='float32', name='main_input') #Embedding Layer used as input to the neural network embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) #secondary input is the protein profile features auxiliary_input = Input(shape=(700, 21), name='aux_input') #get shape of input layers print("Protein Sequence shape: ", main_input.get_shape()) print("Protein Profile shape: ", auxiliary_input.get_shape()) #concatenate 2 input layers concat = Concatenate(axis=-1)([embed, auxiliary_input]) #3x1D Convolutional Hidden Layers with BatchNormalization, Dropout and MaxPooling conv_layer1 = Conv1D(16, 7, kernel_regularizer="l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer1) conv_act = activations.relu(batch_norm) conv_dropout = Dropout(0.2)(conv_act) max_pool_1D_1 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) conv_layer2 = Conv1D(32, 7, padding='same')(concat) batch_norm = BatchNormalization()(conv_layer2) conv_act = activations.relu(batch_norm) conv_dropout = Dropout(0.2)(conv_act) max_pool_1D_2 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) conv_layer3 = Conv1D(64, 7, kernel_regularizer="l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer3) conv_act = activations.relu(batch_norm) conv_dropout = Dropout(0.2)(conv_act) max_pool_1D_3 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) ############################################################################################ #concatenate convolutional layers conv_features = Concatenate(axis=-1)( [max_pool_1D_1, max_pool_1D_2, max_pool_1D_3]) #dense layer before GRU's gru_dense = Dense(600, activation='relu', name="after_cnn_dense")(conv_features) ######## Recurrent Unidirectional Long-Short-Term-Memory Layers ######## gru_f1 = Bidirectional( GRU(200, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=0.5, recurrent_dropout=0.5))(gru_dense) gru_f2 = Bidirectional( GRU(200, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=0.5, recurrent_dropout=0.5))(gru_f1) ############################################################################################ #concatenate GRU with convolutional layers concat_features = Concatenate(axis=-1)([gru_f1, gru_f2, gru_dense]) concat_features = Dropout(0.4)(concat_features) #Dense Fully-Connected DNN layers after_gru_dense = Dense(600, activation='relu')(concat_features) after_gru_dense_dropout = Dropout(0.3)(after_gru_dense) #Final Dense layer with 8 nodes for the 8 output classifications main_output = Dense(8, activation='softmax', name='main_output')(after_gru_dense_dropout) #create model from inputs and outputs model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output]) #use Adam optimizer adam = Adam(lr=0.00015) #compile model using adam optimizer and the cateogorical crossentropy loss function model.compile(optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, metrics=[ 'accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision() ]) #print model summary model.summary() return model
def build_model(): """ Description: Building dummy model Args: None Returns: None """ print("Building dummy model....") #main input is the length of the amino acid in the protein sequence (700,) main_input = Input(shape=(700, ), dtype='float32', name='main_input') #Embedding Layer used as input to the neural network embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) #secondary input is the protein profile features auxiliary_input = Input(shape=(700, 21), name='aux_input') #get shape of input layers print("Protein Sequence shape: ", main_input.get_shape()) print("Protein Profile shape: ", auxiliary_input.get_shape()) #concatenate 2 input layers concat = Concatenate(axis=-1)([embed, auxiliary_input]) ######## 1x1D-Convolutional Layers with BatchNormalization, Dropout and MaxPooling ######## conv_layer1 = Conv1D(16, 7, kernel_regularizer="l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer1) # conv2D_act = activations.relu(batch_norm) conv2D_act = ReLU()(batch_norm) conv_dropout = Dropout(0.2)(conv2D_act) ############################################################################################ #Final Dense layer with 8 nodes for the 8 output classifications main_output = Dense(8, activation='softmax', name='main_output')(conv_dropout) #create model from inputs and outputs model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output]) #use Adam optimizer adam = Adam(lr=0.00015) #compile model using adam optimizer and the cateogorical crossentropy loss function model.compile(optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, metrics=[ 'accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision(), AUC() ]) #print model summary model.summary() return model
def CNN(self, input_shape=None, stride=(1, 1), dilation=(1, 1), kernel_n=3, pooling_size=(2, 2), dropout_p=0.2, n_output=2, learning_rate=1e-3): log.log("CNN Model Initilaizing...") if input_shape != None: self.kernel_n = kernel_n self.input_shape = input_shape self.stride = stride # skips, kernel makes at every convolution self.dilation = dilation # kernel coverage self.pooling_size = pooling_size self.dropout_p = dropout_p self.n_output = n_output self.learning_rate = learning_rate self.model.add( Conv2D(filters=16, kernel_size=self.kernel_n, activation='relu', padding='same', input_shape=self.input_shape, strides=self.stride, dilation_rate=self.dilation)) #self.model.add(MaxPool2D(pool_size=self.pooling_size)) self.model.add( Conv2D(filters=32, kernel_size=self.kernel_n, activation='relu', padding='same', strides=self.stride, dilation_rate=self.dilation)) #self.model.add(MaxPool2D(pool_size=self.pooling_size)) self.model.add( Conv2D(filters=64, kernel_size=self.kernel_n, activation='relu', padding='same', strides=self.stride, dilation_rate=self.dilation)) #self.model.add(MaxPool2D(pool_size=self.pooling_size)) self.model.add( Conv2D(filters=128, kernel_size=self.kernel_n, activation='relu', padding='same', strides=self.stride, dilation_rate=self.dilation)) #self.model.add(MaxPool2D(pool_size=self.pooling_size)) self.model.add(Flatten()) self.model.add( Dense(units=self.input_shape[0] * 128, activation='relu')) self.model.add(Dropout(self.dropout_p)) self.model.add(Dense(units=128, activation='relu')) self.model.add(Dropout(self.dropout_p)) self.model.add(Dense(units=64, activation='relu')) self.model.add(Dropout(self.dropout_p)) self.model.add(Dense(units=32, activation='relu')) self.model.add(Dropout(self.dropout_p)) self.model.add(Dense(units=16, activation='relu')) self.model.add(Dropout(self.dropout_p)) self.model.add(Dense(units=self.n_output)) self.model.compile( optimizer=Adam(learning_rate=self.learning_rate), loss=MeanAbsoluteError(), metrics=[MeanSquaredError()]) log.log("CNN Model Initialized!")
]) dataset = DatasetGenerator(input_paths, label_paths).get() dataset_validation = DatasetGenerator(ValInput_paths, ValLabel_paths).get() # with strategy.scope(): batch_size = 8 model = make_seg_model() model.summary() optimizer = tf.keras.optimizers.Adam(1e-4) model.compile(optimizer=optimizer, loss=tf.keras.losses.MeanAbsoluteError(), metrics=[[MeanSquaredError()]]) # ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, model=model) # checkpoint_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=3, checkpoint_name=modelname) checkpoint_path = args.checkpoint model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_path, monitor='val_loss', mode='auto', save_best_only=True) model_early_stopping_callback = tf.keras.callbacks.EarlyStopping( monitor='val_loss', patience=10) # model_loss_error_callback = LossAndErrorPrintingCallback()
def build_model_hpconfig(args): """ Description: Building models for hyperparameter Tuning Args: args: input arguments Returns: model (keras model) """ #parsing and assigning hyperparameter variables from argparse conv1_filters = int(args.conv1_filters) conv2_filters = int(args.conv2_filters) conv3_filters = int(args.conv3_filters) window_size = int(args.window_size) kernel_regularizer = args.kernel_regularizer max_pool_size = int(args.pool_size) conv_dropout = float(args.conv_dropout) conv1d_initializer = args.conv_weight_initializer recurrent_layer1 = int(args.recurrent_layer1) recurrent_layer2 = int(args.recurrent_layer2) recurrent_dropout = float(args.recurrent_dropout) after_recurrent_dropout = float(args.after_recurrent_dropout) recurrent_recurrent_dropout = float(args.recurrent_recurrent_dropout) recurrent_initalizer = args.recurrent_weight_initializer optimizer = args.optimizer learning_rate = float(args.learning_rate) bidirection = args.bidirection recurrent_layer = str(args.recurrent_layer) dense_dropout = float(args.dense_dropout) dense_1 = int(args.dense_1) dense_initializer = args.dense_weight_initializer train_data = str(args.train_input_data) #main input is the length of the amino acid in the protein sequence (700,) main_input = Input(shape=(700, ), dtype='float32', name='main_input') #Embedding Layer used as input to the neural network embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) #secondary input is the protein profile features auxiliary_input = Input(shape=(700, 21), name='aux_input') #get shape of input layers print("Protein Sequence shape: ", main_input.get_shape()) print("Protein Profile shape: ", auxiliary_input.get_shape()) #concatenate input layers concat = Concatenate(axis=-1)([embed, auxiliary_input]) #3x1D Convolutional Hidden Layers with BatchNormalization, Dropout and MaxPooling conv_layer1 = Conv1D(conv1_filters, window_size, kernel_regularizer=kernel_regularizer, padding='same', kernel_initializer=conv1d_initializer)(concat) batch_norm = BatchNormalization()(conv_layer1) conv_act = activations.relu(batch_norm) conv_dropout = Dropout(conv_dropout)(conv_act) max_pool_1D_1 = MaxPooling1D(pool_size=max_pool_size, strides=1, padding='same')(conv_dropout) conv_layer2 = Conv1D(conv2_filters, window_size, padding='same', kernel_initializer=conv1d_initializer)(concat) batch_norm = BatchNormalization()(conv_layer2) conv_act = activations.relu(batch_norm) conv_dropout = Dropout(conv_dropout)(conv_act) max_pool_1D_2 = MaxPooling1D(pool_size=max_pool_size, strides=1, padding='same')(conv_dropout) conv_layer3 = Conv1D(conv3_filters, window_size, kernel_regularizer=kernel_regularizer, padding='same', kernel_initializer=conv1d_initializer)(concat) batch_norm = BatchNormalization()(conv_layer3) conv_act = activations.relu(batch_norm) conv_dropout = Dropout(conv_dropout)(conv_act) max_pool_1D_3 = MaxPooling1D(pool_size=max_pool_size, strides=1, padding='same')(conv_dropout) #concat pooling layers conv_features = Concatenate(axis=-1)( [max_pool_1D_1, max_pool_1D_2, max_pool_1D_3]) print("Shape of convolutional output: ", conv_features.get_shape()) conv_features = Dense(600, activation='relu')(conv_features) ######## Recurrent Layers ######## if (recurrent_layer == 'lstm'): if (bidirection): print('Entering LSTM Layers') #Creating Bidirectional LSTM layers lstm_f1 = Bidirectional( LSTM(recurrent_layer1, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout, kernel_initializer=recurrent_initalizer))(conv_features) lstm_f2 = Bidirectional( LSTM(recurrent_layer2, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout, kernel_initializer=recurrent_initalizer))(lstm_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)( [lstm_f1, lstm_f2, conv_features]) concat_features = Dropout(after_recurrent_dropout)(concat_features) print('Concatenated LSTM layers') else: #Creating unidirectional LSTM Layers lstm_f1 = LSTM( recurrent_layer1, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout, kernel_initializer=recurrent_initalizer)(conv_features) lstm_f2 = LSTM(recurrent_layer2, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout, kernel_initializer=recurrent_initalizer)(lstm_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)( [lstm_f1, lstm_f2, conv_features]) concat_features = Dropout(after_recurrent_dropout)(concat_features) elif (recurrent_layer == 'gru'): if (bidirection): #Creating Bidirectional GRU layers gru_f1 = Bidirectional( GRU(recurrent_layer1, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout, kernel_initializer=recurrent_initalizer))(conv_features) gru_f2 = Bidirectional( GRU(recurrent_layer2, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout, kernel_initializer=recurrent_initalizer))(gru_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)( [gru_f1, gru_f2, conv_features]) concat_features = Dropout(after_recurrent_dropout)(concat_features) else: #Creating unidirectional GRU Layers gru_f1 = GRU( recurrent_layer1, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout, kernel_initializer=recurrent_initalizer)(conv_features) gru_f2 = GRU(recurrent_layer1, return_sequences=True, activation='tanh', recurrent_activation='sigmoid', dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout, kernel_initializer=recurrent_initalizer)(gru_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)( [gru_f1, gru_f2, conv_features]) concat_features = Dropout(after_recurrent_dropout)(concat_features) else: print('Only LSTM and GRU recurrent layers are used in this model') return #Dense Fully-Connected DNN layers fc_dense1 = Dense(dense_1, activation='relu', kernel_initializer=dense_initializer)(concat_features) fc_dense1_dropout = Dropout(dense_dropout)(fc_dense1) #Final Output layer with 8 nodes for the 8 output classifications main_output = Dense(8, activation='softmax', name='main_output')(fc_dense1_dropout) #create model from inputs and outputs model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output]) #Set optimizer to be used with the model, default is Adam if optimizer == 'adam': optimizer = Adam(lr=learning_rate, name='adam') elif optimizer == 'sgd': optimizer = SGD(lr=0.01, momentum=0.0, nesterov=False, name='SGD') elif optimizer == 'rmsprop': optimizer = RMSprop(learning_rate=learning_rate, centered=True, name='RMSprop') elif optimizer == 'adagrad': optimizer = Adagrad(learning_rate=learning_rate, name='Adagrad') elif optimizer == 'adamax': optimizer = Adamax(learning_rate=learning_rate, name='Adamax') else: optimizer = 'adam' optimizer = Adam(lr=learning_rate, name='adam') #compile model using optimizer and the cateogorical crossentropy loss function model.compile(optimizer=optimizer, loss={'main_output': 'categorical_crossentropy'}, metrics=[ 'accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision() ]) #get summary of model including its layers and num parameters model.summary() return model
def build_model(): """ Description: Building PSP-CD model Args: None Returns: None """ #main input is the length of the amino acid in the protein sequence (700,) main_input = Input(shape=(700, ), dtype='float32', name='main_input') #Embedding Layer used as input to the neural network embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) #secondary input is the protein profile features auxiliary_input = Input(shape=(700, 21), name='aux_input') #get shape of input layers print("Protein Sequence shape: ", main_input.get_shape()) print("Protein Profile shape: ", auxiliary_input.get_shape()) #concatenate 2 input layers concat_features = Concatenate(axis=-1)([embed, auxiliary_input]) ############################################################################################ #Dense Fully-Connected DNN layers dense_1 = Dense(512, activation='relu')(concat_features) dense_1_dropout = Dropout(0.3)(dense_1) dense_2 = Dense(256, activation='relu')(dense_1_dropout) dense_2_dropout = Dropout(0.3)(dense_2) dense_3 = Dense(128, activation='relu')(dense_2_dropout) dense_3_dropout = Dropout(0.3)(dense_3) dense_4 = Dense(64, activation='relu')(dense_3_dropout) dense_4_dropout = Dropout(0.3)(dense_4) dense_5 = Dense(32, activation='relu')(dense_4_dropout) dense_5_dropout = Dropout(0.3)(dense_5) dense_6 = Dense(16, activation='relu')(dense_5_dropout) dense_6_dropout = Dropout(0.3)(dense_6) #Final Dense layer with 8 nodes for the 8 output classifications main_output = Dense(8, activation='softmax', name='main_output')(dense_6_dropout) #create model from inputs and outputs model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output]) #use Adam optimizer adam = Adam(lr=0.00015) #compile model using adam optimizer and the cateogorical crossentropy loss function model.compile(optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, metrics=[ 'accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision() ]) #print model summary model.summary() return model
def build_model(): #main input is the length of the amino acid in the protein sequence (700,) main_input = Input(shape=(700, ), dtype='float32', name='main_input') #Embedding Layer used as input to the neural network embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) #secondary input is the protein profile features auxiliary_input = Input(shape=(700, 21), name='aux_input') #get shape of input layers print("Protein Sequence shape: ", main_input.get_shape()) print("Protein Profile shape: ", auxiliary_input.get_shape()) #concatenate 2 input layers concat = Concatenate(axis=-1)([embed, auxiliary_input]) #3x1D Convolutional Hidden Layers with BatchNormalization and MaxPooling conv_layer1 = Conv1D(64, 7, kernel_regularizer="l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer1) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(0.5)(conv2D_act) # ave_pool_1 = AveragePooling1D(2, 1, padding='same')(conv_dropout) max_pool_1D_1 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) conv_layer2 = Conv1D(128, 7, padding='same')(concat) batch_norm = BatchNormalization()(conv_layer2) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(0.5)(conv2D_act) # ave_pool_2 = AveragePooling1D(2, 1, padding='same')(conv_dropout) max_pool_1D_2 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) conv_layer3 = Conv1D(256, 7, kernel_regularizer="l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer3) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(0.5)(conv2D_act) max_pool_1D_3 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) # ave_pool_3 = AveragePooling1D(2, 1, padding='same')(conv_dropout) #concatenate convolutional layers conv_features = Concatenate(axis=-1)( [max_pool_1D_1, max_pool_1D_2, max_pool_1D_3]) #output node is 1D convolutional layer with 8 filters for the 8 different categories main_output = Conv1D(8, 7, padding='same', activation='softmax', name='main_output')(conv_features) #create model from inputs and outputs model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output]) #use Adam optimizer adam = Adam(lr=0.0003) #compile model using adam optimizer and the cateogorical crossentropy loss function model.compile(optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, metrics=[ 'accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision() ]) model.summary() #set earlyStopping and checkpoint callback earlyStopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1, mode='min') checkpoint_path = "checkpoints/3xConv_cnn_" + str( datetime.date(datetime.now())) + ".h5" checkpointer = ModelCheckpoint(filepath=checkpoint_path, verbose=1, save_best_only=True, monitor='val_acc', mode='max') return model
def train_aae(configs): batch_size = configs["batch_size"] dataset_name = configs["dataset_name"] filter = configs["filter"] label_name = configs["label_name"] train, val = load_dataset(dataset_name, sets=["train", "val"], shuffle=False, label_name=label_name, batch_size=batch_size, filter=filter) epochs = configs["epochs"] latent_dim = configs["latent_dim"] intermediate_dim = configs["intermediate_dim"] use_clf_label = configs["use_clf_label"] weight = configs["reconstruction_weight"] loss_weights = [ weight, (1 - weight) / 3, (1 - weight) / 3, (1 - weight) / 3 ] distance_thresh = configs["distance_thresh"] classification_model = tf.keras.models.load_model("../models/{}_{}".format( configs["clf_type"], dataset_name)) # if not hyperparameter tuning, set up tensorboard logdir = "tensorboard_logs/aae/" + datetime.now().strftime("%Y%m%d-%H%M%S") train_summary_writer = tf.summary.create_file_writer(logdir) with open("../data/{}/metadata.txt".format(dataset_name)) as file: metadata = json.load(file) num_classes = metadata["num_classes"] field_names = metadata["field_names"][:-1] input_dim = len(field_names) datamin = np.array(metadata["col_min"][:-1]) datamax = np.array(metadata["col_max"][:-1]) scaler, unscaler = min_max_scaler_gen(datamin, datamax) data_range = datamax - datamin # packed_train_data = train.map( PackNumericFeatures(field_names, num_classes, scaler=scaler)) packed_val_data = val.map( PackNumericFeatures(field_names, num_classes, scaler=scaler)) # Create a MirroredStrategy. # strategy = tf.distribute.MirroredStrategy() # print('Number of devices: {}'.format(strategy.num_replicas_in_sync)) # # # Open a strategy scope. # with strategy.scope(): # oop version does not provide a good graph trace so using functional instead aae, encoder, decoder, latent_discriminator, cat_discriminator = build_aae_dim_reduce( input_dim, intermediate_dim, latent_dim, num_classes, distance_thresh, None, None, loss_weights) # wmse=weighted_mse(data_range) aae.compile(loss=[ tf.keras.losses.MeanSquaredError(), tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.MeanSquaredError() ], loss_weights=loss_weights, optimizer='adam', metrics=[[MeanSquaredError(name="latent_mse")], [BinaryAccuracy(name="latent_acc")], [BinaryAccuracy(name="label_acc")], [MeanSquaredError(name="label_mse")]]) valid = np.ones((batch_size, 1)) invalid = np.zeros((batch_size, 1)) step = 0 pbar = tqdm(range(epochs), desc="epoch") for epoch in pbar: steps = metadata["num_train"] // batch_size step_pbar = tqdm(total=steps, desc="steps", leave=False, position=1) for feature, label in packed_train_data.take(steps): fake_latent, fake_cat = encoder(feature) # train latent discriminator latent_discriminator.trainable = True real_latent = sample_prior(batch_size, distro="normal", latent_dim=latent_dim) noise_real = np.random.normal(0, 0.5, size=(batch_size, latent_dim)) d_loss_real = latent_discriminator.train_on_batch( real_latent + noise_real, valid) d_loss_fake = latent_discriminator.train_on_batch( fake_latent, invalid) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) latent_discriminator.trainable = False # train cat discriminator cat_discriminator.trainable = True if use_clf_label: real_cat = classification_model(feature) real_cat = np.argmax(real_cat, axis=-1) real_cat = np.eye(num_classes)[real_cat] else: real_cat = label cat_loss_real = cat_discriminator.train_on_batch(real_cat, valid) cat_loss_fake = cat_discriminator.train_on_batch(fake_cat, invalid) cat_loss = 0.5 * np.add(cat_loss_real, cat_loss_fake) cat_discriminator.trainable = False # train generator g_loss = aae.train_on_batch(feature, [feature, valid, valid, real_cat]) # record losses if not tuning with train_summary_writer.as_default(): tf.summary.scalar('latent loss', d_loss[0], step=step) tf.summary.scalar('latent acc', d_loss[1], step=step) tf.summary.scalar('cat loss', cat_loss[0], step=step) tf.summary.scalar('cat acc', cat_loss[1], step=step) for i in range(len(aae.metrics_names)): tf.summary.scalar(aae.metrics_names[i], g_loss[i], step=step) step += 1 step_pbar.update(1) # record distribution after epoch style, label = encoder(feature) with train_summary_writer.as_default(): tf.summary.histogram('cat_disc_out', cat_discriminator(label), step=step) tf.summary.histogram('lat_disc_out', latent_discriminator(style), step=step) tf.summary.histogram('style', style, step=step) tf.summary.histogram('label', label, step=step) tf.summary.histogram('prior style', real_latent + noise_real, step=step) tf.summary.histogram('prior label', real_cat, step=step) step_pbar.reset() postfix = { "latent_acc": 100 * d_loss[1], "cat_acc": 100 * cat_loss[1], "mse": g_loss[5] } pbar.set_postfix(postfix) # trace aae with dummy value and save model feature, label = list(packed_val_data.take(1).as_numpy_iterator())[0] feature = np.zeros((1, input_dim)) label = np.zeros((num_classes)) latent = np.zeros((1, latent_dim)) # trace aae with train_summary_writer.as_default(): tf.summary.trace_on(graph=True, profiler=True) tracer(aae, feature) tf.summary.trace_export(name="aae", step=0, profiler_outdir=logdir) # tf.keras.models.save_model(aae, "../models/aae/test") prefix = "{}_{}_{}".format(dataset_name, latent_dim, use_clf_label) if not os.path.isdir("../models/aae"): os.makedirs("../models/aae") aae.save("../models/aae/{}_aae.h5".format(prefix)) encoder.save("../models/aae/{}_encoder.h5".format(prefix)) decoder.save("../models/aae/{}_decoder.h5".format(prefix)) latent_discriminator.save("../models/aae/{}_lat_disc.h5".format(prefix)) cat_discriminator.save("../models/aae/{}_cat_disc.h5".format(prefix))
def build_model_hpconfig(args): #parsing and assigning hyperparameter variables from argparse conv1_filters=int(args.conv1_filters) conv2_filters=int(args.conv2_filters) conv3_filters=int(args.conv3_filters) window_size=int(args.window_size) kernel_regularizer = args.kernel_regularizer conv_dropout=float(args.conv2d_dropout) pool_size = int(args.pool_size) conv2d_activation=args.conv2d_activation conv2d_dropout=float(args.conv2d_dropout) recurrent_layer1 = int(args.recurrent_layer1) recurrent_layer2 = int(args.recurrent_layer2) recurrent_dropout = float(args.recurrent_dropout) after_recurrent_dropout = float(args.after_recurrent_dropout) recurrent_recurrent_dropout = float(args.recurrent_recurrent_dropout) optimizer=args.optimizer learning_rate = float(args.learning_rate) bidirection = args.bidirection recurrent_layer = args.recurrent_layer dense_dropout = float(args.dense_dropout) dense_1 = int(args.dense_1) dense_2 = int(args.dense_2) dense_3 = int(args.dense_3) dense_4 = int(args.dense_4) #main input is the length of the amino acid in the protein sequence (700,) main_input = Input(shape=(700,), dtype='float32', name='main_input') #Embedding Layer used as input to the neural network embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) #secondary input is the protein profile features auxiliary_input = Input(shape=(700,21), name='aux_input') #concatenate input layers concat = Concatenate(axis=-1)([embed, auxiliary_input]) conv_layer1 = Convolution1D(conv1_filters, window_size, kernel_regularizer = "l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer1) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(conv_dropout)(conv2D_act) # ave_pool_1 = AveragePooling1D(3, 1, padding='same')(conv_dropout) max_pool_1D_1 = MaxPooling1D(pool_size=pool_size, strides=1, padding='same')(conv_dropout) conv_layer2 = Convolution1D(conv2_filters, window_size, padding='same')(concat) batch_norm = BatchNormalization()(conv_layer2) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(conv_dropout)(conv2D_act) # ave_pool_2 = AveragePooling1D(3, 1, padding='same')(conv_dropout) max_pool_1D_2 = MaxPooling1D(pool_size=pool_size, strides=1, padding='same')(conv_dropout) conv_layer3 = Convolution1D(conv3_filters, window_size,kernel_regularizer = "l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer3) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(conv_dropout)(conv2D_act) max_pool_1D_3 = MaxPooling1D(pool_size=pool_size, strides=1, padding='same')(conv_dropout) #concat pooling layers conv_features = Concatenate(axis=-1)([max_pool_1D_1, max_pool_1D_2, max_pool_1D_3]) ######## Recurrent Layers ######## if (recurrent_layer == 'lstm'): if (bidirection): #Creating Bidirectional LSTM layers lstm_f1 = Bidirectional(LSTM(recurrent_layer1,return_sequences=True,activation = 'tanh', recurrent_activation='sigmoid',dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout))(conv_features) lstm_f2 = Bidirectional(LSTM(recurrent_layer2, return_sequences=True,activation = 'tanh',recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout))(lstm_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)([lstm_f1, lstm_f2, conv2_features]) concat_features = Dropout(after_recurrent_dropout)(concat_features) else: #Creating unidirectional LSTM Layers lstm_f1 = LSTM(recurrent_layer1,return_sequences=True,activation = 'tanh', recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout)(conv_features) lstm_f2 = LSTM(recurrent_layer2, return_sequences=True,activation = 'tanh',recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout)(lstm_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)([lstm_f1, lstm_f2, conv_features]) concat_features = Dropout(after_recurrent_dropout)(concat_features) elif (recurrent_layer == 'gru'): if (bidirection): #Creating Bidirectional GRU layers gru_f1 = Bidirectional(GRU(recurrent_layer1,return_sequences=True,activation = 'tanh', recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout))(conv_features) gru_f2 = Bidirectional(GRU(recurrent_layer2, return_sequences=True,activation = 'tanh',recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout))(gru_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)([gru_f1, gru_f2, conv_features]) concat_features = Dropout(after_recurrent_dropout)(concat_features) else: #Creating unidirectional GRU Layers gru_f1 = GRU(recurrent_layer1,return_sequences=True,activation = 'tanh', recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout)(conv_features) gru_f2 = GRU(recurrent_layer1, return_sequences=True,activation = 'tanh',recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout)(gru_f1) #concatenate LSTM with convolutional layers concat_features = Concatenate(axis=-1)([gru_f1, gru_f2, conv_features]) concat_features = Dropout(after_recurrent_dropout)(concat_features) else: print('Only LSTM and GRU recurrent layers are used in this model') return #Dense Fully-Connected DNN layers # concat_features = Flatten()(concat_features) fc_dense1 = Dense(dense_1, activation='relu')(concat_features) fc_dense1_dropout = Dropout(dense_dropout)(fc_dense1) fc_dense2 = Dense(dense_2, activation='relu')(fc_dense1_dropout) fc_dense2_dropout = Dropout(dense_dropout)(fc_dense2) fc_dense3 = Dense(dense_3, activation='relu')(fc_dense2_dropout) fc_dense3_dropout = Dropout(dense_dropout)(fc_dense3) #Final Output layer with 8 nodes for the 8 output classifications # main_output = Dense(8, activation='softmax', name='main_output')(concat_features) main_output = Dense(8, activation='softmax', name='main_output')(fc_dense3_dropout) #create model from inputs and outputs model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output]) #Set optimizer to be used with the model, default is Adam if optimizer == 'adam': optimizer = Adam(lr=learning_rate, name='adam') elif optimizer == 'sgd': optimizer = SGD(lr=0.01, momentum=0.0, nestero=False, name='SGD') elif optimizer == 'rmsprop': optimizer = RMSprop(learning_rate=learning_rate, centered = True, name='RMSprop') elif optimizer == 'adagrad': optimizer = Adagrad(learning_rate = learning_rate, name='Adagrad') elif optimizer == 'adamax': optimizer = Adamax(learning_rate=learning_rate, name='Adamax') else: optimizer = 'adam' optimizer = Adam(lr=learning_rate, name='adam') #Nadam & Ftrl optimizers #use Adam optimizer #optimizer = Adam(lr=0.003) #Adam is fast, but tends to over-fit #SGD is low but gives great results, sometimes RMSProp works best, SWA can easily improve quality, AdaTune #compile model using optimizer and the cateogorical crossentropy loss function model.compile(optimizer = optimizer, loss={'main_output': 'categorical_crossentropy'}, metrics=['accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision()]) #get summary of model including its layers and num parameters model.summary() #set early stopping and checkpoints for model earlyStopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1, mode='min') checkpoint_path = BUCKET_PATH + "/checkpoints/" + str(datetime.date(datetime.now())) +\ '_' + str((datetime.now().strftime('%H:%M'))) + ".h5" checkpointer = ModelCheckpoint(filepath=checkpoint_path,verbose=1,save_best_only=True, monitor='val_acc', mode='max') return model
def build_model(): #main input is the length of the amino acid in the protein sequence (700,) main_input = Input(shape=(700, ), dtype='float32', name='main_input') #Embedding Layer used as input to the neural network embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input) #secondary input is the protein profile features auxiliary_input = Input(shape=(700, 21), name='aux_input') #get shape of input layers print("Protein Sequence shape: ", main_input.get_shape()) print("Protein Profile shape: ", auxiliary_input.get_shape()) #concatenate 2 input layers concat = Concatenate(axis=-1)([embed, auxiliary_input]) #3x1D Convolutional Hidden Layers with BatchNormalization and MaxPooling conv_layer1 = Convolution1D(64, 7, kernel_regularizer="l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer1) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(0.5)(conv2D_act) max_pool_2D_1 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) conv_layer2 = Convolution1D(128, 7, padding='same')(concat) batch_norm = BatchNormalization()(conv_layer2) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(0.5)(conv2D_act) max_pool_2D_2 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) conv_layer3 = Convolution1D(256, 7, kernel_regularizer="l2", padding='same')(concat) batch_norm = BatchNormalization()(conv_layer3) conv2D_act = activations.relu(batch_norm) conv_dropout = Dropout(0.5)(conv2D_act) max_pool_2D_3 = MaxPooling1D(pool_size=2, strides=1, padding='same')(conv_dropout) #concatenate convolutional layers conv_features = Concatenate(axis=-1)( [max_pool_2D_1, max_pool_2D_2, max_pool_2D_3]) #Dense Fully-Connected DNN layers dense_1 = Dense(300, activation='relu')(conv_features) dense_1_dropout = Dropout(dense_dropout)(dense_1) dense_2 = Dense(100, activation='relu')(dense_1_dropout) dense_2_dropout = Dropout(dense_dropout)(dense_2) dense_3 = Dense(50, activation='relu')(dense_2_dropout) dense_3_dropout = Dropout(dense_dropout)(dense_3) dense_4 = Dense(16, activation='relu')(dense_3_dropout) dense_4_dropout = Dropout(dense_dropout)(dense_4) #Final Dense layer with 8 nodes for the 8 output classifications main_output = Dense(8, activation='softmax', name='main_output')(protein_features_dropout) #create model from inputs and outputs model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output]) #use Adam optimizer adam = Adam(lr=lr) #Adam is fast, but tends to over-fit #SGD is low but gives great results, sometimes RMSProp works best, SWA can easily improve quality, AdaTune #compile model using adam optimizer and the cateogorical crossentropy loss function model.compile(optimizer=adam, loss={'main_output': 'categorical_crossentropy'}, metrics=[ 'accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision() ]) model.summary() #set earlyStopping and checkpoint callback earlyStopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1, mode='min') checkpoint_path = "/3x1DConv_dnn_" + str(datetime.date( datetime.now())) + ".h5" checkpointer = ModelCheckpoint(filepath=checkpoint_path, verbose=1, save_best_only=True, monitor='val_acc', mode='max') return model