def run(self):
model = load_model(input()['model'].path,
custom_objects=ak.CUSTOM_OBJECTS)
if self.data_type == 'csv':
dataset = np.loadtxt(input().path)
X = dataset[:, :-1]
y = dataset[:, -1]
else:
if self.data_type == 'image':
dataset = keras.preprocessing.image_dataset_from_directory(
input().path,
batch_size=64,
image_size=(self.image_height, self.image_widht))
else:
dataset = keras.preprocessing.text_dataset_from_directory(
input().path, batch_size=64)
X = np.array()
y = np.array()
for data, labels in dataset:
X = np.vstack((X, data))
y = np.vstack((y, labels))
prediction = model.predict(X)
if self.metric == 'accuracy':
metric = Accuracy()
else:
metric = MeanSquaredError()
result = metric(y, prediction)
print(f'{self.metric}:', result)
def setCNNModel(n_steps,
n_features=1,
filters=64,
kernel_size=2,
pool_size=2,
f=None):
"""
:param n_steps: - number of time steps
:param n_features: -number features, 1 for time series
:param filters:-For convolution level
:param kernel_size:
:param pool_size:
:param f:
:return:
"""
# define model
model = Sequential()
model.add(
Conv1D(filters=64,
kernel_size=2,
activation='relu',
input_shape=(n_steps, n_features)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(50, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse', metrics=[MeanSquaredError()])
model.summary()
model.summary(print_fn=lambda x: f.write(x + '\n'))
return model
def LSTM(self,
input_shape=None,
dropout_p=0.2,
n_output=2,
learning_rate=1e-3):
log.log("LSTM Model Initilaizing...")
if input_shape != None:
self.input_shape = input_shape
self.dropout_p = dropout_p
self.n_output = n_output
self.learning_rate = learning_rate
self.model.add(
LSTM(16,
dropout=self.dropout_p,
input_shape=self.input_shape,
return_sequences=True))
self.model.add(
LSTM(32, dropout=self.dropout_p, return_sequences=True))
self.model.add(
LSTM(64, dropout=self.dropout_p, return_sequences=True))
self.model.add(Flatten())
self.model.add(Dense(units=32, activation='relu'))
self.model.add(Dropout(self.dropout_p))
self.model.add(Dense(units=16, activation='relu'))
self.model.add(Dropout(self.dropout_p))
self.model.add(Dense(units=8, activation='relu'))
self.model.add(Dropout(self.dropout_p))
self.model.add(Dense(units=self.n_output))
#print(self.model.summary())
self.model.compile(
optimizer=Adam(learning_rate=self.learning_rate),
loss=MeanAbsoluteError(),
metrics=[MeanSquaredError()])
log.log("LSTM Model Initialized!")
def lstm_model(self, infer: bool = False):
input_data = Input(shape=(self.sequence_len, self.input_dim),
dtype=tf_float32,
batch_size=None)
BiLSTM_layer = Bidirectional(
LSTM(self.rnn_size,
return_sequences=True,
return_state=False,
stateful=False,
dropout=self.dropout))
lstm_output = BiLSTM_layer(input_data)
'''
dense = Dense(self.output_dim,
activation='linear')
'''
output_data = TimeDistributed(
Dense(self.output_dim, activation='linear'))(lstm_output)
model = Model(inputs=input_data,
outputs=output_data,
name=self.model_name)
optimizer = Adam(learning_rate=self.learning_rate,
clipnorm=self.gradient_clip)
model.compile(optimizer, loss='mse', metrics=[MeanSquaredError()])
model.summary()
return model
def _eval_metrics_fn():
return {
"acc": Accuracy(),
"mse": MeanSquaredError(),
"acc_fn": lambda labels, outputs: tf.equal(
tf.cast(outputs, tf.int32), tf.cast(labels, tf.int32)
),
}
def setLSTMModel(units,
type_LSTM,
possible_types,
n_steps,
n_features,
f=None):
"""
:param units:
:param type_LSTM:
:param possible_types:
:param n_steps:
:param n_features:
:param f:
:return:
"""
# define model
model = Sequential()
code_LSTM = possible_types.get(type_LSTM)
if code_LSTM == 0: # Vanilla LSTM
model.add(
LSTM(units, activation='relu', input_shape=(n_steps, n_features)))
pass
elif code_LSTM == 1: # stacked LSTM
model.add(
LSTM(units,
activation='relu',
return_sequences=True,
input_shape=(n_steps, n_features)))
model.add(LSTM(units, activation='relu'))
elif code_LSTM == 2: # Bidirectional LSTM
model.add(
Bidirectional(LSTM(units, activation='relu'),
input_shape=(n_steps, n_features)))
elif code_LSTM == 3: # CNN LSTM
model.add(
TimeDistributed(Conv1D(filters=64,
kernel_size=1,
activation='relu'),
input_shape=(None, n_steps / 2, n_features)))
model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(units, activation='relu'))
else:
model.add(
LSTM(units, activation='relu', input_shape=(n_steps, n_features)))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse', metrics=[MeanSquaredError()])
model.summary()
model.summary(print_fn=lambda x: f.write(x + '\n'))
return model
def createModel(): """ Initialization of the model 3 hidden layer activation relu, loss mean squared error """ inp = Input(shape=(3,)) x = Dense(4,activation='relu',use_bias=False)(inp) x = Dense(4,activation='relu',use_bias=False)(x) x = Dense(1,activation='relu',use_bias=False)(x) model = Model(inp,x) model.compile(loss="mean_squared_error",optimizer='sgd',metrics=[MeanSquaredError()]) return model
def lstm_model(self, infer: bool = False):
model = Sequential()
model.add(
Input(shape=(self.sequence_len, self.input_dim),
dtype=tf_float32,
batch_size=None))
for i in range(self.num_of_layers):
model.add(
LSTM(self.rnn_size,
return_sequences=True,
return_state=False,
stateful=False))
model.add(TimeDistributed(Dense(self.output_dim, activation='linear')))
optimizer = Adam(learning_rate=self.learning_rate,
clipnorm=self.gradient_clip)
model.compile(optimizer, loss='mse', metrics=[MeanSquaredError()])
model.summary()
return model
def __init__(self, hparams, name, log_dir):
self.univariate = hparams.get('UNIVARIATE', True)
self.batch_size = int(hparams.get('BATCH_SIZE', 32))
self.epochs = int(hparams.get('EPOCHS', 500))
self.patience = int(hparams.get('PATIENCE', 15))
self.val_frac = hparams.get('VAL_FRAC', 0.15)
self.T_x = int(hparams.get('T_X', 32))
self.metrics = [
MeanSquaredError(name='mse'),
RootMeanSquaredError(name='rmse'),
MeanAbsoluteError(name='mae'),
MeanAbsolutePercentageError(name='mape')
]
self.standard_scaler = StandardScaler()
self.forecast_start = datetime.datetime.today()
model = None
super(NNModel, self).__init__(model,
self.univariate,
name,
log_dir=log_dir)
def __init__(self,
rnn_layer_sizes=[128],
layer_normalize=[True],
dropouts=[0.1],
show_summary=True,
patience=3,
epochs=1000,
batch_size=128,
lr=0.001,
loss='MSE',
max_seq_len=128,
embedding_size=200,
monitor_loss='val_loss',
metrics=[
MeanSquaredError(name='MSE'),
MeanAbsoluteError(name='MAE'),
MeanSquaredLogarithmicError(name='MSLE'),
]):
self.lr = lr
self.batch_size = batch_size
self.rnn_layer_sizes = rnn_layer_sizes
self.layer_normalize = layer_normalize
self.dropouts = dropouts
self.max_seq_len = max_seq_len
self.show_summary = show_summary
self.patience = patience
self.epochs = epochs
self.loss = loss
self.embedding_size = embedding_size
self.monitor_loss = monitor_loss
self.metrics = metrics
self.earlystop = tensorflow.keras.callbacks.EarlyStopping(
monitor=self.monitor_loss,
patience=self.patience,
verbose=1,
restore_best_weights=True,
mode='min')
self.unk_token = '[unk]'
self.pad_token = '[pad]'
def train_model(self, train_timeseries, val_timeseries, features):
adam = Adam(lr=0.001, decay=1e-3)
model = Sequential()
model.add(
LSTM(32,
activation='relu',
input_shape=(self.past_days, features),
return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(16, activation='relu', return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(8, activation='relu', return_sequences=False))
model.add(Dense(1))
print(model.summary())
model.compile(loss="mean_squared_error",
optimizer=adam,
metrics=[MeanSquaredError(),
RootMeanSquaredError()])
reduce_lr_plateau = ReduceLROnPlateau(monitor='val_loss',
factor=0.4,
patience=3,
verbose=1)
early_stopping = EarlyStopping(monitor='val_loss',
patience=4,
verbose=1,
mode='min',
min_delta=0.0001)
history = model.fit(train_timeseries,
validation_data=val_timeseries,
epochs=self.epochs,
callbacks=[early_stopping, reduce_lr_plateau])
return model, history
def build_model():
#main input is the length of the amino acid in the protein sequence (700,)
main_input = Input(shape=(700, ), dtype='float32', name='main_input')
#Embedding Layer used as input to the neural network
embed = Embedding(output_dim=21, input_dim=21,
input_length=700)(main_input)
#secondary input is the protein profile features
auxiliary_input = Input(shape=(700, 21), name='aux_input')
#auxiliary_input = Masking(mask_value=0)(auxiliary_input)
#get shape of input layers
print("Protein Sequence shape: ", main_input.get_shape())
print("Protein Profile shape: ", auxiliary_input.get_shape())
#concatenate 2 input layers
concat = Concatenate(axis=-1)([embed, auxiliary_input])
######## Recurrent Bi-Directional Long-Short-Term-Memory Layers ########
lstm_f1 = Bidirectional(
LSTM(400,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=0.5,
recurrent_dropout=0.5))(conv_features)
lstm_f2 = Bidirectional(
LSTM(300,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=0.5,
recurrent_dropout=0.5))(lstm_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)([lstm_f1, lstm_f2, conv_features])
concat_features = Dropout(0.4)(concat_features)
#Dense Fully-Connected DNN layers
dense_1 = Dense(300, activation='relu')(conv_features)
dense_1_dropout = Dropout(dense_dropout)(dense_1)
dense_2 = Dense(100, activation='relu')(dense_1_dropout)
dense_2_dropout = Dropout(dense_dropout)(dense_2)
dense_3 = Dense(50, activation='relu')(dense_2_dropout)
dense_3_dropout = Dropout(dense_dropout)(dense_3)
dense_4 = Dense(16, activation='relu')(dense_3_dropout)
dense_4_dropout = Dropout(dense_dropout)(dense_4)
#Final Dense layer with 8 nodes for the 8 output classifications
main_output = Dense(8, activation='softmax',
name='main_output')(dense_4_dropout)
#create model from inputs and outputs
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output])
#use Adam optimizer
adam = Adam(lr=0.0003)
#Adam is fast, but tends to over-fit
#SGD is low but gives great results, sometimes RMSProp works best, SWA can easily improve quality, AdaTune
#compile model using adam optimizer and the cateogorical crossentropy loss function
model.compile(optimizer=adam,
loss={'main_output': 'categorical_crossentropy'},
metrics=[
'accuracy',
MeanSquaredError(),
FalseNegatives(),
FalsePositives(),
TrueNegatives(),
TruePositives(),
MeanAbsoluteError(),
Recall(),
Precision()
])
model.summary()
#set earlyStopping and checkpoint callback
earlyStopping = EarlyStopping(monitor='val_loss',
patience=5,
verbose=1,
mode='min')
checkpoint_path = "/blstm_3x1Dconv_dnn_" + str(
datetime.date(datetime.now())) + ".h5"
checkpointer = ModelCheckpoint(filepath=checkpoint_path,
verbose=1,
save_best_only=True,
monitor='val_acc',
mode='max')
return model
def build_model():
"""
Description:
Building DCBGRU model
Args:
None
Returns:
None
"""
#main input is the length of the amino acid in the protein sequence (700,)
main_input = Input(shape=(700, ), dtype='float32', name='main_input')
#Embedding Layer used as input to the neural network
embed = Embedding(output_dim=21, input_dim=21,
input_length=700)(main_input)
#secondary input is the protein profile features
auxiliary_input = Input(shape=(700, 21), name='aux_input')
#get shape of input layers
print("Protein Sequence shape: ", main_input.get_shape())
print("Protein Profile shape: ", auxiliary_input.get_shape())
#concatenate 2 input layers
concat = Concatenate(axis=-1)([embed, auxiliary_input])
#3x1D Convolutional Hidden Layers with BatchNormalization, Dropout and MaxPooling
conv_layer1 = Conv1D(16, 7, kernel_regularizer="l2",
padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer1)
conv_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.2)(conv_act)
max_pool_1D_1 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
conv_layer2 = Conv1D(32, 7, padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer2)
conv_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.2)(conv_act)
max_pool_1D_2 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
conv_layer3 = Conv1D(64, 7, kernel_regularizer="l2",
padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer3)
conv_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.2)(conv_act)
max_pool_1D_3 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
############################################################################################
#concatenate convolutional layers
conv_features = Concatenate(axis=-1)(
[max_pool_1D_1, max_pool_1D_2, max_pool_1D_3])
#dense layer before GRU's
gru_dense = Dense(600, activation='relu',
name="after_cnn_dense")(conv_features)
######## Recurrent Unidirectional Long-Short-Term-Memory Layers ########
gru_f1 = Bidirectional(
GRU(200,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=0.5,
recurrent_dropout=0.5))(gru_dense)
gru_f2 = Bidirectional(
GRU(200,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=0.5,
recurrent_dropout=0.5))(gru_f1)
############################################################################################
#concatenate GRU with convolutional layers
concat_features = Concatenate(axis=-1)([gru_f1, gru_f2, gru_dense])
concat_features = Dropout(0.4)(concat_features)
#Dense Fully-Connected DNN layers
after_gru_dense = Dense(600, activation='relu')(concat_features)
after_gru_dense_dropout = Dropout(0.3)(after_gru_dense)
#Final Dense layer with 8 nodes for the 8 output classifications
main_output = Dense(8, activation='softmax',
name='main_output')(after_gru_dense_dropout)
#create model from inputs and outputs
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output])
#use Adam optimizer
adam = Adam(lr=0.00015)
#compile model using adam optimizer and the cateogorical crossentropy loss function
model.compile(optimizer=adam,
loss={'main_output': 'categorical_crossentropy'},
metrics=[
'accuracy',
MeanSquaredError(),
FalseNegatives(),
FalsePositives(),
TrueNegatives(),
TruePositives(),
MeanAbsoluteError(),
Recall(),
Precision()
])
#print model summary
model.summary()
return model
def build_model():
"""
Description:
Building dummy model
Args:
None
Returns:
None
"""
print("Building dummy model....")
#main input is the length of the amino acid in the protein sequence (700,)
main_input = Input(shape=(700, ), dtype='float32', name='main_input')
#Embedding Layer used as input to the neural network
embed = Embedding(output_dim=21, input_dim=21,
input_length=700)(main_input)
#secondary input is the protein profile features
auxiliary_input = Input(shape=(700, 21), name='aux_input')
#get shape of input layers
print("Protein Sequence shape: ", main_input.get_shape())
print("Protein Profile shape: ", auxiliary_input.get_shape())
#concatenate 2 input layers
concat = Concatenate(axis=-1)([embed, auxiliary_input])
######## 1x1D-Convolutional Layers with BatchNormalization, Dropout and MaxPooling ########
conv_layer1 = Conv1D(16, 7, kernel_regularizer="l2",
padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer1)
# conv2D_act = activations.relu(batch_norm)
conv2D_act = ReLU()(batch_norm)
conv_dropout = Dropout(0.2)(conv2D_act)
############################################################################################
#Final Dense layer with 8 nodes for the 8 output classifications
main_output = Dense(8, activation='softmax',
name='main_output')(conv_dropout)
#create model from inputs and outputs
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output])
#use Adam optimizer
adam = Adam(lr=0.00015)
#compile model using adam optimizer and the cateogorical crossentropy loss function
model.compile(optimizer=adam,
loss={'main_output': 'categorical_crossentropy'},
metrics=[
'accuracy',
MeanSquaredError(),
FalseNegatives(),
FalsePositives(),
TrueNegatives(),
TruePositives(),
MeanAbsoluteError(),
Recall(),
Precision(),
AUC()
])
#print model summary
model.summary()
return model
def CNN(self,
input_shape=None,
stride=(1, 1),
dilation=(1, 1),
kernel_n=3,
pooling_size=(2, 2),
dropout_p=0.2,
n_output=2,
learning_rate=1e-3):
log.log("CNN Model Initilaizing...")
if input_shape != None:
self.kernel_n = kernel_n
self.input_shape = input_shape
self.stride = stride # skips, kernel makes at every convolution
self.dilation = dilation # kernel coverage
self.pooling_size = pooling_size
self.dropout_p = dropout_p
self.n_output = n_output
self.learning_rate = learning_rate
self.model.add(
Conv2D(filters=16,
kernel_size=self.kernel_n,
activation='relu',
padding='same',
input_shape=self.input_shape,
strides=self.stride,
dilation_rate=self.dilation))
#self.model.add(MaxPool2D(pool_size=self.pooling_size))
self.model.add(
Conv2D(filters=32,
kernel_size=self.kernel_n,
activation='relu',
padding='same',
strides=self.stride,
dilation_rate=self.dilation))
#self.model.add(MaxPool2D(pool_size=self.pooling_size))
self.model.add(
Conv2D(filters=64,
kernel_size=self.kernel_n,
activation='relu',
padding='same',
strides=self.stride,
dilation_rate=self.dilation))
#self.model.add(MaxPool2D(pool_size=self.pooling_size))
self.model.add(
Conv2D(filters=128,
kernel_size=self.kernel_n,
activation='relu',
padding='same',
strides=self.stride,
dilation_rate=self.dilation))
#self.model.add(MaxPool2D(pool_size=self.pooling_size))
self.model.add(Flatten())
self.model.add(
Dense(units=self.input_shape[0] * 128, activation='relu'))
self.model.add(Dropout(self.dropout_p))
self.model.add(Dense(units=128, activation='relu'))
self.model.add(Dropout(self.dropout_p))
self.model.add(Dense(units=64, activation='relu'))
self.model.add(Dropout(self.dropout_p))
self.model.add(Dense(units=32, activation='relu'))
self.model.add(Dropout(self.dropout_p))
self.model.add(Dense(units=16, activation='relu'))
self.model.add(Dropout(self.dropout_p))
self.model.add(Dense(units=self.n_output))
self.model.compile(
optimizer=Adam(learning_rate=self.learning_rate),
loss=MeanAbsoluteError(),
metrics=[MeanSquaredError()])
log.log("CNN Model Initialized!")
])
dataset = DatasetGenerator(input_paths, label_paths).get()
dataset_validation = DatasetGenerator(ValInput_paths, ValLabel_paths).get()
# with strategy.scope():
batch_size = 8
model = make_seg_model()
model.summary()
optimizer = tf.keras.optimizers.Adam(1e-4)
model.compile(optimizer=optimizer,
loss=tf.keras.losses.MeanAbsoluteError(),
metrics=[[MeanSquaredError()]])
# ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, model=model)
# checkpoint_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=3, checkpoint_name=modelname)
checkpoint_path = args.checkpoint
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_path,
monitor='val_loss',
mode='auto',
save_best_only=True)
model_early_stopping_callback = tf.keras.callbacks.EarlyStopping(
monitor='val_loss', patience=10)
# model_loss_error_callback = LossAndErrorPrintingCallback()
def build_model_hpconfig(args):
"""
Description:
Building models for hyperparameter Tuning
Args:
args: input arguments
Returns:
model (keras model)
"""
#parsing and assigning hyperparameter variables from argparse
conv1_filters = int(args.conv1_filters)
conv2_filters = int(args.conv2_filters)
conv3_filters = int(args.conv3_filters)
window_size = int(args.window_size)
kernel_regularizer = args.kernel_regularizer
max_pool_size = int(args.pool_size)
conv_dropout = float(args.conv_dropout)
conv1d_initializer = args.conv_weight_initializer
recurrent_layer1 = int(args.recurrent_layer1)
recurrent_layer2 = int(args.recurrent_layer2)
recurrent_dropout = float(args.recurrent_dropout)
after_recurrent_dropout = float(args.after_recurrent_dropout)
recurrent_recurrent_dropout = float(args.recurrent_recurrent_dropout)
recurrent_initalizer = args.recurrent_weight_initializer
optimizer = args.optimizer
learning_rate = float(args.learning_rate)
bidirection = args.bidirection
recurrent_layer = str(args.recurrent_layer)
dense_dropout = float(args.dense_dropout)
dense_1 = int(args.dense_1)
dense_initializer = args.dense_weight_initializer
train_data = str(args.train_input_data)
#main input is the length of the amino acid in the protein sequence (700,)
main_input = Input(shape=(700, ), dtype='float32', name='main_input')
#Embedding Layer used as input to the neural network
embed = Embedding(output_dim=21, input_dim=21,
input_length=700)(main_input)
#secondary input is the protein profile features
auxiliary_input = Input(shape=(700, 21), name='aux_input')
#get shape of input layers
print("Protein Sequence shape: ", main_input.get_shape())
print("Protein Profile shape: ", auxiliary_input.get_shape())
#concatenate input layers
concat = Concatenate(axis=-1)([embed, auxiliary_input])
#3x1D Convolutional Hidden Layers with BatchNormalization, Dropout and MaxPooling
conv_layer1 = Conv1D(conv1_filters,
window_size,
kernel_regularizer=kernel_regularizer,
padding='same',
kernel_initializer=conv1d_initializer)(concat)
batch_norm = BatchNormalization()(conv_layer1)
conv_act = activations.relu(batch_norm)
conv_dropout = Dropout(conv_dropout)(conv_act)
max_pool_1D_1 = MaxPooling1D(pool_size=max_pool_size,
strides=1,
padding='same')(conv_dropout)
conv_layer2 = Conv1D(conv2_filters,
window_size,
padding='same',
kernel_initializer=conv1d_initializer)(concat)
batch_norm = BatchNormalization()(conv_layer2)
conv_act = activations.relu(batch_norm)
conv_dropout = Dropout(conv_dropout)(conv_act)
max_pool_1D_2 = MaxPooling1D(pool_size=max_pool_size,
strides=1,
padding='same')(conv_dropout)
conv_layer3 = Conv1D(conv3_filters,
window_size,
kernel_regularizer=kernel_regularizer,
padding='same',
kernel_initializer=conv1d_initializer)(concat)
batch_norm = BatchNormalization()(conv_layer3)
conv_act = activations.relu(batch_norm)
conv_dropout = Dropout(conv_dropout)(conv_act)
max_pool_1D_3 = MaxPooling1D(pool_size=max_pool_size,
strides=1,
padding='same')(conv_dropout)
#concat pooling layers
conv_features = Concatenate(axis=-1)(
[max_pool_1D_1, max_pool_1D_2, max_pool_1D_3])
print("Shape of convolutional output: ", conv_features.get_shape())
conv_features = Dense(600, activation='relu')(conv_features)
######## Recurrent Layers ########
if (recurrent_layer == 'lstm'):
if (bidirection):
print('Entering LSTM Layers')
#Creating Bidirectional LSTM layers
lstm_f1 = Bidirectional(
LSTM(recurrent_layer1,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=recurrent_dropout,
recurrent_dropout=recurrent_recurrent_dropout,
kernel_initializer=recurrent_initalizer))(conv_features)
lstm_f2 = Bidirectional(
LSTM(recurrent_layer2,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=recurrent_dropout,
recurrent_dropout=recurrent_recurrent_dropout,
kernel_initializer=recurrent_initalizer))(lstm_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)(
[lstm_f1, lstm_f2, conv_features])
concat_features = Dropout(after_recurrent_dropout)(concat_features)
print('Concatenated LSTM layers')
else:
#Creating unidirectional LSTM Layers
lstm_f1 = LSTM(
recurrent_layer1,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=recurrent_dropout,
recurrent_dropout=recurrent_recurrent_dropout,
kernel_initializer=recurrent_initalizer)(conv_features)
lstm_f2 = LSTM(recurrent_layer2,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=recurrent_dropout,
recurrent_dropout=recurrent_recurrent_dropout,
kernel_initializer=recurrent_initalizer)(lstm_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)(
[lstm_f1, lstm_f2, conv_features])
concat_features = Dropout(after_recurrent_dropout)(concat_features)
elif (recurrent_layer == 'gru'):
if (bidirection):
#Creating Bidirectional GRU layers
gru_f1 = Bidirectional(
GRU(recurrent_layer1,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=recurrent_dropout,
recurrent_dropout=recurrent_recurrent_dropout,
kernel_initializer=recurrent_initalizer))(conv_features)
gru_f2 = Bidirectional(
GRU(recurrent_layer2,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=recurrent_dropout,
recurrent_dropout=recurrent_recurrent_dropout,
kernel_initializer=recurrent_initalizer))(gru_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)(
[gru_f1, gru_f2, conv_features])
concat_features = Dropout(after_recurrent_dropout)(concat_features)
else:
#Creating unidirectional GRU Layers
gru_f1 = GRU(
recurrent_layer1,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=recurrent_dropout,
recurrent_dropout=recurrent_recurrent_dropout,
kernel_initializer=recurrent_initalizer)(conv_features)
gru_f2 = GRU(recurrent_layer1,
return_sequences=True,
activation='tanh',
recurrent_activation='sigmoid',
dropout=recurrent_dropout,
recurrent_dropout=recurrent_recurrent_dropout,
kernel_initializer=recurrent_initalizer)(gru_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)(
[gru_f1, gru_f2, conv_features])
concat_features = Dropout(after_recurrent_dropout)(concat_features)
else:
print('Only LSTM and GRU recurrent layers are used in this model')
return
#Dense Fully-Connected DNN layers
fc_dense1 = Dense(dense_1,
activation='relu',
kernel_initializer=dense_initializer)(concat_features)
fc_dense1_dropout = Dropout(dense_dropout)(fc_dense1)
#Final Output layer with 8 nodes for the 8 output classifications
main_output = Dense(8, activation='softmax',
name='main_output')(fc_dense1_dropout)
#create model from inputs and outputs
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output])
#Set optimizer to be used with the model, default is Adam
if optimizer == 'adam':
optimizer = Adam(lr=learning_rate, name='adam')
elif optimizer == 'sgd':
optimizer = SGD(lr=0.01, momentum=0.0, nesterov=False, name='SGD')
elif optimizer == 'rmsprop':
optimizer = RMSprop(learning_rate=learning_rate,
centered=True,
name='RMSprop')
elif optimizer == 'adagrad':
optimizer = Adagrad(learning_rate=learning_rate, name='Adagrad')
elif optimizer == 'adamax':
optimizer = Adamax(learning_rate=learning_rate, name='Adamax')
else:
optimizer = 'adam'
optimizer = Adam(lr=learning_rate, name='adam')
#compile model using optimizer and the cateogorical crossentropy loss function
model.compile(optimizer=optimizer,
loss={'main_output': 'categorical_crossentropy'},
metrics=[
'accuracy',
MeanSquaredError(),
FalseNegatives(),
FalsePositives(),
TrueNegatives(),
TruePositives(),
MeanAbsoluteError(),
Recall(),
Precision()
])
#get summary of model including its layers and num parameters
model.summary()
return model
def build_model():
"""
Description:
Building PSP-CD model
Args:
None
Returns:
None
"""
#main input is the length of the amino acid in the protein sequence (700,)
main_input = Input(shape=(700, ), dtype='float32', name='main_input')
#Embedding Layer used as input to the neural network
embed = Embedding(output_dim=21, input_dim=21,
input_length=700)(main_input)
#secondary input is the protein profile features
auxiliary_input = Input(shape=(700, 21), name='aux_input')
#get shape of input layers
print("Protein Sequence shape: ", main_input.get_shape())
print("Protein Profile shape: ", auxiliary_input.get_shape())
#concatenate 2 input layers
concat_features = Concatenate(axis=-1)([embed, auxiliary_input])
############################################################################################
#Dense Fully-Connected DNN layers
dense_1 = Dense(512, activation='relu')(concat_features)
dense_1_dropout = Dropout(0.3)(dense_1)
dense_2 = Dense(256, activation='relu')(dense_1_dropout)
dense_2_dropout = Dropout(0.3)(dense_2)
dense_3 = Dense(128, activation='relu')(dense_2_dropout)
dense_3_dropout = Dropout(0.3)(dense_3)
dense_4 = Dense(64, activation='relu')(dense_3_dropout)
dense_4_dropout = Dropout(0.3)(dense_4)
dense_5 = Dense(32, activation='relu')(dense_4_dropout)
dense_5_dropout = Dropout(0.3)(dense_5)
dense_6 = Dense(16, activation='relu')(dense_5_dropout)
dense_6_dropout = Dropout(0.3)(dense_6)
#Final Dense layer with 8 nodes for the 8 output classifications
main_output = Dense(8, activation='softmax',
name='main_output')(dense_6_dropout)
#create model from inputs and outputs
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output])
#use Adam optimizer
adam = Adam(lr=0.00015)
#compile model using adam optimizer and the cateogorical crossentropy loss function
model.compile(optimizer=adam,
loss={'main_output': 'categorical_crossentropy'},
metrics=[
'accuracy',
MeanSquaredError(),
FalseNegatives(),
FalsePositives(),
TrueNegatives(),
TruePositives(),
MeanAbsoluteError(),
Recall(),
Precision()
])
#print model summary
model.summary()
return model
def build_model():
#main input is the length of the amino acid in the protein sequence (700,)
main_input = Input(shape=(700, ), dtype='float32', name='main_input')
#Embedding Layer used as input to the neural network
embed = Embedding(output_dim=21, input_dim=21,
input_length=700)(main_input)
#secondary input is the protein profile features
auxiliary_input = Input(shape=(700, 21), name='aux_input')
#get shape of input layers
print("Protein Sequence shape: ", main_input.get_shape())
print("Protein Profile shape: ", auxiliary_input.get_shape())
#concatenate 2 input layers
concat = Concatenate(axis=-1)([embed, auxiliary_input])
#3x1D Convolutional Hidden Layers with BatchNormalization and MaxPooling
conv_layer1 = Conv1D(64, 7, kernel_regularizer="l2",
padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer1)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.5)(conv2D_act)
# ave_pool_1 = AveragePooling1D(2, 1, padding='same')(conv_dropout)
max_pool_1D_1 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
conv_layer2 = Conv1D(128, 7, padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer2)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.5)(conv2D_act)
# ave_pool_2 = AveragePooling1D(2, 1, padding='same')(conv_dropout)
max_pool_1D_2 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
conv_layer3 = Conv1D(256, 7, kernel_regularizer="l2",
padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer3)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.5)(conv2D_act)
max_pool_1D_3 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
# ave_pool_3 = AveragePooling1D(2, 1, padding='same')(conv_dropout)
#concatenate convolutional layers
conv_features = Concatenate(axis=-1)(
[max_pool_1D_1, max_pool_1D_2, max_pool_1D_3])
#output node is 1D convolutional layer with 8 filters for the 8 different categories
main_output = Conv1D(8,
7,
padding='same',
activation='softmax',
name='main_output')(conv_features)
#create model from inputs and outputs
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output])
#use Adam optimizer
adam = Adam(lr=0.0003)
#compile model using adam optimizer and the cateogorical crossentropy loss function
model.compile(optimizer=adam,
loss={'main_output': 'categorical_crossentropy'},
metrics=[
'accuracy',
MeanSquaredError(),
FalseNegatives(),
FalsePositives(),
TrueNegatives(),
TruePositives(),
MeanAbsoluteError(),
Recall(),
Precision()
])
model.summary()
#set earlyStopping and checkpoint callback
earlyStopping = EarlyStopping(monitor='val_loss',
patience=5,
verbose=1,
mode='min')
checkpoint_path = "checkpoints/3xConv_cnn_" + str(
datetime.date(datetime.now())) + ".h5"
checkpointer = ModelCheckpoint(filepath=checkpoint_path,
verbose=1,
save_best_only=True,
monitor='val_acc',
mode='max')
return model
def train_aae(configs):
batch_size = configs["batch_size"]
dataset_name = configs["dataset_name"]
filter = configs["filter"]
label_name = configs["label_name"]
train, val = load_dataset(dataset_name,
sets=["train", "val"],
shuffle=False,
label_name=label_name,
batch_size=batch_size,
filter=filter)
epochs = configs["epochs"]
latent_dim = configs["latent_dim"]
intermediate_dim = configs["intermediate_dim"]
use_clf_label = configs["use_clf_label"]
weight = configs["reconstruction_weight"]
loss_weights = [
weight, (1 - weight) / 3, (1 - weight) / 3, (1 - weight) / 3
]
distance_thresh = configs["distance_thresh"]
classification_model = tf.keras.models.load_model("../models/{}_{}".format(
configs["clf_type"], dataset_name))
# if not hyperparameter tuning, set up tensorboard
logdir = "tensorboard_logs/aae/" + datetime.now().strftime("%Y%m%d-%H%M%S")
train_summary_writer = tf.summary.create_file_writer(logdir)
with open("../data/{}/metadata.txt".format(dataset_name)) as file:
metadata = json.load(file)
num_classes = metadata["num_classes"]
field_names = metadata["field_names"][:-1]
input_dim = len(field_names)
datamin = np.array(metadata["col_min"][:-1])
datamax = np.array(metadata["col_max"][:-1])
scaler, unscaler = min_max_scaler_gen(datamin, datamax)
data_range = datamax - datamin
#
packed_train_data = train.map(
PackNumericFeatures(field_names, num_classes, scaler=scaler))
packed_val_data = val.map(
PackNumericFeatures(field_names, num_classes, scaler=scaler))
# Create a MirroredStrategy.
# strategy = tf.distribute.MirroredStrategy()
# print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
#
# # Open a strategy scope.
# with strategy.scope():
# oop version does not provide a good graph trace so using functional instead
aae, encoder, decoder, latent_discriminator, cat_discriminator = build_aae_dim_reduce(
input_dim, intermediate_dim, latent_dim, num_classes, distance_thresh,
None, None, loss_weights)
# wmse=weighted_mse(data_range)
aae.compile(loss=[
tf.keras.losses.MeanSquaredError(),
tf.keras.losses.BinaryCrossentropy(),
tf.keras.losses.BinaryCrossentropy(),
tf.keras.losses.MeanSquaredError()
],
loss_weights=loss_weights,
optimizer='adam',
metrics=[[MeanSquaredError(name="latent_mse")],
[BinaryAccuracy(name="latent_acc")],
[BinaryAccuracy(name="label_acc")],
[MeanSquaredError(name="label_mse")]])
valid = np.ones((batch_size, 1))
invalid = np.zeros((batch_size, 1))
step = 0
pbar = tqdm(range(epochs), desc="epoch")
for epoch in pbar:
steps = metadata["num_train"] // batch_size
step_pbar = tqdm(total=steps, desc="steps", leave=False, position=1)
for feature, label in packed_train_data.take(steps):
fake_latent, fake_cat = encoder(feature)
# train latent discriminator
latent_discriminator.trainable = True
real_latent = sample_prior(batch_size,
distro="normal",
latent_dim=latent_dim)
noise_real = np.random.normal(0,
0.5,
size=(batch_size, latent_dim))
d_loss_real = latent_discriminator.train_on_batch(
real_latent + noise_real, valid)
d_loss_fake = latent_discriminator.train_on_batch(
fake_latent, invalid)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
latent_discriminator.trainable = False
# train cat discriminator
cat_discriminator.trainable = True
if use_clf_label:
real_cat = classification_model(feature)
real_cat = np.argmax(real_cat, axis=-1)
real_cat = np.eye(num_classes)[real_cat]
else:
real_cat = label
cat_loss_real = cat_discriminator.train_on_batch(real_cat, valid)
cat_loss_fake = cat_discriminator.train_on_batch(fake_cat, invalid)
cat_loss = 0.5 * np.add(cat_loss_real, cat_loss_fake)
cat_discriminator.trainable = False
# train generator
g_loss = aae.train_on_batch(feature,
[feature, valid, valid, real_cat])
# record losses if not tuning
with train_summary_writer.as_default():
tf.summary.scalar('latent loss', d_loss[0], step=step)
tf.summary.scalar('latent acc', d_loss[1], step=step)
tf.summary.scalar('cat loss', cat_loss[0], step=step)
tf.summary.scalar('cat acc', cat_loss[1], step=step)
for i in range(len(aae.metrics_names)):
tf.summary.scalar(aae.metrics_names[i],
g_loss[i],
step=step)
step += 1
step_pbar.update(1)
# record distribution after epoch
style, label = encoder(feature)
with train_summary_writer.as_default():
tf.summary.histogram('cat_disc_out',
cat_discriminator(label),
step=step)
tf.summary.histogram('lat_disc_out',
latent_discriminator(style),
step=step)
tf.summary.histogram('style', style, step=step)
tf.summary.histogram('label', label, step=step)
tf.summary.histogram('prior style',
real_latent + noise_real,
step=step)
tf.summary.histogram('prior label', real_cat, step=step)
step_pbar.reset()
postfix = {
"latent_acc": 100 * d_loss[1],
"cat_acc": 100 * cat_loss[1],
"mse": g_loss[5]
}
pbar.set_postfix(postfix)
# trace aae with dummy value and save model
feature, label = list(packed_val_data.take(1).as_numpy_iterator())[0]
feature = np.zeros((1, input_dim))
label = np.zeros((num_classes))
latent = np.zeros((1, latent_dim))
# trace aae
with train_summary_writer.as_default():
tf.summary.trace_on(graph=True, profiler=True)
tracer(aae, feature)
tf.summary.trace_export(name="aae", step=0, profiler_outdir=logdir)
# tf.keras.models.save_model(aae, "../models/aae/test")
prefix = "{}_{}_{}".format(dataset_name, latent_dim, use_clf_label)
if not os.path.isdir("../models/aae"):
os.makedirs("../models/aae")
aae.save("../models/aae/{}_aae.h5".format(prefix))
encoder.save("../models/aae/{}_encoder.h5".format(prefix))
decoder.save("../models/aae/{}_decoder.h5".format(prefix))
latent_discriminator.save("../models/aae/{}_lat_disc.h5".format(prefix))
cat_discriminator.save("../models/aae/{}_cat_disc.h5".format(prefix))
def build_model_hpconfig(args):
#parsing and assigning hyperparameter variables from argparse
conv1_filters=int(args.conv1_filters)
conv2_filters=int(args.conv2_filters)
conv3_filters=int(args.conv3_filters)
window_size=int(args.window_size)
kernel_regularizer = args.kernel_regularizer
conv_dropout=float(args.conv2d_dropout)
pool_size = int(args.pool_size)
conv2d_activation=args.conv2d_activation
conv2d_dropout=float(args.conv2d_dropout)
recurrent_layer1 = int(args.recurrent_layer1)
recurrent_layer2 = int(args.recurrent_layer2)
recurrent_dropout = float(args.recurrent_dropout)
after_recurrent_dropout = float(args.after_recurrent_dropout)
recurrent_recurrent_dropout = float(args.recurrent_recurrent_dropout)
optimizer=args.optimizer
learning_rate = float(args.learning_rate)
bidirection = args.bidirection
recurrent_layer = args.recurrent_layer
dense_dropout = float(args.dense_dropout)
dense_1 = int(args.dense_1)
dense_2 = int(args.dense_2)
dense_3 = int(args.dense_3)
dense_4 = int(args.dense_4)
#main input is the length of the amino acid in the protein sequence (700,)
main_input = Input(shape=(700,), dtype='float32', name='main_input')
#Embedding Layer used as input to the neural network
embed = Embedding(output_dim=21, input_dim=21, input_length=700)(main_input)
#secondary input is the protein profile features
auxiliary_input = Input(shape=(700,21), name='aux_input')
#concatenate input layers
concat = Concatenate(axis=-1)([embed, auxiliary_input])
conv_layer1 = Convolution1D(conv1_filters, window_size, kernel_regularizer = "l2", padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer1)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(conv_dropout)(conv2D_act)
# ave_pool_1 = AveragePooling1D(3, 1, padding='same')(conv_dropout)
max_pool_1D_1 = MaxPooling1D(pool_size=pool_size, strides=1, padding='same')(conv_dropout)
conv_layer2 = Convolution1D(conv2_filters, window_size, padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer2)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(conv_dropout)(conv2D_act)
# ave_pool_2 = AveragePooling1D(3, 1, padding='same')(conv_dropout)
max_pool_1D_2 = MaxPooling1D(pool_size=pool_size, strides=1, padding='same')(conv_dropout)
conv_layer3 = Convolution1D(conv3_filters, window_size,kernel_regularizer = "l2", padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer3)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(conv_dropout)(conv2D_act)
max_pool_1D_3 = MaxPooling1D(pool_size=pool_size, strides=1, padding='same')(conv_dropout)
#concat pooling layers
conv_features = Concatenate(axis=-1)([max_pool_1D_1, max_pool_1D_2, max_pool_1D_3])
######## Recurrent Layers ########
if (recurrent_layer == 'lstm'):
if (bidirection):
#Creating Bidirectional LSTM layers
lstm_f1 = Bidirectional(LSTM(recurrent_layer1,return_sequences=True,activation = 'tanh', recurrent_activation='sigmoid',dropout=recurrent_dropout, recurrent_dropout=recurrent_recurrent_dropout))(conv_features)
lstm_f2 = Bidirectional(LSTM(recurrent_layer2, return_sequences=True,activation = 'tanh',recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout))(lstm_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)([lstm_f1, lstm_f2, conv2_features])
concat_features = Dropout(after_recurrent_dropout)(concat_features)
else:
#Creating unidirectional LSTM Layers
lstm_f1 = LSTM(recurrent_layer1,return_sequences=True,activation = 'tanh', recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout)(conv_features)
lstm_f2 = LSTM(recurrent_layer2, return_sequences=True,activation = 'tanh',recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout)(lstm_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)([lstm_f1, lstm_f2, conv_features])
concat_features = Dropout(after_recurrent_dropout)(concat_features)
elif (recurrent_layer == 'gru'):
if (bidirection):
#Creating Bidirectional GRU layers
gru_f1 = Bidirectional(GRU(recurrent_layer1,return_sequences=True,activation = 'tanh', recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout))(conv_features)
gru_f2 = Bidirectional(GRU(recurrent_layer2, return_sequences=True,activation = 'tanh',recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout))(gru_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)([gru_f1, gru_f2, conv_features])
concat_features = Dropout(after_recurrent_dropout)(concat_features)
else:
#Creating unidirectional GRU Layers
gru_f1 = GRU(recurrent_layer1,return_sequences=True,activation = 'tanh', recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout)(conv_features)
gru_f2 = GRU(recurrent_layer1, return_sequences=True,activation = 'tanh',recurrent_activation='sigmoid',dropout=recurrent_dropout,recurrent_dropout=recurrent_recurrent_dropout)(gru_f1)
#concatenate LSTM with convolutional layers
concat_features = Concatenate(axis=-1)([gru_f1, gru_f2, conv_features])
concat_features = Dropout(after_recurrent_dropout)(concat_features)
else:
print('Only LSTM and GRU recurrent layers are used in this model')
return
#Dense Fully-Connected DNN layers
# concat_features = Flatten()(concat_features)
fc_dense1 = Dense(dense_1, activation='relu')(concat_features)
fc_dense1_dropout = Dropout(dense_dropout)(fc_dense1)
fc_dense2 = Dense(dense_2, activation='relu')(fc_dense1_dropout)
fc_dense2_dropout = Dropout(dense_dropout)(fc_dense2)
fc_dense3 = Dense(dense_3, activation='relu')(fc_dense2_dropout)
fc_dense3_dropout = Dropout(dense_dropout)(fc_dense3)
#Final Output layer with 8 nodes for the 8 output classifications
# main_output = Dense(8, activation='softmax', name='main_output')(concat_features)
main_output = Dense(8, activation='softmax', name='main_output')(fc_dense3_dropout)
#create model from inputs and outputs
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output])
#Set optimizer to be used with the model, default is Adam
if optimizer == 'adam':
optimizer = Adam(lr=learning_rate, name='adam')
elif optimizer == 'sgd':
optimizer = SGD(lr=0.01, momentum=0.0, nestero=False, name='SGD')
elif optimizer == 'rmsprop':
optimizer = RMSprop(learning_rate=learning_rate, centered = True, name='RMSprop')
elif optimizer == 'adagrad':
optimizer = Adagrad(learning_rate = learning_rate, name='Adagrad')
elif optimizer == 'adamax':
optimizer = Adamax(learning_rate=learning_rate, name='Adamax')
else:
optimizer = 'adam'
optimizer = Adam(lr=learning_rate, name='adam')
#Nadam & Ftrl optimizers
#use Adam optimizer
#optimizer = Adam(lr=0.003)
#Adam is fast, but tends to over-fit
#SGD is low but gives great results, sometimes RMSProp works best, SWA can easily improve quality, AdaTune
#compile model using optimizer and the cateogorical crossentropy loss function
model.compile(optimizer = optimizer, loss={'main_output': 'categorical_crossentropy'}, metrics=['accuracy', MeanSquaredError(), FalseNegatives(), FalsePositives(), TrueNegatives(), TruePositives(), MeanAbsoluteError(), Recall(), Precision()])
#get summary of model including its layers and num parameters
model.summary()
#set early stopping and checkpoints for model
earlyStopping = EarlyStopping(monitor='val_loss', patience=5, verbose=1, mode='min')
checkpoint_path = BUCKET_PATH + "/checkpoints/" + str(datetime.date(datetime.now())) +\
'_' + str((datetime.now().strftime('%H:%M'))) + ".h5"
checkpointer = ModelCheckpoint(filepath=checkpoint_path,verbose=1,save_best_only=True, monitor='val_acc', mode='max')
return model
def build_model():
#main input is the length of the amino acid in the protein sequence (700,)
main_input = Input(shape=(700, ), dtype='float32', name='main_input')
#Embedding Layer used as input to the neural network
embed = Embedding(output_dim=21, input_dim=21,
input_length=700)(main_input)
#secondary input is the protein profile features
auxiliary_input = Input(shape=(700, 21), name='aux_input')
#get shape of input layers
print("Protein Sequence shape: ", main_input.get_shape())
print("Protein Profile shape: ", auxiliary_input.get_shape())
#concatenate 2 input layers
concat = Concatenate(axis=-1)([embed, auxiliary_input])
#3x1D Convolutional Hidden Layers with BatchNormalization and MaxPooling
conv_layer1 = Convolution1D(64, 7, kernel_regularizer="l2",
padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer1)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.5)(conv2D_act)
max_pool_2D_1 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
conv_layer2 = Convolution1D(128, 7, padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer2)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.5)(conv2D_act)
max_pool_2D_2 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
conv_layer3 = Convolution1D(256,
7,
kernel_regularizer="l2",
padding='same')(concat)
batch_norm = BatchNormalization()(conv_layer3)
conv2D_act = activations.relu(batch_norm)
conv_dropout = Dropout(0.5)(conv2D_act)
max_pool_2D_3 = MaxPooling1D(pool_size=2, strides=1,
padding='same')(conv_dropout)
#concatenate convolutional layers
conv_features = Concatenate(axis=-1)(
[max_pool_2D_1, max_pool_2D_2, max_pool_2D_3])
#Dense Fully-Connected DNN layers
dense_1 = Dense(300, activation='relu')(conv_features)
dense_1_dropout = Dropout(dense_dropout)(dense_1)
dense_2 = Dense(100, activation='relu')(dense_1_dropout)
dense_2_dropout = Dropout(dense_dropout)(dense_2)
dense_3 = Dense(50, activation='relu')(dense_2_dropout)
dense_3_dropout = Dropout(dense_dropout)(dense_3)
dense_4 = Dense(16, activation='relu')(dense_3_dropout)
dense_4_dropout = Dropout(dense_dropout)(dense_4)
#Final Dense layer with 8 nodes for the 8 output classifications
main_output = Dense(8, activation='softmax',
name='main_output')(protein_features_dropout)
#create model from inputs and outputs
model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output])
#use Adam optimizer
adam = Adam(lr=lr)
#Adam is fast, but tends to over-fit
#SGD is low but gives great results, sometimes RMSProp works best, SWA can easily improve quality, AdaTune
#compile model using adam optimizer and the cateogorical crossentropy loss function
model.compile(optimizer=adam,
loss={'main_output': 'categorical_crossentropy'},
metrics=[
'accuracy',
MeanSquaredError(),
FalseNegatives(),
FalsePositives(),
TrueNegatives(),
TruePositives(),
MeanAbsoluteError(),
Recall(),
Precision()
])
model.summary()
#set earlyStopping and checkpoint callback
earlyStopping = EarlyStopping(monitor='val_loss',
patience=5,
verbose=1,
mode='min')
checkpoint_path = "/3x1DConv_dnn_" + str(datetime.date(
datetime.now())) + ".h5"
checkpointer = ModelCheckpoint(filepath=checkpoint_path,
verbose=1,
save_best_only=True,
monitor='val_acc',
mode='max')
return model