def model_c_cnn_bilstm_crf(n_classes,
convs=[3, 5, 7],
dense_size=200,
lstm_size=400,
dropout_rate=0.5,
features_to_use=['onehot', 'pssm'],
filter_size=256,
CRF_input_dim=200,
lr=0.001):
'''
:param n_classes:
:param convs:
:param dense_size:
:param lstm_size:
:param dropout_rate:
:param features_to_use:
:param filter_size:
:return:
'''
visible = Input(shape=(None, 408))
# slice different feature types
biophysical = slice_tensor(2, 0, 16, name='biophysicalfeatures')(visible)
embedding = slice_tensor(2, 16, 66, name='skipgramembd')(visible)
onehot = slice_tensor(2, 66, 87, name='onehot')(visible)
pssm = slice_tensor(2, 87, 108, name='pssm')(visible)
elmo = slice_tensor(2, 108, 408, name='elmo')(visible)
# create input based-on selected features
input_dict = {
'pssm': pssm,
'onehot': onehot,
'embedding': embedding,
'elmo': elmo,
'biophysical': biophysical
}
features = []
for feature in features_to_use:
features.append(input_dict[feature])
## batch normalization on the input features
if len(features_to_use) == 1:
conclayers = features
input = BatchNormalization(name='batchnorm_input')(features[0])
else:
input = BatchNormalization(name='batchnorm_input')(
concatenate(features))
conclayers = [input]
# performing the conlvolutions
for idx, conv in enumerate(convs):
idx = str(idx + 1)
conclayers.append(
BatchNormalization(name='batch_norm_conv' + idx)(Conv1D(
filter_size,
conv,
activation="relu",
padding="same",
name='conv' + idx,
kernel_regularizer=regularizers.l2(0.001))(input)))
conc = concatenate(conclayers)
# Dropout and Dense Layer before LSTM
if dropout_rate > 0:
drop_before = Dropout(dropout_rate, name='dropoutonconvs')(conc)
dense_convinp = Dense(dense_size,
activation='relu',
name='denseonconvs')(drop_before)
else:
dense_convinp = Dense(dense_size,
activation='relu',
name='denseonconvs')(conc)
# Batch normalize the results of dropout
dense_convinpn = BatchNormalization(name='batch_norm_dense')(dense_convinp)
# LSTM
lstm = Bidirectional(
CuDNNLSTM(lstm_size, return_sequences=True,
name='bilstm'))(dense_convinpn)
drop_after_lstm = Dropout(dropout_rate)(lstm)
dense_out = Dense(CRF_input_dim, activation='relu')(drop_after_lstm)
timedist = TimeDistributed(Dense(n_classes, name='crf_in'))(dense_out)
crf = ChainCRF(name="crf1")
crf_output = crf(timedist)
model = Model(inputs=visible, outputs=crf_output)
adam = optimizers.Adam(lr=lr)
model.compile(loss=crf.loss,
optimizer=adam,
weighted_metrics=['accuracy'],
sample_weight_mode='temporal')
print(model.summary())
return model, 'model_c_cnn_bilstm_CRF#' + '#'.join(
features_to_use) + '@conv' + '_'.join(
[str(c) for c in convs]) + '@dense_' + str(
dense_size) + '@lstm' + str(lstm_size) + '@droplstm' + str(
dropout_rate) + '@filtersize_' + str(
filter_size) + '@lr_' + str(lr)
def model_f_multiscale_cnn(n_classes,
convs=[3, 5, 7],
dropout_rate=0.5,
features_to_use=['onehot', 'pssm'],
filter_size=256,
lr=0.001,
multiscalecnn_layers=3,
cnn_regularizer=0.00005,
use_CRF=False):
'''
:param n_classes:
:param convs:
:param dropout_rate:
:param features_to_use:
:param filter_size:
:param lr:
:param multicnn_layers:
:param cnn_regularizer:
:param use_CRF:
:return:
'''
visible = Input(shape=(None, 408))
# slice different feature types
biophysical = slice_tensor(2, 0, 16, name='biophysicalfeatures')(visible)
embedding = slice_tensor(2, 16, 66, name='skipgramembd')(visible)
onehot = slice_tensor(2, 66, 87, name='onehot')(visible)
pssm = slice_tensor(2, 87, 108, name='pssm')(visible)
elmo = slice_tensor(2, 108, 408, name='elmo')(visible)
input_dict = {
'pssm': pssm,
'onehot': onehot,
'embedding': embedding,
'elmo': elmo,
'biophysical': biophysical
}
gating = Dense(len(convs) * filter_size, activation='sigmoid')
# create input
features = []
for feature in features_to_use:
features.append(input_dict[feature])
if len(features_to_use) == 1:
conclayers = features
input = BatchNormalization(name='batchnorm_input')(features[0])
else:
input = BatchNormalization(name='batchnorm_input')(
concatenate(features))
conclayers = []
# performing the conlvolutions
for idx, conv in enumerate(convs):
idx = str(idx + 1)
conclayers.append(
Conv1D(filter_size,
conv,
activation="relu",
padding="same",
name='conv' + idx,
kernel_regularizer=regularizers.l2(cnn_regularizer))(input))
current_multi_cnn_input = concatenate(conclayers)
# Multiscale CNN application
for layer_idx in range(multiscalecnn_layers - 1):
current_multi_cnn_output = multiscale_CNN(current_multi_cnn_input,
gating, filter_size, convs,
cnn_regularizer)
current_multi_cnn_input = Dropout(dropout_rate)(
current_multi_cnn_output)
dense_out = Dense(len(convs) * filter_size,
activation='relu')(current_multi_cnn_input)
if use_CRF:
timedist = TimeDistributed(Dense(n_classes,
name='timedist'))(dense_out)
crf = ChainCRF(name="crf1")
crf_output = crf(timedist)
model = Model(inputs=visible, outputs=crf_output)
adam = optimizers.Adam(lr=lr)
model.compile(loss=crf.loss,
optimizer=adam,
weighted_metrics=['accuracy'],
sample_weight_mode='temporal')
else:
timedist = TimeDistributed(Dense(n_classes,
activation='softmax'))(dense_out)
model = Model(inputs=visible, outputs=timedist)
adam = optimizers.Adam(lr=lr)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
weighted_metrics=['accuracy'],
sample_weight_mode='temporal')
print(model.summary())
return model, 'model_f_multiscale_cnn#' + '#'.join(
features_to_use) + '@conv' + '_'.join([
str(c) for c in convs
]) + '@dropout_rate' + str(dropout_rate) + '@filtersize_' + str(
filter_size) + '@lr_' + str(lr) + '@use_CRF_' + str(
use_CRF) + '@multiscalecnn_layers' + str(
multiscalecnn_layers) + '@cnn_regularizer' + str(
cnn_regularizer)