def build_model(max_features, maxlen):
'''
Derived CNN model from Keegan Hines' Snowman
https://github.com/keeganhines/snowman/
'''
text_input = Input(shape=(maxlen, ), name='text_input')
x = Embedding(input_dim=max_features, input_length=maxlen,
output_dim=128)(text_input)
conv_a = Conv1D(15, 2, activation='relu')(x)
conv_b = Conv1D(15, 3, activation='relu')(x)
conv_c = Conv1D(15, 4, activation='relu')(x)
conv_d = Conv1D(15, 5, activation='relu')(x)
conv_e = Conv1D(15, 6, activation='relu')(x)
pool_a = GlobalMaxPooling1D()(conv_a)
pool_b = GlobalMaxPooling1D()(conv_b)
pool_c = GlobalMaxPooling1D()(conv_c)
pool_d = GlobalMaxPooling1D()(conv_d)
pool_e = GlobalMaxPooling1D()(conv_e)
flattened = concatenate([pool_a, pool_b, pool_c, pool_d, pool_e])
drop = Dropout(.2)(flattened)
dense = Dense(1)(drop)
out = Activation("sigmoid")(dense)
model = Model(inputs=text_input, outputs=out)
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
def GRUdeep(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
x = Activation('tanh')(embed_input_)
x = SpatialDropout1D(0.1, name='embed_drop')(x)
if params['bidirectional']:
x = Bidirectional(CuDNNGRU(params['lstm_units'],
return_sequences=True))(x)
x = Bidirectional(CuDNNGRU(params['lstm_units'],
return_sequences=True))(x)
x = GlobalMaxPooling1D()(x)
else:
x = CuDNNGRU(params['lstm_units'], return_sequences=True)(x)
x = CuDNNGRU(params['lstm_units'], return_sequences=True)(x)
x = GlobalMaxPooling1D()(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(256, activation='relu')(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def text_cnn(max_seq_index, max_seq_length):
text_input = Input(shape = (max_seq_length,), name='text_input')
x = Embedding(output_dim=15,
input_dim=max_seq_index,
input_length=max_seq_length)(text_input)
conv_a = Conv1D(15,2, activation='relu')(x)
conv_b = Conv1D(15,4, activation='relu')(x)
conv_c = Conv1D(15,6, activation='relu')(x)
pool_a = GlobalMaxPooling1D()(conv_a)
pool_b = GlobalMaxPooling1D()(conv_b)
pool_c = GlobalMaxPooling1D()(conv_c)
flattened = concatenate(
[pool_a, pool_b, pool_c])
drop = Dropout(.2)(flattened)
dense = Dense(1)(drop)
out = Activation("sigmoid")(dense)
model = Model(inputs=text_input, outputs=out)
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
def build_model(self):
input1 = Input(shape=(self.config.max_sent_len,
self.config.word_emb_dim))
input2 = Input(shape=(self.config.max_sent_len,
self.config.word_emb_dim))
lstm1 = LSTM(300,
dropout=0.3,
activity_regularizer=l1_l2(0.01),
return_sequences=True,
activation='relu')(input1)
lstm2 = LSTM(300,
dropout=0.3,
activity_regularizer=l1_l2(0.01),
return_sequences=True,
activation='relu')(input2)
pool1 = GlobalMaxPooling1D()(lstm1)
pool2 = GlobalMaxPooling1D()(lstm2)
diffvec = merge([pool1, pool2], mode=abs_diff, output_shape=(300, ))
product = Dot(-1)([pool1, pool2])
features = merge([pool1, pool2, diffvec, product],
mode='concat',
concat_axis=-1)
out = Dense(2,
kernel_regularizer=regularizers.l1_l2(0.01),
activity_regularizer=regularizers.l1_l2(0.01),
activation='softmax')(features)
model = Model(inputs=[input1, input2], outputs=[out])
return model
def GRUHierarchical(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
l_lstm = Bidirectional(
CuDNNGRU(params['lstm_units'], return_sequences=True))(embed_input_)
l_dense = TimeDistributed(Dense(256))(l_lstm)
l_att = GlobalMaxPooling1D()(l_dense)
sentEncoder = Model(input_, l_att)
review_input = Input(shape=(1, params['sequence_length']))
review_encoder = TimeDistributed(sentEncoder)(review_input)
l_lstm_sent = Bidirectional(
CuDNNGRU(params['lstm_units'], return_sequences=True))(review_encoder)
l_dense_sent = TimeDistributed(Dense(256))(l_lstm_sent)
l_att = GlobalMaxPooling1D()(l_dense_sent)
x = Dense(6, activation='sigmoid')(l_att)
model = Model(inputs=review_input, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def __call__(self, a, b):
a_max = GlobalMaxPooling1D()(a)
a_avg = GlobalAveragePooling1D()(a)
b_max = GlobalMaxPooling1D()(b)
b_avg = GlobalAveragePooling1D()(b)
return self.model(merge([a_avg, a_max, b_avg, b_max], mode='concat')) # shape=(batch_size, 4 * units)
def r_c_nn(train_lexical_data, train_sentence_data, train_label,
test_lexical_data, test_sentence_data, test_label):
train_label = np_utils.to_categorical(train_label, label_count)
test_label_categorical = np_utils.to_categorical(test_label, label_count)
lexical_branch = Sequential()
lexical_branch.add(
Bidirectional(SimpleRNN(500, return_sequences=True),
merge_mode='concat',
input_shape=(lexical_length, embedding_length)))
lexical_branch.add(Activation('tanh'))
lexical_branch.add(GlobalMaxPooling1D())
lexical_branch.add(Dense(200, activation='tanh'))
sentence_branch = Sequential()
sentence_branch.add(
Convolution1D(500,
3,
border_mode='same',
input_shape=(timestep, sentence_embedding_length),
W_regularizer=l2(0.01)))
sentence_branch.add(GlobalMaxPooling1D())
sentence_branch.add(Dropout(0.6))
sentence_branch.add(Dense(200, activation='tanh', W_regularizer=l2(0.01)))
merged = Merge([lexical_branch, sentence_branch], mode='concat')
final_model = Sequential()
final_model.add(merged)
final_model.add(Dense(50, activation='tanh', W_regularizer=l2(0.01)))
final_model.add(Dense(label_count, activation='softmax', name='output'))
sgd = SGD(lr=0.01, decay=1e-5, momentum=0.9, nesterov=True)
final_model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
epoch = 50
for i in range(epoch):
print i, '/', epoch
final_model.fit(
[train_lexical_data, train_sentence_data],
train_label,
batch_size=64,
nb_epoch=1,
shuffle=True,
verbose=1,
validation_data=([test_lexical_data,
test_sentence_data], test_label_categorical))
# 带权重的类别
# model.fit([train_lexical_data, train_sentence_data],train_label, batch_size=64,
# nb_epoch=10, class_weight=classWeight, shuffle=True,verbose=1)
# loss_and_metrics = model.evaluate([test_lexical_data,test_sentence_data], test_label_categorical, batch_size=32)
# print loss_and_metrics
test_classes = final_model.predict_classes(
[test_lexical_data, test_sentence_data], batch_size=64)
putOut(test_classes, test_label)
def cnn_word_model_ner(text_size, n_classes, tag_size, embedding_size,
filters_cnn_emb, filters_cnn_tag, kernel_sizes,
coef_reg_cnn_emb, coef_reg_cnn_tag, coef_reg_den,
dropout_rate, dense_size):
inp_emb = Input(shape=(text_size, embedding_size))
inp_tag = Input(shape=(text_size, tag_size))
outputs_emb = []
for i in range(len(kernel_sizes)):
output_i = Conv1D(filters_cnn_emb,
kernel_size=kernel_sizes[i],
activation=None,
kernel_regularizer=l2(coef_reg_cnn_emb),
padding='same')(inp_emb)
output_i = BatchNormalization()(output_i)
output_i = Activation('relu')(output_i)
output_i = GlobalMaxPooling1D()(output_i)
outputs_emb.append(output_i)
outputs_tag = []
for i in range(len(kernel_sizes)):
output_i = Conv1D(filters_cnn_tag,
kernel_size=kernel_sizes[i],
activation=None,
kernel_regularizer=l2(coef_reg_cnn_tag),
padding='same')(inp_tag)
output_i = BatchNormalization()(output_i)
output_i = Activation('relu')(output_i)
output_i = GlobalMaxPooling1D()(output_i)
outputs_tag.append(output_i)
output_emb = concatenate(outputs_emb, axis=1)
print('Concatenate emb shape:', output_emb.shape)
output_tag = concatenate(outputs_tag, axis=1)
print('Concatenate tag shape:', output_tag.shape)
output = concatenate([output_emb, output_tag], axis=1)
print('Concatenate shape:', output.shape)
output = Dropout(rate=dropout_rate)(output)
output = Dense(dense_size,
activation=None,
kernel_regularizer=l2(coef_reg_den))(output)
output = BatchNormalization()(output)
output = Activation('relu')(output)
output = Dropout(rate=dropout_rate)(output)
output = Dense(n_classes,
activation=None,
kernel_regularizer=l2(coef_reg_den))(output)
output = BatchNormalization()(output)
act_output = Activation('softmax')(output)
model = Model(inputs=[inp_emb, inp_tag], outputs=act_output)
return model
def cnnModel():
embedding_layer = Embedding(MAX_NB_WORDS+1,EMBEDDING_DIM,
embeddings_initializer=initializers.he_uniform(20),
#weights=[embedding_matrix],
input_length=INPUT_LEN,
trainable=True)
# #trainable=False))
model1 = Sequential()
model1.add(embedding_layer)
model1.add(Convolution1D(128, 4,padding='same',init='he_normal'))
model1.add(BatchNormalization())
model1.add(Activation('relu'))
model1.add(Convolution1D(128, 4, padding='same',activation='relu',init='he_normal'))
model1.add(GlobalMaxPooling1D())
model2= Sequential()
model2.add(embedding_layer)
model2.add(Convolution1D(128, 3,padding='same',init='he_normal'))
model2.add(BatchNormalization())
model2.add(Activation('relu'))
model2.add(Convolution1D(128, 3, padding='same',activation='relu',init='he_normal'))
model2.add(GlobalMaxPooling1D())
model3 = Sequential()
model3.add(embedding_layer)
model3.add(Convolution1D(128, 5,padding='same',init='he_normal'))
model3.add(BatchNormalization())
model3.add(Activation('relu'))
model3.add(Convolution1D(128, 5, padding='same',activation='relu',init='he_normal'))
model3.add(GlobalMaxPooling1D())
model4 = Sequential()
model4.add(embedding_layer)
model4.add(Convolution1D(128, 7,padding='same',init='he_normal'))
model4.add(BatchNormalization())
model4.add(Activation('relu'))
model4.add(Convolution1D(128, 7, padding='same',activation='relu',init='he_normal'))
model4.add(GlobalMaxPooling1D())
model = Sequential()
model.add(Merge([model1,model2,model3,model4],mode='concat',concat_axis=1))
#model.add(GRU(128, dropout=0.2, recurrent_dropout=0.1))
model.add(Dropout(0.3))
model.add(Dense(128,activation='relu',init='he_normal'))
model.add(Dense(LAW_COUNT,activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adamax',
metrics=[Jaccard_Sim])
model.summary()
return model
def Model1(dim, max_ques_len, max_ans_len, vocab_lim, embedding):
inp_q = Input(shape=(max_ques_len, ))
embedding_q = Embedding(vocab_lim,
dim,
input_length=max_ques_len,
weights=[embedding],
trainable=False)(inp_q)
conv_q = Convolution1D(100, 5, border_mode='same',
activation='relu')(embedding_q)
conv_q = Dropout(0.25)(conv_q)
pool_q = GlobalMaxPooling1D()(conv_q)
inp_a = Input(shape=(max_ans_len, ))
embedding_a = Embedding(vocab_lim,
dim,
input_length=max_ans_len,
weights=[embedding],
trainable=False)(inp_a)
conv_a = Convolution1D(100, 5, border_mode='same',
activation='relu')(embedding_a)
conv_a = Dropout(0.25)(conv_a)
pool_a = GlobalMaxPooling1D()(conv_a)
#sim = SimLayer(1)([pool_q, pool_a])
sim = merge([Dense(100, bias=False)(pool_q), pool_a], mode='dot')
# print pool_a, pool_q
# model1 = merge([pool_q, pool_a, sim], mode='concat')
# model = Model(input=[inp_q, inp_a], output=[model1])
# model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# print model.summary()
# return model
model_sim = merge([pool_q, pool_a, sim], mode='concat')
print model_sim
# #model_final = Flatten()(model_sim)
model_final = Dropout(0.5)(model_sim)
model_final = Dense(201)(model_final)
model_final = Dropout(0.5)(model_final)
model_final = Dense(1, activation='sigmoid')(model_final)
model = Model(input=[inp_q, inp_a], output=[model_final])
print(model.output_shape)
model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['accuracy'])
print model.summary()
return model
def LSTMdeep2(params):
embed_dropout_rate = 0.1
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
x = Activation('tanh')(embed_input_)
x = SpatialDropout1D(embed_dropout_rate, name='embed_drop')(x)
if params['bidirectional']:
x = Bidirectional(
CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform'))(x)
x = Bidirectional(
CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform'))(x)
x1 = GlobalMaxPooling1D()(x)
x2 = GlobalAveragePooling1D()(x)
x3 = AttentionWithContext()(x)
else:
x = CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform')(x)
x = CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform')(x)
x = GlobalMaxPooling1D()(x)
merge_layer = concatenate([x1, x2, x3])
x = Dropout(params['dropout_rate'])(merge_layer)
x = Dense(256)(x)
x = PReLU()(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def bilstm_self_mult_attention_model(self, params):
"""
Method builds uncompiled model of BiLSTM with self multiplicative attention
Args:
params: disctionary of parameters for NN
Returns:
Uncompiled model
"""
inp = Input(shape=(params['text_size'], params['embedding_size']))
output = Bidirectional(LSTM(params['units_lstm'], activation='tanh',
return_sequences=True,
kernel_regularizer=l2(params['coef_reg_lstm']),
dropout=params['dropout_rate'],
recurrent_dropout=params['rec_dropout_rate']))(inp)
output = MaxPooling1D(pool_size=2, strides=3)(output)
output = multiplicative_self_attention(output, n_hidden=params.get("self_att_hid"),
n_output_features=params.get("self_att_out"))
output = GlobalMaxPooling1D()(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(params['dense_size'], activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
output = Activation('relu')(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(self.n_classes, activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
act_output = Activation(params.get("last_layer_activation", "sigmoid"))(output)
model = Model(inputs=inp, outputs=act_output)
return model
def bigru_model(params):
inp = Input(shape=(params['text_size'], params['embedding_size']))
output = Bidirectional(
GRU(params['units_lstm'],
activation='tanh',
return_sequences=True,
kernel_regularizer=l2(params['coef_reg_lstm']),
dropout=params['dropout_rate'],
recurrent_dropout=params['rec_dropout_rate']))(inp)
output = GlobalMaxPooling1D()(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(params['dense_size'],
activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
output = Activation('relu')(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(nb_classes,
activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
act_output = Activation(params.get("last_layer_activation",
"sigmoid"))(output)
model = Model(inputs=inp, outputs=act_output)
model.compile(
optimizer="Adam",
loss="binary_crossentropy",
metrics=['accuracy'],
#loss_weights=loss_weights,
#sample_weight_mode=sample_weight_mode,
# weighted_metrics=weighted_metrics,
# target_tensors=target_tensors
)
return model
def build_model(self):
data = self.data
inputs = Input(shape=(data.seq_length, ))
embeddings = Embedding(
data.max_feature,
data.embed_dim,
weights=[data.embed_matrix],
trainable=False)(inputs)
x = SpatialDropout1D(self.dropout)(embeddings)
x = Bidirectional(CuDNNLSTM(data.embed_dim, return_sequences=True))(x)
x = Bidirectional(CuDNNLSTM(data.embed_dim, return_sequences=True))(x)
ave_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
conc = concatenate([ave_pool, max_pool])
# bn = BatchNormalization()(pool)
x = Dense(self.dense_size, activation="relu")(conc)
# drop = Dropout(self.dropout)(x)
outputs = Dense(6, activation="sigmoid")(x)
model = Model(inputs=inputs, outputs=outputs)
optimizer = self.get_optimizer(self.lr, self.optim_name)
model.compile(
loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
self.model = model
def bilstm_model(self, params):
"""
Build un-compiled BiLSTM
Args:
params: dictionary of parameters for NN
Returns:
Un-compiled model
"""
inp = Input(shape=(params['text_size'], params['embedding_size']))
output = Bidirectional(LSTM(params['units_lstm'], activation='tanh',
return_sequences=True,
kernel_regularizer=l2(params['coef_reg_lstm']),
dropout=params['dropout_rate'],
recurrent_dropout=params['rec_dropout_rate']))(inp)
output = GlobalMaxPooling1D()(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(params['dense_size'], activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
output = Activation('relu')(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(self.n_classes, activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
act_output = Activation(params.get("last_layer_activation", "sigmoid"))(output)
model = Model(inputs=inp, outputs=act_output)
return model
def build_model(ft=False):
# CNN Model Hyperparameters
embedding_dim = 300
vocabsize = len(vocabulary_inv)
dropout_prob = 0.5
nb_filter = 100
hidden_dims = 100
filter_len = [1, 2, 3, 4]
if ft == False:
graph_input1 = Input(shape=(sequence_length, embedding_dim))
embed1 = Embedding(vocabsize, embedding_dim, input_length=sequence_length, name="embedding")(graph_input1)
convs1 = []
for fsz in filter_len:
conv1 = Conv1D(filters=nb_filter, kernel_size=fsz, activation='relu')(graph_input1)
pool1 = GlobalMaxPooling1D()(conv1)
convs1.append(pool1)
y1 = Concatenate()(convs1) if len(convs1) > 1 else convs1[0]
z1 = Dropout(dropout_prob)(y1)
z1 = Dense(hidden_dims, kernel_constraint=maxnorm(2))(z1)
z1 = BatchNormalization()(z1)
z1 = Activation('tanh')(z1)
z1 = Dropout(dropout_prob)(z1)
z1 = Dense(nb_classes)(z1)
z1 = BatchNormalization()(z1)
model_output1 = Activation('softmax')(z1)
cnn_model = Model(inputs=graph_input1, outputs=model_output1)
return cnn_model
def architect_model(input_shape, n_classes):
inp = Input(shape=input_shape, dtype="float32")
x = Conv1D(input_shape[1] * 2,
kernel_size=4,
padding="causal",
activation="relu")(inp)
x = MaxPooling1D(pool_size=4)(x)
x = Dropout(0.25)(x)
x = Conv1D(input_shape[1] * 2,
kernel_size=4,
padding="causal",
activation="relu")(x)
x = MaxPooling1D(pool_size=4)(x)
x = Dropout(0.25)(x)
x = Conv1D(input_shape[1] * 2,
kernel_size=4,
padding="causal",
activation="relu")(x)
x = MaxPooling1D(pool_size=4)(x)
x = Dropout(0.25)(x)
x = Concatenate()([GlobalAveragePooling1D()(x), GlobalMaxPooling1D()(x)])
# Fully connected
x = Dense(2048, activation="relu")(x)
x = Dense(2048, activation="relu")(x)
out = Dense(n_classes, activation="softmax")(x)
return Model(inputs=inp, outputs=out)
def cnn_model(input_shape):
inputs = Input(input_shape)
x = inputs
levels = 64
for level in range(3):
x = Conv1D(levels, 3, activation='relu')(x)
x = BatchNormalization()(x)
x = MaxPooling1D(pool_size=2, strides=2)(x)
levels *= 2
# Global Layers
x = GlobalMaxPooling1D()(x)
for fc in range(2):
x = Dense(256, activation='relu')(x)
x = Dropout(0.5)(x)
labels = Dense(10, activation='softmax')(x)
model = Model(inputs=[inputs], outputs=[labels])
sgd = keras.optimizers.SGD(lr=0.0003,
momentum=0.9,
decay=1e-5,
nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
return model
def cnn_model(params):
inp = Input(shape=(params['text_size'], params['embedding_size']))
outputs = []
for i in range(len(params['kernel_sizes_cnn'])):
output_i = Conv1D(params['filters_cnn'], kernel_size=params['kernel_sizes_cnn'][i],
activation=None,
kernel_regularizer=l2(params['coef_reg_cnn']),
padding='same')(inp)
output_i = BatchNormalization()(output_i)
output_i = Activation('relu')(output_i)
output_i = GlobalMaxPooling1D()(output_i)
outputs.append(output_i)
output = concatenate(outputs, axis=1)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(params['dense_size'], activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
output = BatchNormalization()(output)
output = Activation('relu')(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(nb_classes, activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
output = BatchNormalization()(output)
act_output = Activation("softmax")(output)
model = Model(inputs=inp, outputs=act_output)
model.compile(optimizer="Adam",
loss="binary_crossentropy",
metrics=['accuracy'],
#loss_weights=loss_weights,
#sample_weight_mode=sample_weight_mode,
# weighted_metrics=weighted_metrics,
# target_tensors=target_tensors
)
return model
def __call__(self, sent, align, **kwargs):
result = self.model(merge([sent, align], mode='concat')) # Shape: (i, n)
avged = GlobalAveragePooling1D()(result, mask=self.words)
maxed = GlobalMaxPooling1D()(result, mask=self.words)
merged = merge([avged, maxed])
result = BatchNormalization()(merged)
return result
def GRUconcat3(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
if params['bidirectional']:
x = Bidirectional(CuDNNGRU(params['lstm_units'],
return_sequences=True))(embed_input_)
else:
x = CuDNNGRU(params['lstm_units'], return_sequences=True)(embed_input_)
x1 = GlobalMaxPooling1D()(x)
x2 = GlobalAveragePooling1D()(x)
x3 = AttentionWithContext()(x)
x4 = Attention(params['sequence_length'])(x)
merge_layer = concatenate([x1, x2, x3, x4])
x = Dropout(params['dropout_rate'])(merge_layer)
x = Dense(256, activation='relu')(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def simple_cnn_model(self):
model = Sequential()
# model.add(Masking(mask_value=0., input_shape=(self.SEQ_LEN, self.FEATURES_DIM)))
model.add(Conv1D(input_shape=(self.SEQ_LEN, self.FEATURES_DIM),
filters=32,
kernel_size=14,
padding='same',
name='conv1'))
model.add(Activation(activation='relu', name='act1'))
model.add(MaxPooling1D(pool_size=3))
model.add(GlobalMaxPooling1D())
model.add(Dense(1024))
# model.add(BatchNormalization())
model.add(Activation(activation='relu'))
model.add(Dense(self.n_classes))
model.add(Activation('softmax'))
optimizer = Adam(lr=0.002)
gpu_model = multi_gpu_model(model, gpus=self.n_gpus)
gpu_model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=['categorical_accuracy'])
return (gpu_model, model)
def bilstm_bilstm_model(self,
units_lstm_1: int,
units_lstm_2: int,
dense_size: int,
coef_reg_lstm: float = 0.,
coef_reg_den: float = 0.,
dropout_rate: float = 0.,
rec_dropout_rate: float = 0.,
**kwargs) -> Model:
"""
Build un-compiled two-layers BiLSTM.
Args:
units_lstm_1: number of units for the first LSTM layer.
units_lstm_2: number of units for the second LSTM layer.
dense_size: number of units for dense layer.
coef_reg_lstm: l2-regularization coefficient for LSTM. Default: ``0.0``.
coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
dropout_rate: dropout rate to be used after BiLSTM and between dense layers. Default: ``0.0``.
rec_dropout_rate: dropout rate for LSTM. Default: ``0.0``.
kwargs: other non-used parameters
Returns:
keras.models.Model: uncompiled instance of Keras Model
"""
inp = Input(shape=(self.opt['text_size'], self.opt['embedding_size']))
output = Bidirectional(
LSTM(units_lstm_1,
activation='tanh',
return_sequences=True,
kernel_regularizer=l2(coef_reg_lstm),
dropout=dropout_rate,
recurrent_dropout=rec_dropout_rate))(inp)
output = Dropout(rate=dropout_rate)(output)
output = Bidirectional(
LSTM(units_lstm_2,
activation='tanh',
return_sequences=True,
kernel_regularizer=l2(coef_reg_lstm),
dropout=dropout_rate,
recurrent_dropout=rec_dropout_rate))(output)
output = GlobalMaxPooling1D()(output)
output = Dropout(rate=dropout_rate)(output)
output = Dense(dense_size,
activation=None,
kernel_regularizer=l2(coef_reg_den))(output)
output = Activation('relu')(output)
output = Dropout(rate=dropout_rate)(output)
output = Dense(self.n_classes,
activation=None,
kernel_regularizer=l2(coef_reg_den))(output)
act_output = Activation(
self.opt.get("last_layer_activation", "sigmoid"))(output)
model = Model(inputs=inp, outputs=act_output)
return model
def get_raw_model(self):
branches = []
tweet_input = Input(shape=(self.sentence_length,
self.word_vectors_container.dimension),
dtype='float32')
x = Dropout(0.2)(tweet_input)
for size, filters_count in [(2, 10), (3, 10), (4, 10), (5, 10)]:
for i in range(filters_count):
branch = Conv1D(filters=1,
kernel_size=size,
padding='valid',
activation='relu')(x)
branch = GlobalMaxPooling1D()(branch)
branches.append(branch)
x = concatenate(branches, axis=1)
x = Dropout(0.2)(x)
x = Dense(30, activation='relu')(x)
x = Dense(len(self.corpus.label_encoder.classes_))(x)
output = Activation('softmax')(x)
model = Model(inputs=[tweet_input], outputs=[output])
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['categorical_accuracy'])
return model
def build_and_load_model(path):
DIM = 200
NUM = 100000
embedding_matrix = np.zeros((NUM, DIM))
tweet_input = Input(shape=(SENTENCE_LENGTH, ), dtype='int32')
tweet_encoder = Embedding(NUM,
DIM,
input_length=SENTENCE_LENGTH,
weights=[embedding_matrix],
trainable=False)(tweet_input)
branches = []
x = Dropout(0.2)(tweet_encoder)
for size, filters_count in [(2, 10), (3, 10), (4, 10), (5, 10)]:
for i in range(filters_count):
branch = Conv1D(filters=1,
kernel_size=size,
padding='valid',
activation='relu')(x)
branch = GlobalMaxPooling1D()(branch)
branches.append(branch)
x = concatenate(branches, axis=1)
x = Dropout(0.2)(x)
x = Dense(30, activation='relu')(x)
x = Dense(1)(x)
output = Activation('sigmoid')(x)
model = Model(inputs=[tweet_input], outputs=[output])
model.load_weights(path)
import gc
gc.collect()
return model
def lstmmodel():
inp = Input(shape=(maxlen, ))
x = Embedding(embedding_matrix_1.shape[0],
embedding_matrix_1.shape[1],
weights=[embedding_matrix_1],
trainable=False)(inp)
x = SpatialDropout1D(.1)(x)
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
#x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
x1 = Attention(maxlen)(x)
x2 = GlobalAveragePooling1D()(x)
x3 = GlobalMaxPooling1D()(x)
x = concatenate([x1, x2, x3])
x = Dropout(.1)(x)
x = Dense(100, activation="relu")(x)
x = BatchNormalization()(x)
x = Dropout(.1)(x)
x = Dense(100, activation="relu")(x)
x = BatchNormalization()(x)
x = Dropout(.1)(x)
x = Dense(1, activation="sigmoid")(x)
model = Model(inputs=[inp], outputs=x)
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-3, clipnorm=0.85),
metrics=['accuracy'])
return model
def build_local(input_set_size, vector_size, maxlen, max_sentence_nb,
mul_label_set_size):
sentence_input = Input(shape=(
max_sentence_nb,
maxlen,
))
sentence_model = build_sentence_model(input_set_size, vector_size, maxlen)
paragraph = TimeDistributed(sentence_model)(sentence_input)
#gru = Bidirectional(GRU(128,return_sequences=False), merge_mode='sum')(paragraph)
#drop5 = Dropout(0.5)(gru)
outs = []
for i in range(0, 4):
if i == 0:
merged = paragraph
else:
merged = merge([outs[i - 1], paragraph], mode='concat')
dense = TimeDistributed(Dense(256, activation='relu'))(merged)
p = TimeDistributed(Dense(mul_label_set_size[i],
activation='sigmoid'))(dense)
#p = Dense(mul_label_set_size[i], activation='sigmoid')(merged)
outs.append(p)
# out
outs = [GlobalMaxPooling1D()(outs[i]) for i in range(4)]
model = Model(input=[sentence_input], output=outs)
model.summary()
model.compile(
loss='binary_crossentropy',
optimizer='adam',
)
return model
def TextCNN(config, model_view=True):
inputs = Input(shape=(config['INPUT_LEN'], ), dtype='int32', name='input')
embedding_cnn = Embedding(config['MAX_NB_WORDS'] + 1,
config['EMBEDDING_DIM'],
input_length=config['INPUT_LEN'])(inputs)
conv_out = []
for filter in [1, 2, 3, 4, 5, 6]:
conv1 = Convolution1D(256, filter, padding='same')(embedding_cnn)
conv1 = Activation('relu')(conv1)
x_conv1 = GlobalMaxPooling1D()(conv1)
conv_out.append(x_conv1)
x = concatenate(conv_out)
x = Dense(128)(x)
x = Dropout(0.5)(x)
x = Activation('relu')(x)
x = Dense(452)(x)
outputs = Activation('sigmoid', name='outputs')(x)
model = Model(inputs=inputs, outputs=outputs)
model.compile(loss='binary_crossentropy',
optimizer='adamax',
metrics=[Jaccard_Sim])
if model_view:
model.summary()
return model
def bigru_model(self, params):
"""
Method builds uncompiled model BiGRU
Args:
params: disctionary of parameters for NN
Returns:
Uncompiled model
"""
inp = Input(shape=(params['text_size'], params['embedding_size']))
output = Bidirectional(
GRU(params['units_lstm'],
activation='tanh',
return_sequences=True,
kernel_regularizer=l2(params['coef_reg_lstm']),
dropout=params['dropout_rate'],
recurrent_dropout=params['rec_dropout_rate']))(inp)
output = GlobalMaxPooling1D()(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(params['dense_size'],
activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
output = Activation('relu')(output)
output = Dropout(rate=params['dropout_rate'])(output)
output = Dense(self.n_classes,
activation=None,
kernel_regularizer=l2(params['coef_reg_den']))(output)
act_output = Activation('sigmoid')(output)
model = Model(inputs=inp, outputs=act_output)
return model
def __call__(self, inputs):
x = self.model(inputs)
avg_x = GlobalAveragePooling1D()(x)
max_x = GlobalMaxPooling1D()(x)
x = concatenate([avg_x, max_x])
x = BatchNormalization()(x)
return x
