def model(reader):
inshape = (reader.max_len, )
known_in = L.Input(shape=inshape, name='known_input')
unknown_in = L.Input(shape=inshape, name='unknown_input')
embedding = L.Embedding(
len(reader.vocabulary_above_cutoff) + 2,
5,
input_length=reader.max_len)
conv = L.Convolution1D(
filters=1000,
kernel_size=10,
strides=1,
activation='relu',
name='convolution_10')
repr_known = L.GlobalMaxPooling1D(name='repr_known')(conv(
embedding(known_in)))
repr_unknown = L.GlobalMaxPooling1D(name='repr_unknown')(conv(
embedding(unknown_in)))
full_input = L.Concatenate()([repr_known, repr_unknown])
dense = L.Dense(500, activation='relu')(full_input)
output = L.Dense(2, activation='softmax', name='output')(dense)
model = Model(inputs=[known_in, unknown_in], outputs=output)
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def build_model_multi_cnn_his(text_len, embed_len):
"""
построение модели cnn из статьи
:return:
"""
input_tensor = Input(shape=(text_len, embed_len))
branch_a = layers.Conv1D(filters=1, kernel_size=2, padding='valid', activation='relu')(input_tensor)
branch_a = layers.GlobalMaxPooling1D()(branch_a)
branch_b = layers.Conv1D(filters=1, kernel_size=3, padding='valid', activation='relu')(input_tensor)
branch_b = layers.GlobalMaxPooling1D()(branch_b)
branch_c = layers.Conv1D(filters=1, kernel_size=4, padding='valid', activation='relu')(input_tensor)
branch_c = layers.GlobalMaxPooling1D()(branch_c)
branch_d = layers.Conv1D(filters=1, kernel_size=5, padding='valid', activation='relu')(input_tensor)
branch_d = layers.GlobalMaxPooling1D()(branch_d)
output = layers.concatenate([branch_a, branch_b, branch_c, branch_d], axis=-1)
x = layers.Dropout(0.2)(output)
x = layers.Dense(30, activation='relu')(x)
output_tensor = layers.Dense(1, activation='sigmoid')(x)
model = Model(input_tensor, output_tensor)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def model(reader):
inshape = (reader.max_len, )
known_in = L.Input(shape=inshape, name='known_input')
unknown_in = L.Input(shape=inshape, name='unknown_input')
embedding = L.Embedding(len(reader.vocabulary_above_cutoff) + 2,
5,
input_length=reader.max_len)
known_embed = embedding(known_in)
unknown_embed = embedding(unknown_in)
conv8 = L.Convolution1D(filters=500,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8')
conv4 = L.Convolution1D(filters=500,
kernel_size=4,
strides=1,
activation='relu',
name='convolutional_4')
repr_known1 = L.GlobalMaxPooling1D(name='known_repr_8')(conv8(known_embed))
repr_unknown1 = L.GlobalMaxPooling1D(name='unknown_repr_8')(
conv8(unknown_embed))
repr_known2 = L.GlobalMaxPooling1D(name='known_repr_4')(conv4(known_embed))
repr_unknown2 = L.GlobalMaxPooling1D(name='unknown_repr_4')(
conv4(unknown_embed))
repr_known = L.Concatenate()([repr_known1, repr_known2])
repr_unknown = L.Concatenate()([repr_unknown1, repr_unknown2])
abs_diff = L.merge(inputs=[repr_known, repr_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
dense1 = L.Dense(500, activation='relu')(abs_diff)
dense2 = L.Dense(500, activation='relu')(dense1)
pruned = L.Dropout(0.3)(dense2)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def build_model_multi_cnn_with_embed(embed_len, vocab_power, sentence_len,
embedding_matrix):
"""
построение модели cnn с начальным embedding слоем
:return:
"""
tweet_input = Input(shape=(26, ), dtype='int32')
tweet_encoder = Embedding(vocab_power,
embed_len,
input_length=sentence_len,
weights=[embedding_matrix],
trainable=True)(tweet_input)
x = layers.Dropout(0.2)(tweet_encoder)
branch_a = layers.Conv1D(32, 2, activation='relu')(x)
branch_a = layers.MaxPooling1D(2)(branch_a)
branch_a = layers.Conv1D(32, 3, activation='relu')(branch_a)
branch_a = layers.GlobalMaxPooling1D()(branch_a)
branch_b = layers.Conv1D(32, 3, activation='relu')(x)
branch_b = layers.MaxPooling1D(2)(branch_b)
branch_b = layers.Conv1D(32, 3, activation='relu')(branch_b)
branch_b = layers.GlobalMaxPooling1D()(branch_b)
branch_c = layers.Conv1D(32, 4, activation='relu')(x)
branch_c = layers.MaxPooling1D(2)(branch_c)
branch_c = layers.Conv1D(32, 2, activation='relu')(branch_c)
branch_c = layers.GlobalMaxPooling1D()(branch_c)
branch_d = layers.Conv1D(32, 5, activation='relu')(x)
branch_d = layers.MaxPooling1D(2)(branch_d)
branch_d = layers.Conv1D(32, 2, activation='relu')(branch_d)
branch_d = layers.GlobalMaxPooling1D()(branch_d)
output = layers.concatenate([branch_a, branch_b, branch_c, branch_d],
axis=1)
x = layers.Dropout(0.4)(output)
x = layers.Dense(30, activation='relu')(x)
output_tensor = layers.Dense(1, activation='sigmoid')(x)
model = Model(tweet_input, output_tensor)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def set_up_model(vocab_size, input_length, embedding=np.zeros((1, 0))):
"""embedding optional"""
model = Sequential()
if (embedding.size):
print("Embedding layer using weights calculated in embedding matrix ")
model.add(
layers.Embedding(vocab_size,
embedding_length,
input_length=input_length,
weights=[embedding],
trainable=False))
else:
print("Embedding layer with random intiialized word vectors")
model.add(
layers.Embedding(vocab_size,
embedding_length,
input_length=input_length,
trainable=False))
model.add(
layers.Conv1D(filter_n,
filter_heigth,
strides=strides,
padding='valid',
activation='elu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dropout(0.5))
#model.add(layers.Dense(10, activation='relu'))#sigmoid for multicalss, softmax for single classes
model.add(layers.Dense(3, activation='softmax')
) #sigmoid for multicalss, softmax for single classes
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
return model
def trainModel(train_data,train_lable,test_data,test_lable):
#####下面的代码进行训练 核心参数是神经网络的层数，卷积核的大小，密集层的细胞个数和层数
model = models.Sequential()
model.add(layers.Conv1D(64,5,activation='tanh',input_shape=(700,1)))
model.add(layers.MaxPooling1D(3))
model.add(layers.Conv1D(32,5,activation='tanh'))
model.add(layers.MaxPooling1D(3))
model.add(layers.Conv1D(32,5,activation='tanh'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(8))
model.add(layers.Dense(4))
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer=RMSprop(),loss='mse')
history = model.fit(train_data,train_lable,
epochs=30,
batch_size=20,
validation_data=(test_data,test_lable),
callbacks= callback_list1
)
return history
###########################################################################################################
def cnn_model(num_classes, input_shape, embedding_matrix, max_length):
""" Creates CNN model for classification of emotions with Glove embeddings
:param num_classes: number of classes
:type num_classes: int
:param input_shape: shape of the input
:type input_shape: tuple
:param embedding_matrix: embedding matrix for Keras Embedding layer
:type embedding_matrix: numpy.array
:param max_length: maximum length of the text sequence
:type max_length: int
:return: CNN model
"""
model = k.Sequential()
model.add(kl.Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=max_length,
trainable=False,
name='embedding_layer'))
model.add(kl.Convolution1D(32, 3, activation='relu', input_shape=input_shape))
model.add(kl.MaxPooling1D())
model.add(kl.Convolution1D(64, 3, activation='relu'))
model.add(kl.GlobalMaxPooling1D())
model.add(kl.Dense(128))
model.add(kl.Dropout(0.2))
model.add(kl.Activation('relu'))
model.add(kl.Dense(num_classes))
model.add(kl.Activation('sigmoid'))
return model
def getModel_api():
posts_input = Input(shape=(None, ), dtype='int32', name='posts')
# embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input) # 感觉写反了
embedded_posts = layers.Embedding(vocabulary_size, 256)(posts_input)
x = layers.Conv1D(128, 5,
activation='relu')(embedded_posts) # 128个特征 窗口长度5
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
# x = layers.MaxPooling1D(5)(x) # 加上后会报错：Computed output size would be negative:
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.GlobalMaxPooling1D()(x)
x = layers.Dense(128, activation='relu')(x)
age_prediction = layers.Dense(1, name='age')(x) # 年龄输出 输出层都具有名称
income_prediction = layers.Dense(num_income_groups,
activation='sigmoid',
name='income')(x) # 定义收入输出
gender_prediction = layers.Dense(1, activation='sigmoid',
name='gender')(x) # 性别输出
model = models.Model(
posts_input, [age_prediction, income_prediction, gender_prediction
]) # 年龄-回归 收入-多分类单标签 性别-二分类
# 多输出模型的编译选项：多重损失, 你可以在编译时使用损失组成的列表或 字典来为不同输出指定不同损失
model.compile(
loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'],
optimizer='rmsprop',
loss_weights=[0.25, 1.0, 10.0]) # 平衡损失
# model.compile(loss=['age':'mse', 'income':'categorical_crossentropy',
# 'gender':'binary_crossentropy'], optimizer='rmsprop', loss_weights=['age':0.25, 'income':1.0, 'gender':10.0])
return model
def build_model(embedding_matrix, word_index, max_len, lstm_units,
verbose = False, compile = True, multi=True, gpu_num=4):
#logger.info('Build model')
sequence_input = L.Input(shape=(max_len,), dtype='int32')
embedding_layer = L.Embedding(*embedding_matrix.shape,
weights=[embedding_matrix],
trainable=False)
x = embedding_layer(sequence_input)
x = L.SpatialDropout1D(0.3)(x)
x = L.Bidirectional(L.CuDNNLSTM(lstm_units, return_sequences=True))(x)
x = L.Bidirectional(L.CuDNNLSTM(lstm_units, return_sequences=True))(x)
att = Attention(max_len)(x)
avg_pool1 = L.GlobalAveragePooling1D()(x)
max_pool1 = L.GlobalMaxPooling1D()(x)
x = L.concatenate([att,avg_pool1, max_pool1])
preds = L.Dense(1, activation='sigmoid')(x)
model = Model(sequence_input, preds)
if multi:
print('use multi gpus')
model = ModelMGPU(model, gpus=gpu_num)
if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
return model
def cnn_2x_siamese(voc_size, max_len, dropout=0.5):
"""Two siamese branches, each embedding a statement.
Binary classifier on top.
Args:
voc_size: size of the vocabulary for the input statements.
max_len: maximum length for the input statements.
Returns:
A Keras model instance.
"""
pivot_input = layers.Input(shape=(max_len, ), dtype='int32')
statement_input = layers.Input(shape=(max_len, ), dtype='int32')
x = layers.Embedding(output_dim=256,
input_dim=voc_size,
input_length=max_len)(pivot_input)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(3)(x)
x = layers.Convolution1D(256, 7, activation='relu')(x)
embedded_pivot = layers.GlobalMaxPooling1D()(x)
encoder_model = Model(pivot_input, embedded_pivot)
embedded_statement = encoder_model(statement_input)
concat = layers.merge([embedded_pivot, embedded_statement], mode='concat')
x = layers.Dense(256, activation='relu')(concat)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='sigmoid')(x)
model = Model([pivot_input, statement_input], prediction)
return model
def create_model(params):
embedding_dim = 256
max_len = int(params['max_len'])
vocab_size = int(params['vocab_size'])
filter_sizes = [2, 3, 5, 7]
conv_filters = []
input_tensor = layers.Input(shape=(max_len, ))
input_layer = layers.Embedding(vocab_size, embedding_dim,
input_length=70)(input_tensor)
for f_size in filter_sizes:
conv_filter = layers.Conv1D(128, f_size,
activation='relu')(input_layer)
conv_filter = layers.GlobalMaxPooling1D()(conv_filter)
conv_filters.append(conv_filter)
conc_layer = layers.Concatenate()(conv_filters)
graph = Model(inputs=input_tensor, outputs=conc_layer)
model = Sequential()
model.add(graph)
model.add(layers.Dropout(0.5))
model.add(layers.Dense(len(conv_filters), activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def imdb_conv1d():
"""Trains and evaluates a 1D CNN using text sequences from the IMDB
dataset."""
max_features = 10000
maxlen = 500
(x_train, y_train), (x_test,
y_test) = imdb.load_data(num_words=max_features)
x_train = [x[::-1] for x in x_train]
x_test = [x[::-1] for x in x_test]
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
model = Sequential()
model.add(layers.Embedding(max_features, 128))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(lr=1e-4),
loss='binary_crossentropy',
metrics=['acc'])
history = model.fit(x_train,
y_train,
epochs=10,
batch_size=128,
validation_split=0.2)
plot_history(history)
def GCAE(param):
filter_sizes = [2, 3, 4]
num_filters = 128
inp1 = layers.Input(shape=(param['sentence_len'], ))
x1 = layers.Embedding(input_dim=param['vocab_size'],
output_dim=param['embed_size'])(inp1)
# x1 = layers.SpatialDropout1D(rate = 0.2)(x1)
inp2 = layers.Input(shape=(1, ))
x2 = layers.Embedding(input_dim=param['num_subject'],
output_dim=param['embed_size'])(inp2)
x_fla = layers.Flatten()(x2)
maxpool_pool = []
for filter_size in filter_sizes:
conv1 = layers.Conv1D(num_filters,
kernel_size=filter_size,
activation='tanh')(x1)
conv2 = layers.Conv1D(num_filters,
kernel_size=filter_size,
activation=None)(x1)
x2 = layers.RepeatVector(param['sentence_len'] - filter_size +
1)(x_fla)
conv = Aspect_conv()([conv1, conv2, x2])
maxpool_pool.append(layers.GlobalMaxPooling1D()(conv))
z = layers.Concatenate(axis=1)(maxpool_pool)
z = layers.Dropout(0.1)(z)
outp = layers.Dense(param['num_class'], activation='softmax')(z)
model = Model(inputs=[inp1, inp2], outputs=outp)
optimizer = optimizers.Adam()
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model
def trainCNNModelOnEmbeddings(self, X_train, y_train, X_test, y_test):
model_eval = ModelEvaluation()
embedding_dim = 100
input_dim = X_train.shape[1]
model = models.Sequential()
model.add(layers.Embedding(input_dim, embedding_dim,
input_length=None))
model.add(layers.Conv1D(128, 5, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(10, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
model.summary()
history = model.fit(X_train,
y_train,
epochs=1,
verbose=False,
validation_data=(X_test, y_test),
batch_size=10)
loss, accuracy = model.evaluate(X_train, y_train, verbose=False)
print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=False)
print("Testing Accuracy: {:.4f}".format(accuracy))
model_eval.plot_history(history)
return True
def cnn_2x(voc_size, max_len, dropout=0.5):
"""One branch embedding a statement. Binary classifier on top.
Args:
voc_size: size of the vocabulary for the input statements.
max_len: maximum length for the input statements.
Returns:
A Keras model instance.
"""
statement_input = layers.Input(shape=(max_len,), dtype='int32')
x = layers.Embedding(
output_dim=256,
input_dim=voc_size,
input_length=max_len)(statement_input)
x = layers.Convolution1D(256, 7, activation='relu')(x)
x = layers.MaxPooling1D(3)(x)
x = layers.Convolution1D(256, 7, activation='relu')(x)
embedded_statement = layers.GlobalMaxPooling1D()(x)
x = layers.Dense(256, activation='relu')(embedded_statement)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='sigmoid')(x)
model = Model(statement_input, prediction)
return model
def build_model(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
embedding_layer = L.Embedding(len(word_index) + 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
x = embedding_layer(sequence_input)
x = L.SpatialDropout1D(0.2)(x)
x = L.Bidirectional(L.CuDNNLSTM(64, return_sequences=True))(x)
att = Attention(maxlen)(x)
avg_pool1 = L.GlobalAveragePooling1D()(x)
max_pool1 = L.GlobalMaxPooling1D()(x)
x = L.concatenate([att,avg_pool1, max_pool1])
preds = L.Dense(1, activation='sigmoid')(x)
model = Model(sequence_input, preds)
if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
return model
def model_cnn1(we_layer, config):
"""
First model:
- one embeddings layer
- multiple (parallel and different kernels size) convolutionnal layers
- One fully connected layer
:param we_layer: the embeddings layer to use
:param config: the parameters of the layer
:return: a compiled Keras model
"""
model_conf = config.keras_model
in_layer = layers.Input(shape=(config.tw_length, ),
dtype='int32',
name='input_layer')
we = we_layer(in_layer)
submodels = []
for ks in model_conf.cnn_kernels:
conv = layers.Conv1D(model_conf.cnn_filters, ks, padding='valid')(we)
conv = layers.LeakyReLU()(conv)
submodels.append(layers.GlobalMaxPooling1D()(conv))
x = layers.concatenate(submodels)
x = layers.Dropout(model_conf.dropout)(x)
x = layers.Dense(model_conf.dense_size)(x)
x = layers.LeakyReLU()(x)
x = layers.Dropout(model_conf.dropout)(x)
out_layer = layers.Dense(units=2, activation='softmax')(x)
model = models.Model(in_layer, out_layer)
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
model.summary()
return model
def __init__(self, embedding_matrix):
super(Extractor_Model, self).__init__()
#hyperparameters
num_filters = 100
sequence_length = 540
embedding_dimension = 100
num_words = 4860
#model
self.model = Sequential()
self.model.add(layers.Embedding(input_dim=num_words, output_dim=embedding_dimension,
embeddings_initializer=initializers.Constant(embedding_matrix),
input_length=sequence_length, trainable=False))
self.conv_2 = layers.Conv1D(filters=num_filters, kernel_size=2, padding='same', activation='relu')
self.conv_3 = layers.Conv1D(filters=num_filters, kernel_size=3, padding='same', activation='relu')
self.conv_4 = layers.Conv1D(filters=num_filters, kernel_size=4, padding='same', activation='relu')
self.conv_5 = layers.Conv1D(filters=num_filters, kernel_size=5, padding='same', activation='relu')
self.conv_6 = layers.Conv1D(filters=num_filters, kernel_size=6, padding='same', activation='relu')
self.global_max_pool = layers.GlobalMaxPooling1D()
#optimizer
learning_rate = 0.001
decay_rate = learning_rate/((1 + 10 * np.random.randint(0, 2)) ** 0.75)
self.optimizer = tf.keras.optimizers.Adam(lr=learning_rate, decay=decay_rate)
def addCharEmbeddingLayers(self, inputNodes, mergeInputLayers, chars_input,
charEmbeddings):
# Use LSTM for char embeddings from Lample et al., 2016
if self.params['charEmbeddings'].lower() == 'lstm':
chars = layers.TimeDistributed(layers.Embedding(
input_dim=charEmbeddings.shape[0],
output_dim=charEmbeddings.shape[1],
weights=[charEmbeddings],
trainable=True,
mask_zero=False),
name='char_emd')(chars_input)
charLSTMSize = self.params['charLSTMSize']
chars = layers.TimeDistributed(layers.Bidirectional(
layers.LSTM(charLSTMSize, return_sequences=False)),
name="char_lstm")(chars)
else: # Use CNNs for character embeddings from Ma and Hovy, 2016
chars = layers.TimeDistributed(
layers.Embedding( # Conv layer does not support masking
input_dim=charEmbeddings.shape[0],
output_dim=charEmbeddings.shape[1],
trainable=True,
weights=[charEmbeddings]),
name='char_emd')(chars_input)
charFilterSize = self.params['charFilterSize']
charFilterLength = self.params['charFilterLength']
chars = layers.TimeDistributed(layers.Conv1D(charFilterSize,
charFilterLength,
padding='same'),
name="char_cnn")(chars)
chars = layers.TimeDistributed(layers.GlobalMaxPooling1D(),
name="char_pooling")(chars)
mergeInputLayers.append(chars)
inputNodes.append(chars_input)
self.params['featureNames'].append('characters')
def create_channel(self, x, kernel_size, feature_map):
"""
Creates a layer, working channel wise
Arguments:
x : Input for convoltuional channel
kernel_size : Kernel size for creating Conv1D
feature_map : Feature map
Returns:
x : Channel including (Conv1D + {GlobalMaxPooling & GlobalAveragePooling} + Dense [+ Dropout])
"""
x = layers.SeparableConv1D(feature_map,
kernel_size=kernel_size,
activation='relu',
strides=1,
padding='valid',
depth_multiplier=4)(x)
x1 = layers.GlobalMaxPooling1D()(x)
x2 = layers.GlobalAveragePooling1D()(x)
x = layers.concatenate([x1, x2])
x = layers.Dense(self.hidden_units)(x)
if self.dropout_rate:
x = layers.Dropout(self.dropout_rate)(x)
return x
def build_conv_model(self):
"""
Combining CNNs and RNNs to process long sequences
https://keras.io/layers/convolutional/
1D convolution layer (e.g. temporal convolution)
Not great performance but faster
"""
self.model = models.Sequential()
self.model.add(
layers.Conv1D(
filters=32, # the dimensionality of the output space
kernel_size=
5, # an integer or tuple/list of a single integer, specifying the length of the 1D convolution window.
activation='relu',
input_shape=(None, float_data.shape[-1])))
self.model.add(layers.MaxPooling1D(3))
self.model.add(layers.Conv1D(32, 5, activation='relu'))
self.model.add(layers.MaxPooling1D(3))
self.model.add(layers.Conv1D(32, 5, activation='relu'))
self.model.add(layers.GlobalMaxPooling1D())
self.model.add(
layers.Dense(1)
) # a regression problem, no activation function on the last Dense layer
self.model.compile(optimizer=optimizers.RMSprop(lr=0.001), loss='mae')
self.model.summary()
def CNN_text_classifier(embedding_dim, vocab_size, maxlen):
'''It is a sequential five layer neural network where first layer is an Embedding layer, second layer is
1-dimensional convolution layer using relu activation function, third layer is maxpooling layer followed by
a dense layer using relu activation and finally the last dense layer using sigmoid activation function'''
model = Sequential() # Sequential is a Keras API which allows us to create models layer-by-layer piecewise.
model.add(layers.Embedding(vocab_size, embedding_dim, input_length=maxlen)) # It is the first embedding layer of
# the model which turns positive integers(indexes) into dense vectors of fixed size. input_dim = vocab_size(4933 in
# this case). Output_dimension = embedding_dim(100 in this case), is the dimension of dense embedding.
# input_length = maxlen(100 in this case), is the length of input sequence.
model.add(layers.Conv1D(128, 5, activation='relu')) # adding 1-dimensional convolution layer as second layer with
# filters = 128, output filters in the convolution(dimensionality of the output space). kernel_size = 5, which
# signifies the length of the 1D convolution window.
model.add(layers.GlobalMaxPooling1D()) # using maxpooling layer to reduce the spatial size of the representation.
model.add(layers.Dense(10, activation='relu')) # Dense is used to create densely-connected
# Neural Network layers. we have taken units:10, which means output array shape will be(*, 10). input_dim is the
# dimension of input fed to the layer and activation:relu, basically uses rectified linear unit to convert input
# signal into output signal at a A-NN node. It has been seen that it provides better convergence and also it
# also rectifies vanishing gradient problem.
model.add(layers.Dense(1, activation='sigmoid')) # it is the last layer whose output shape array shape
# will be(*, 1). The activation function used is Sigmoid whose range is between 0 and 1.
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
# compile is a method of sequential class which configures the model for training. We are using binary_crossentropy
# loss function to calculate loss, and adam optimizer. we want to evaluate just the accuracy metric.
model.summary() # prints a summary representation of the model.
return model
def create_model():
model = Sequential()
model.add(
lrs.Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=False))
model.add(lrs.SpatialDropout1D(0.2))
model.add(
lrs.Bidirectional(lrs.LSTM(128,
return_sequences=True,
dropout=0.0,
recurrent_dropout=0.0),
merge_mode='concat'))
model.add(
lrs.Conv1D(64,
kernel_size=2,
padding='valid',
kernel_initializer='glorot_uniform'))
model.add(lrs.GlobalMaxPooling1D()) # avg pooling
model.add(lrs.Dense(6, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=Nadam(lr=0.001),
metrics=['accuracy']) # default 0.002
return model
def model_cnn3(we_layer, config):
"""
Keras model that reproduce the dying relu phenomena
"""
model_conf = config.keras_model
in_layer = layers.Input(shape=(config.tw_length, ),
dtype='int32',
name='input_layer')
we = we_layer(in_layer)
submodels = []
for ks in model_conf.cnn_kernels:
conv = layers.Conv1D(model_conf.cnn_filters,
ks,
padding='valid',
activation='relu')(we)
submodels.append(layers.GlobalMaxPooling1D()(conv))
x = layers.concatenate(submodels)
x = layers.Dropout(model_conf.dropout)(x)
x = layers.Dense(model_conf.dense_size, activation='relu')(x)
x = layers.Dropout(model_conf.dropout)(x)
out_layer = layers.Dense(units=2, activation='softmax')(x)
model = models.Model(in_layer, out_layer)
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
model.summary()
return model
def transfer_kernels(models: List[Model] = None) -> Model:
# acc(train/valid/test): 0.87/0.87/0.863 | 2 epochs, commit ad1e | Adam lr 0.001
# Pre: models[0] is birnn
birnn = BaseModels.birnn() if models is None else models[0]
inputs = layers.Input(shape=(MAX_SEQUENCE_LENGTH, ))
X = inputs
for layer in birnn.layers[:-4]:
X = layer(X)
layer.trainable = False
Y = layers.Embedding(
input_dim=MAX_WORDS,
output_dim=256,
input_length=MAX_SEQUENCE_LENGTH,
)(inputs)
X = layers.concatenate([X, Y], axis=1)
bigram_branch = layers.Conv1D(filters=128,
kernel_size=2,
padding='valid',
activation='relu',
strides=1)(X)
bigram_branch = layers.GlobalMaxPooling1D()(bigram_branch)
trigram_branch = layers.Conv1D(filters=128,
kernel_size=3,
padding='valid',
activation='relu',
strides=1)(X)
trigram_branch = layers.GlobalMaxPooling1D()(trigram_branch)
fourgram_branch = layers.Conv1D(filters=128,
kernel_size=4,
padding='valid',
activation='relu',
strides=1)(X)
fourgram_branch = layers.GlobalMaxPooling1D()(fourgram_branch)
merged = layers.concatenate(
[bigram_branch, trigram_branch, fourgram_branch], axis=1)
X = layers.Dropout(.5)(merged)
X = layers.Dense(128, activation='relu')(X)
X = layers.Dense(1, activation='sigmoid')(X)
return Model(inputs=inputs,
outputs=X,
name=_name_model('transfer-kernels'))
def build_DNN(max_words=20000, maxlen=200, embedding_dim=400, classification_type=2):
S_inputs = layers.Input(shape=(None,), dtype='int32')
embeddings = layers.Embedding(max_words, embedding_dim, input_length=maxlen)(S_inputs)
dense_1 = layers.Dense(100, activation='tanh')(embeddings)
max_pooling = layers.GlobalMaxPooling1D()(dense_1)
outputs = layers.Dense(classification_type, activation='softmax')(max_pooling)
model = models.Model(inputs=S_inputs, outputs=outputs)
return model
def instantiate_shared_encoder(x):
emb = shared_emb(x)
grams = layers.LeakyReLU(0.2)(layers.merge(
[gr(emb) for gr in shared_grams], mode="concat"))
chmax = layers.GlobalMaxPooling1D()(grams)
dense1 = layers.LeakyReLU(0.2)(shared_dense1(chmax))
dense2 = shared_dense2(dense1)
return layers.Reshape((32, 1))(dense2)
def model(self):
model = models.Sequential()
model.add(layers.Embedding(self.vocab_size + 1, self.embedding_dim, input_length=self.max_words))
# model.add(layers.GlobalAveragePooling1D())
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(self.class_num, activation='softmax', kernel_initializer=he_normal()))
print(model.summary())
return model
def model(reader):
inshape = (None, )
known_in = L.Input(shape=inshape, name='known_input', dtype='int32')
unknown_in = L.Input(shape=inshape, name='unknown_input', dtype='int32')
embedding = L.Embedding(
len(reader.channels[0].vocabulary_above_cutoff) + 2, 5)
known_embed = embedding(known_in)
unknown_embed = embedding(unknown_in)
conv8 = L.Convolution1D(filters=5000,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8')
repr_known = L.GlobalMaxPooling1D()(conv8(known_embed))
repr_unknown = L.GlobalMaxPooling1D()(conv8(unknown_embed))
abs_diff = L.merge(inputs=[repr_known, repr_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
dense1 = L.Dense(500, activation='relu')(abs_diff)
dense2 = L.Dense(500, activation='relu')(dense1)
dense3 = L.Dense(500, activation='relu')(dense2)
dense4 = L.Dense(500, activation='relu')(dense3)
dense5 = L.Dense(500, activation='relu')(dense4)
pruned = L.Dropout(0.3)(dense5)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
optimizer = O.Adam(lr=0.0005)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def create_model(num_filters, kernel_size, vocab_size, embedding_dim, maxlen):
model = Sequential()
model.add(layers.Embedding(vocab_size, embedding_dim, input_length=maxlen))
model.add(layers.Conv1D(num_filters, kernel_size, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(10, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
return model
