def build_LSTM_model(self,embedding_matrix):
self.model = Sequential()
self.model.add(
Embedding(input_dim=embedding_matrix.shape[0],
output_dim=embedding_matrix.shape[1],
weights=[embedding_matrix],
input_length=75,
trainable=False))
self.model.add(SpatialDropout1D(0.5))
self.model.add(Conv1D(self.filters, kernel_size=self.kernel_size,kernel_regularizer=regularizers.l2(0.00001), padding='same'))
self.model.add(LeakyReLU(alpha=0.2))
self.model.add(MaxPooling1D(pool_size=2))
self.model.add(Bidirectional(LSTM(self.lstm_units,dropout=0.5, recurrent_dropout=0.5,return_sequences=True)))
self.model.add(SpatialDropout1D(0.5))
self.model.add(Conv1D(self.filters, kernel_size=self.kernel_size,kernel_regularizer=regularizers.l2(0.00001), padding='same'))
self.model.add(LeakyReLU(alpha=0.2))
self.model.add(MaxPooling1D(pool_size=2))
self.model.add(Bidirectional(LSTM(self.lstm_units,dropout=0.5, recurrent_dropout=0.5,return_sequences=True)))
self.model.add(SpatialDropout1D(0.5))
self.model.add(Conv1D(self.filters, kernel_size=self.kernel_size,kernel_regularizer=regularizers.l2(0.00001), padding='same'))
self.model.add(LeakyReLU(alpha=0.2))
self.model.add(MaxPooling1D(pool_size=2))
self.model.add(Bidirectional(LSTM(self.lstm_units,dropout=0.5, recurrent_dropout=0.5)))
self.model.add(Dense(4,activation='softmax'))
self.model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
def define_model(self, length, vocab_size, num_outcome_classes):
'''
Defines and compiles a convolutional neural network model
Parameters
___________
length: int
Max length of the text strings
vocab_size: int
Vocabulary size of the text documents
'''
model = Sequential()
model.add(Input(shape=(length,)))
model.add(Embedding(vocab_size, 250))
model.add(SpatialDropout1D(.25))
model.add(Conv1D(filters=32, kernel_size=4, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(SpatialDropout1D(.25))
model.add(GlobalMaxPooling1D())
model.add(Dense(32, activation='relu'))
model.add(Dense(num_outcome_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', metrics.AUC()])
self.model = model
def ahln(i):
o = Conv1D(hidden_size, kernel_size, padding=padding)(i)
if use_batch_norm:
o = BatchNormalization()(o)
o = Activation('relu')(o)
if dropout > 0:
o = SpatialDropout1D(dropout)(o)
o = Conv1D(hidden_size, kernel_size, padding=padding)(o)
if use_batch_norm:
o = BatchNormalization()(o)
if use_skip_conn:
if o.shape[-1] != i.shape[-1]:
i = TimeDense(o.shape[-1])(i)
if use_batch_norm:
i = BatchNormalization()(i)
o = Add()([i, o])
o = Activation('relu')(o)
if dropout > 0:
o = SpatialDropout1D(dropout)(o)
pool_list = [o]
for pool_size in [4**n for n in range(10) if 4**n <= recept_field]:
pool_list.append(CausalAveragePooling1D(pool_size)(o))
pool_list.append(CausalMaxPooling1D(pool_size)(o))
pool_list.append(CausalMinPooling1D(pool_size)(o))
o = Concatenate()(pool_list)
if not return_sequences:
o = GlobalAveragePooling1D()(o)
return o
def get_convo_nn2(no_word=200, n_gram=21, no_char=178):
input1 = Input(shape=(n_gram, ))
input2 = Input(shape=(n_gram, ))
a = Embedding(no_char, 32, input_length=n_gram)(input1)
a = SpatialDropout1D(0.15)(a)
a = BatchNormalization()(a)
a_concat = []
for i in range(1, 9):
a_concat.append(conv_unit(a, n_gram, no_word, window=i))
for i in range(9, 12):
a_concat.append(conv_unit(a, n_gram, no_word - 50, window=i))
a_concat.append(conv_unit(a, n_gram, no_word - 100, window=12))
a_sum = Maximum()(a_concat)
b = Embedding(12, 12, input_length=n_gram)(input2)
b = SpatialDropout1D(0.15)(b)
x = Concatenate(axis=-1)([a, a_sum, b])
#x = Concatenate(axis=-1)([a_sum, b])
x = BatchNormalization()(x)
x = Flatten()(x)
x = Dense(100, activation='relu')(x)
out = Dense(1, activation='sigmoid')(x)
model = Model(inputs=[input1, input2], outputs=out)
model.compile(optimizer=Adam(),
loss='binary_crossentropy',
metrics=['acc'])
return model
def define_model(max_nb_words, embedding_dim, max_seq_length):
'''
Defines model, includes embedding layer, dropouts, and
LSTM layers.
Parameters
----------
max_nb_words : int
embedding_dim : int
max_seq_length : int
Returns:
--------
model : obj
'''
# LSTM model2
model = Sequential()
# embedding layer in which words are encoded as real-valued vectors and where
# similarity between words is translated to closeness in vector space
model.add(
Embedding(max_nb_words, embedding_dim, input_length=max_seq_length))
# whole feature map may be dropped out, prevents co-adaptation of feature & its
# neighbors
model.add(SpatialDropout1D(0.4))
# LSTM layer with 100 memory units
model.add(
LSTM(100, dropout=0.4, recurrent_dropout=0.4, return_sequences=True))
# added dropout
model.add(SpatialDropout1D(0.4))
model.add(LSTM(100, dropout=0.4, recurrent_dropout=0.4))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def model(vocabulary=7000,
embedding_dim=128,
n_classes=1,
max_length=64,
hidden_units=300,
dropout=0.5,
train=True):
embedding_weights = None
a = Input(shape=(max_length, ), dtype='int32', name='premise')
b = Input(shape=(max_length, ), dtype='int32', name='hypothesis')
# ---------- Embedding layer ---------- #
embedding = EmbeddingLayer(vocabulary,
embedding_dim,
embedding_weights,
max_length=max_length)
embedded_a = embedding(a)
embedded_b = embedding(b)
batchnorm = True
if batchnorm == True:
bn_embedded_a = BatchNormalization(axis=2)(embedded_a)
bn_embedded_b = BatchNormalization(axis=2)(embedded_b)
embedded_a = SpatialDropout1D(0.25)(bn_embedded_a)
embedded_b = SpatialDropout1D(0.25)(bn_embedded_b)
# ---------- Encoding layer ---------- #
encoded_a = EncodingLayer(hidden_units, max_length,
dropout=dropout)(embedded_a)
encoded_b = EncodingLayer(hidden_units, max_length,
dropout=dropout)(embedded_b)
# ---------- Local inference layer ---------- #
m_a, m_b = LocalInferenceLayer()([encoded_a, encoded_b])
# ---------- Inference composition layer ---------- #
composed_a = InferenceCompositionLayer(hidden_units,
max_length,
dropout=dropout)(m_a)
composed_b = InferenceCompositionLayer(hidden_units,
max_length,
dropout=dropout)(m_b)
# ---------- Pooling layer ---------- #
pooled = PoolingLayer()([composed_a, composed_b])
# ---------- Classification layer ---------- #
prediction = MLPLayer(hidden_units, n_classes, dropout=dropout)(pooled)
model = Model(inputs=[a, b], outputs=prediction)
# model.compile(optimizer=Adam(lr=self.learning_rate), #learning_rate=0.0004,
# loss='categorical_crossentropy', metrics=['accuracy'])
return model
def attention_layer(self, x):
attentioned = self.fast_attention(x)
attentioned = SpatialDropout1D(args.hidden_dropout)(attentioned)
x = LayerNormalization(epsilon=1e-6)(add([x, attentioned]))
forwarded = Dense(4*args.attention_dim, activation=tf.nn.relu)(x)
forwarded = Dense(args.attention_dim, activation=None)(forwarded)
forwarded = SpatialDropout1D(args.hidden_dropout)(forwarded)
return LayerNormalization(epsilon=1e-6)(add([x, forwarded]))
def construct_fcn(input_dimension,
output_dimension,
dropout=0.5,
model_name=""):
"""Construct an FCN model for multivariate time series classification
(Karim et al. 2019 - Multivariate LSTM-FCNs for time series classification)
Args:
input_dimension (int): Input dimension of the model
output_dimension (int): Output dimension of the model
dropout (float): Amount of dropout to apply in the first two
convolutional blocks
model_name (str): Name of the model
Returns:
model (tf.keras.Model): Constructed CN model
"""
inputs = Input(shape=(None, input_dimension))
mask = Masking().compute_mask(inputs)
X = Conv1D(128, 8, padding='same', kernel_initializer='he_uniform')(inputs)
X = Activation('relu')(X)
X = BatchNormalization()(X)
X = SpatialDropout1D(dropout)(X)
X = ApplyMask()(X, mask)
X = squeeze_excite_block(X, mask)
X = Conv1D(256, 5, padding='same', kernel_initializer='he_uniform')(X)
X = Activation('relu')(X)
X = BatchNormalization()(X)
X = SpatialDropout1D(dropout)(X)
X = ApplyMask()(X, mask)
X = squeeze_excite_block(X, mask)
X = Conv1D(128, 3, padding='same', kernel_initializer='he_uniform')(X)
X = Activation('relu')(X)
X = BatchNormalization()(X)
X = GlobalAveragePooling1D()(X, mask)
if output_dimension != 1:
# Classification
outputs = Dense(units=output_dimension, activation='softmax')(X)
else:
# Regression
outputs = Dense(units=output_dimension)(X)
model = Model(inputs=inputs, outputs=outputs, name=model_name)
return model
def train_model(X_train_pad, y_train, vocab_size):
es = EarlyStopping(patience=10, restore_best_weights=True)
model = Sequential()
model.add(layers.Embedding(input_dim=vocab_size +1, output_dim=10, mask_zero=True, input_length=150))
model.add(SpatialDropout1D(0.2))
model.add(layers.Conv1D(filters=64, kernel_size=4, activation='relu', padding='same'))
model.add(SpatialDropout1D(0.2))
model.add(layers.Flatten())
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1, activation='linear'))
model.compile(loss='mse',
optimizer='rmsprop',
metrics=['mae'])
model.fit(X_train_pad, y_train, validation_split=0.3, epochs=50, batch_size=16, callbacks=[es])
return model
def define_model(self, length, vocab_size, num_outcome_classes):
'''
Defines and compiles a convolutional neural network model
Parameters
___________
length: int
Max length of the text strings
vocab_size: int
Vocabulary size of the text documents
'''
input = Input(shape=(length, ))
embed = Embedding(vocab_size, 200)(input)
gru = Bidirectional(GRU(128, return_sequences=True))(embed)
drop1 = SpatialDropout1D(0.2)(gru)
conv = Conv1D(filters=256, kernel_size=4, activation='relu')(drop1)
pool = MaxPooling1D(pool_size=2)(conv)
att = Attention()([gru, pool])
flat = Flatten()(att)
drop2 = Dropout(.2)(flat)
dense1 = Dense(64, activation='relu')(drop2)
output = Dense(num_outcome_classes, activation='softmax')(dense1)
model = Model(inputs=input, outputs=output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['categorical_accuracy',
metrics.AUC()])
self.model = model
def lstm_model(self, x_train, x_test, y_train, y_test):
classes_num = self.dataset.getParameters()["classes_num"]
model = Sequential()
model.add(
Embedding(self.VOCAB_SIZE,
self.EMBEDING_DIM,
input_length=self.INPUT_LENGTH))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(classes_num, activation=self.ACTIVATION))
model.compile(loss=self.LOSSFUNC,
optimizer='adam',
metrics=['accuracy'])
es_callback = EarlyStopping(monitor='val_loss', patience=3)
model.summary()
history = model.fit(x_train,
y_train,
epochs=self.EPOCHS,
verbose=1,
validation_data=(x_test, y_test),
batch_size=self.BATCH_SIZE,
callbacks=[es_callback])
loss, accuracy = model.evaluate(x_train, y_train, verbose=1)
print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(x_test, y_test, verbose=1)
print("Testing Accuracy: {:.4f}".format(accuracy))
return history
def train_model(df):
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS, filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~', lower=True)
tokenizer.fit_on_texts(df['tweet'].values)
save_tokenizer(tokenizer, 'Chapter05/twitter_tokenizer.pkl')
X = tokenizer.texts_to_sequences(df['tweet'].values)
X = pad_sequences(X)
Y = df['sentiment'].values
X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size=0.20, random_state=42, stratify=df['sentiment'])
model = Sequential()
optimizer = tf.keras.optimizers.Adam(0.00001)
model.add(Embedding(MAX_NUM_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.5, recurrent_dropout=0.5, return_sequences=True))
model.add(LSTM(100, dropout=0.5, recurrent_dropout=0.5))
model.add(Dense(1, activation='sigmoid'))
loss='binary_crossentropy' #Binary in this case
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
epochs = 15
batch_size = 32
es = [EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)]
history = model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, validation_split=0.3, callbacks=es)
accr = model.evaluate(X_test,Y_test)
print('Test set\n Loss: {:0.3f}\n Accuracy: {:0.3f}'.format(accr[0],accr[1]))
model.save('Chapter05/twitter_model.h5')
evaluate(model, X_test, Y_test)
plot_model(history)
def build_model(lr=0.0,
lr_d=0.0,
units=0,
spatial_dr=0.0,
kernel_size1=3,
kernel_size2=2,
dense_units=128,
dr=0.1,
conv_size=32):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path,
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="min")
early_stop = EarlyStopping(monitor="val_loss", mode="min", patience=3)
inp = Input(shape=(max_len, ))
x = Embedding(39457,
embed_size,
weights=[embedding_matrix],
trainable=False)(inp)
x1 = SpatialDropout1D(spatial_dr)(x)
x_gru = Bidirectional(LSTM(units, return_sequences=True))(x1)
x1 = Conv1D(conv_size,
kernel_size=kernel_size1,
padding='valid',
kernel_initializer='he_uniform')(x_gru)
avg_pool1_gru = GlobalAveragePooling1D()(x1)
max_pool1_gru = GlobalMaxPooling1D()(x1)
x_lstm = Bidirectional(LSTM(units, return_sequences=True))(x1)
x1 = Conv1D(conv_size,
kernel_size=kernel_size1,
padding='valid',
kernel_initializer='he_uniform')(x_lstm)
avg_pool1_lstm = GlobalAveragePooling1D()(x1)
max_pool1_lstm = GlobalMaxPooling1D()(x1)
x = concatenate(
[avg_pool1_gru, max_pool1_gru, avg_pool1_lstm, max_pool1_lstm])
x = BatchNormalization()(x)
x = Dropout(dr)(Dense(dense_units, activation='relu')(x))
x = Dense(2, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
model.compile(loss="binary_crossentropy",
optimizer=Adam(lr=lr, decay=lr_d),
metrics=["accuracy"])
history = model.fit(X_train,
y_ohe,
batch_size=128,
epochs=5,
validation_split=0.1,
verbose=1,
callbacks=[check_point, early_stop])
model = load_model(file_path)
return model
def build(self, input_shape):
with K.name_scope(
self.name
): # name scope used to make sure weights get unique names
self.layers = []
self.res_output_shape = input_shape
for k in range(2):
name = 'conv1D_{}'.format(k)
with K.name_scope(
name
): # name scope used to make sure weights get unique names
self._add_and_activate_layer(
Conv1D(filters=self.nb_filters,
kernel_size=self.kernel_size,
dilation_rate=self.dilation_rate,
padding=self.padding,
name=name,
kernel_initializer=self.kernel_initializer))
with K.name_scope('norm_{}'.format(k)):
if self.use_batch_norm:
self._add_and_activate_layer(BatchNormalization())
elif self.use_layer_norm:
self._add_and_activate_layer(LayerNormalization())
self._add_and_activate_layer(Activation('relu'))
self._add_and_activate_layer(
SpatialDropout1D(rate=self.dropout_rate))
if not self.last_block:
# 1x1 conv to match the shapes (channel dimension).
name = 'conv1D_{}'.format(k + 1)
with K.name_scope(name):
# make and build this layer separately because it directly uses input_shape
self.shape_match_conv = Conv1D(
filters=self.nb_filters,
kernel_size=1,
padding='same',
name=name,
kernel_initializer=self.kernel_initializer)
else:
self.shape_match_conv = Lambda(lambda x: x, name='identity')
self.shape_match_conv.build(input_shape)
self.res_output_shape = self.shape_match_conv.compute_output_shape(
input_shape)
self.final_activation = Activation(self.activation)
self.final_activation.build(
self.res_output_shape) # probably isn't necessary
# this is done to force Keras to add the layers in the list to self._layers
for layer in self.layers:
self.__setattr__(layer.name, layer)
super(ResidualBlock, self).build(
input_shape) # done to make sure self.built is set True
def create(input_shape,
num_lstm_units=62,
alpha1=1.0,
alpha2=1.0,
channel_dropout_rate=0):
inputs = Input(shape=input_shape, name="input")
x = inputs
if channel_dropout_rate > 0:
x = SpatialDropout1D(channel_dropout_rate, name="channel_dropout")(x)
lstm_1 = LSTM(num_lstm_units, return_sequences=True, name="lstm_1")
blstm_1 = Bidirectional(lstm_1, merge_mode="concat", name="blstm_1")(x)
lstm_2 = LSTM(num_lstm_units, return_sequences=True, name="lstm_2")
blstm_2 = Bidirectional(lstm_2, merge_mode="concat",
name="blstm_2")(blstm_1)
alpha1 = tf.constant(alpha1, dtype=blstm_1.dtype, shape=[1])
alpha2 = tf.constant(alpha2, dtype=blstm_2.dtype, shape=[1])
blstm_1_weighted = Multiply(name="alpha1")([alpha1, blstm_1])
blstm_2_weighted = Multiply(name="alpha2")([alpha2, blstm_2])
concat = Concatenate(name="blstm_concat")(
[blstm_1_weighted, blstm_2_weighted])
avg_over_time = GlobalAveragePooling1D(name="avg_over_time")(concat)
lang_vec_z = tf.math.l2_normalize(avg_over_time, axis=1)
return Model(inputs=inputs,
outputs=lang_vec_z,
name="angular_proximity_lstm")
def define_model(self, length, vocab_size):
'''
Defines and compiles a convolutional neural network model
Parameters
___________
length: int
Max length of the text strings
vocab_size: int
Vocabulary size of the text documents
'''
model = Sequential()
model.add(Embedding(vocab_size, 125, input_length=length))
model.add(SpatialDropout1D(0.2))
model.add(GRU(64))
#model.add(Bidirectional(GRU(75)))
model.add(Dropout(0.25))
model.add(Dense(3, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy', metrics.AUC()])
#model = Model(inputs=inputs1, outputs=outputs)
#model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', metrics.AUC()])
self.model = model
def build(self, input_shape):
if input_shape[1] == 0:
return
if input_shape[1] != len(self.input_dims):
raise ValueError('The inputs dimension on axis 1 must be the same as the length of [input_dims].')
if context.executing_eagerly() and context.context().num_gpus():
with ops.device('cpu:0'):
self.embeddings = []
for i, (input_dim, output_dim) in enumerate(zip(self.input_dims, self.output_dims)):
self.embeddings.append(self.add_weight(
shape=(input_dim, output_dim),
initializer=self.embeddings_initializer,
name=f'embeddings_{i}',
regularizer=self.embeddings_regularizer,
constraint=self.embeddings_constraint))
else:
self.embeddings = []
for i, (input_dim, output_dim) in enumerate(zip(self.input_dims, self.output_dims)):
self.embeddings.append(self.add_weight(
shape=(input_dim, output_dim),
initializer=self.embeddings_initializer,
name=f'embeddings_{i}',
regularizer=self.embeddings_regularizer,
constraint=self.embeddings_constraint))
self.dropouts = []
if self.dropout_rate > 0:
self.dropouts = [SpatialDropout1D(self.dropout_rate) for i in range(len(self.input_dims))]
self.built = True
def get_model():
inp = Input(shape=(maxlen, ))
x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)
x = SpatialDropout1D(0.4)(x)
x = Reshape((a, b, 300))(x)
#print(x)
conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], 2), activation='relu')(x)
conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], 2), activation='relu')(x)
conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], 2), activation='relu')(x)
conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], 2), activation='relu')(x)
maxpool_0 = MaxPool2D()(conv_0)
maxpool_1 = MaxPool2D()(conv_1)
maxpool_2 = MaxPool2D()(conv_2)
maxpool_3 = MaxPool2D()(conv_3)
z = concatenate([maxpool_0, maxpool_1, maxpool_2, maxpool_3],axis=1)
z = Flatten()(z)
z = Dropout(0.1)(z)
outp = Dense(1, activation="sigmoid")(z)
model = Model(inputs=inp, outputs=outp)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def create(input_shape,
num_outputs,
output_activation="log_softmax",
channel_dropout_rate=0):
inputs = Input(shape=input_shape, name="input")
x = inputs
if channel_dropout_rate > 0:
x = SpatialDropout1D(
channel_dropout_rate,
name="channel_dropout_{:.2f}".format(channel_dropout_rate))(x)
x = Bidirectional(GRU(512, return_sequences=True),
merge_mode="concat",
name="BGRU_1")(x)
x = Bidirectional(GRU(512), merge_mode="concat", name="BGRU_2")(x)
x = BatchNormalization(name="BGRU_2_bn")(x)
x = Dense(1024, activation="relu", name="fc_relu_1")(x)
x = BatchNormalization(name="fc_relu_1_bn")(x)
x = Dense(1024, activation="relu", name="fc_relu_2")(x)
x = BatchNormalization(name="fc_relu_2_bn")(x)
outputs = Dense(num_outputs, activation=None, name="output")(x)
if output_activation:
outputs = Activation(getattr(tf.nn, output_activation),
name=str(output_activation))(outputs)
return Model(inputs=inputs, outputs=outputs, name="BGRU")
def build(self, args, num_words, num_chars):
self.scale = tf.constant(math.sqrt(args.attention_dim / args.heads))
word_embeddings = tf.keras.Input(shape=[None, MorphoDataset.Dataset.EMBEDDING_SIZE], dtype=tf.float32)
charseqs = tf.keras.Input(shape=[None], dtype=tf.int32)
charseq_ids = tf.keras.Input(shape=[None], dtype=tf.int32)
positional_encoding = tf.keras.Input(shape=[None, args.attention_dim], dtype=tf.float32)
chars_embedded = Embedding(input_dim=num_chars, output_dim=args.cle_dim, mask_zero=False)(charseqs)
convoluted = []
for width in range(2, args.cnn_max_width + 1):
hidden = chars_embedded
for _ in range(args.cle_layers):
hidden = Conv1D(args.cnn_filters, kernel_size=width, strides=1, padding='valid', activation=tf.nn.relu)(hidden)
convoluted.append(GlobalMaxPool1D()(hidden))
chars_hidden = concatenate(convoluted, axis=1)
chars_hidden = Dense(args.we_dim, activation=tf.nn.tanh)(chars_hidden)
char_embedding = Lambda(lambda args: tf.gather(*args))([chars_hidden, charseq_ids])
embedded = concatenate([word_embeddings, char_embedding], axis=2)
embedded = Dense(args.attention_dim, activation=tf.nn.tanh)(embedded)
embedded = add([embedded, positional_encoding])
embedded = SpatialDropout1D(args.input_dropout)(embedded)
x = embedded
for _ in range(args.layers):
x = self.attention_layer(x)
predictions = []
for tag in range(MorphoDataset.TAGS):
bias_init = tf.constant_initializer(self.bias(tag))
tag_prediction = Dense(units=MorphoDataset.TAG_SIZES[tag]-1, activation=None, bias_initializer=bias_init)(x)
predictions.append(tag_prediction)
self.model = tf.keras.Model(inputs=[word_embeddings, charseq_ids, charseqs, positional_encoding], outputs=predictions)
def make_model(embed_size, embedding_matrix, max_features=20000, maxlen=50):
inp = Input(shape=(maxlen, ))
x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(
GRU(128,
return_sequences=True,
activation="tanh",
recurrent_activation="sigmoid",
use_bias="True",
reset_after="True",
unroll="False"))(x)
# For CuDNN implementation with tf2
x = Dropout(0.2)(x)
x = Conv1D(64,
kernel_size=3,
padding="valid",
kernel_initializer="glorot_uniform")(x)
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
x = concatenate([avg_pool, max_pool])
preds = Dense(6, activation="sigmoid")(x)
model = Model(inp, preds)
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-4),
metrics=['accuracy'])
return model
def baseline_model_LSTM(output_len):
model = Sequential()
model.add(
Embedding(MAX_NB_WORDS,
EMBEDDING_DIM,
input_length=MAX_SEQUENCE_LENGTH))
model.add(SpatialDropout1D(0.3))
model.add(
Bidirectional(LSTM(EMBEDDING_DIM, dropout=0.3, recurrent_dropout=0.3)))
model.add(Dense(EMBEDDING_DIM, activation=relu))
model.add(Dropout(0.8))
model.add(Dense(EMBEDDING_DIM, activation=relu))
model.add(Dropout(0.8))
model.add(Dense(output_len, activation=softmax))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#model = Sequential()
#model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH))
#model.add(SpatialDropout1D(0.2))
#model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
#model.add(Dense(output_len, activation=sigmoid))
#opt = Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
#model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
print(model.summary())
plot_model(model,
to_file='model_{}.png'.format('LSTM'),
show_shapes=True,
show_layer_names=True)
return model
def build_model(sent_length, embeddings_weight, class_num):
content = Input(shape=(sent_length, ), dtype='int32')
embedding = Embedding(name="sentence_cuted",
input_dim=embeddings_weight.shape[0],
weights=[embeddings_weight],
output_dim=embeddings_weight.shape[1],
trainable=False)
x = SpatialDropout1D(0.2)(embedding(content))
x = Bidirectional(GRU(200, return_sequences=True))(x)
# x = Bidirectional(GRU(200, return_sequences=True))(x)
avg_pool = GlobalAveragePooling1D()(x) # 全军平均池化
max_pool = GlobalMaxPooling1D()(x) # 全局最大池化
conc = concatenate([avg_pool, max_pool]) # 特征放到一起，
x = Dense(1000)(conc) # 全连接层
x = BatchNormalization()(x)
x = Activation(activation="relu")(x)
x = Dropout(0.2)(x)
x = Dense(500)(x)
x = BatchNormalization()(x)
x = Activation(activation="relu")(x)
output = Dense(class_num, activation="softmax")(x)
model = tf.keras.models.Model(inputs=content, outputs=output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def __init__(self, latent_dim, learning_rate=6e-4, max_length=100, training=True, embeddings=[]):
super(TextSpacy, self).__init__()
self.optimizer = tf.keras.optimizers.Adam(learning_rate)
self.training = training
self.dropoutRate = 0.2
spacy_embeddings = self.get_embeddings() if len(embeddings) == 0 else embeddings
model = tf.keras.Sequential()
model.add(
Embedding(
spacy_embeddings.shape[0],
spacy_embeddings.shape[1],
input_length=max_length,
trainable=False,
weights=[spacy_embeddings],
)
)
model.add(SpatialDropout1D(self.dropoutRate))
model.add(Bidirectional(LSTM(64, return_sequences=True, kernel_regularizer=regularizers.l2(0.001))))
model.add(Dropout(self.dropoutRate))
model.add(Bidirectional(LSTM(128, return_sequences=True, kernel_regularizer=regularizers.l2(0.001))))
model.add(Dropout(self.dropoutRate))
model.add(Bidirectional(LSTM(256, kernel_regularizer=regularizers.l2(0.001))))
model.add(Dropout(self.dropoutRate))
model.add(Dense(1024, activation="relu", kernel_regularizer=regularizers.l2(0.001)))
model.add(Dropout(self.dropoutRate))
model.add(Dense(512, activation="relu", kernel_regularizer=regularizers.l2(0.001)))
model.add(Dropout(self.dropoutRate))
model.add(Dense(256, activation="relu", kernel_regularizer=regularizers.l2(0.001)))
model.add(Dropout(self.dropoutRate))
model.add(Dense(latent_dim, activation="linear"))
self.model = model
def create_model(vocab_size, embed_dim,rnn_neurons,rnn_neurons2,rnn_neurons3,rnn_neurons4,batch_size):
model = Sequential()
model.add(Embedding(vocab_size, embed_dim, batch_input_shape=[batch_size,None]))
model.add(SpatialDropout1D(0.2))
model.add(GRU(rnn_neurons, return_sequences=True,
stateful=True, recurrent_initializer='glorot_uniform', batch_input_shape=[batch_size,None, embed_dim]))
model.add(Dropout(0.2))
model.add(GRU(rnn_neurons2, return_sequences=True,
stateful=True, recurrent_initializer='glorot_uniform'))
model.add(Dropout(0.2))
model.add(GRU(rnn_neurons3, return_sequences=True,
stateful=True, recurrent_initializer='glorot_uniform'))
model.add(Dropout(0.2))
model.add(GRU(rnn_neurons4, return_sequences=True,
stateful=True, recurrent_initializer='glorot_uniform'))
model.add(Dropout(0.2))
model.add(Dense(vocab_size))
model.compile('adam', loss=sparse_cat_loss, metrics=['accuracy'])
return model
def get_model():
inp = Input(shape=(1, ), dtype="string", name="Input_layer")
embedding_layer = Lambda(ELMoEmbedding,
output_shape=(300, ),
name="Elmo_embedding")(inp)
x = SpatialDropout1D(0.2)(embedding_layer)
x = Bidirectional(GRU(80, return_sequences=True))(x)
# output1: classification
y = Bidirectional(
LSTM(80, return_sequences=False, recurrent_dropout=0.2,
dropout=0.2))(x)
y = Dense(720, activation='relu')(y)
y = Dropout(0.3)(y)
y = Dense(360, activation='relu')(y)
y = Dropout(0.3)(y)
outp1 = Dense(2, activation="softmax", name="classification")(y)
#output2: regression
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
conc = concatenate([avg_pool, max_pool])
outp2 = Dense(1, activation='sigmoid', name='regression')(conc)
model = Model(inputs=inp, outputs=[outp1, outp2])
model.compile(loss=['sparse_categorical_crossentropy', 'mse'],
optimizer='adam',
metrics={
'classification': 'accuracy',
'regression': 'mse'
})
return model
def get_model(embedding_matrix=embedding_matrix):
words = Input(shape=(MAX_LEN, ))
x = Embedding(*embedding_matrix.shape,
weights=[embedding_matrix],
trainable=False)(words)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True))(x)
x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True))(x)
hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
hidden = add(
[hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu')(hidden)])
result = Dense(1, activation='sigmoid')(hidden)
model = Model(inputs=words, outputs=result)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[f1_score])
return model
def get_model(name: str,
cell_name: str,
embedding,
max_words: int = MAX_WORDS,
layer_size: int = LAYER_SIZE,
output_size: int = 1):
input_layer = Input(shape=(max_words, ), dtype="int32", name="input")
embedded_layer = embedding(input_layer)
if IS_DROP_OUT:
embedded_layer = SpatialDropout1D(DROP_OUT_RATE)(embedded_layer)
if cell_name == "LSTM":
rnn = LSTM(layer_size, recurrent_activation=cumax)
elif cell_name == "GRU":
rnn = BiGRU(layer_size)
elif cell_name == "NP":
rnn = LSTM(layer_size, peephole=False, recurrent_activation=cumax)
else:
rnn = LSTMVariants(layer_size, cell_name, recurrent_activation=cumax)
rnn_layer = rnn(embedded_layer)
if IS_DROP_OUT:
rnn_layer = Dropout(DROP_OUT_RATE)(rnn_layer)
output_layer = Dense(output_size, activation="sigmoid")(rnn_layer)
model = Model(inputs=[input_layer], outputs=[output_layer], name=name)
model.compile(loss="binary_crossentropy",
optimizer=Adam(),
metrics=["acc"])
model.summary()
return model
def loader(input_shape, num_outputs, output_activation="log_softmax", channel_dropout_rate=0):
inputs = Input(shape=input_shape, name="input")
conv2d_input = inputs
x = Reshape((input_shape[0] or -1, input_shape[1], 1), name="reshape_to_image")(conv2d_input)
x = FrameLayer2D(256, (1, 5), (1, 1), name="frame2d_1")(x)
x = FrameLayer2D(128, (1, 3), (1, 2), name="frame2d_2")(x)
x = FrameLayer2D(64, (1, 3), (1, 3), name="frame2d_3")(x)
x = FrameLayer2D(32, (1, 3), (1, 3), name="frame2d_4")(x)
rows, cols, channels = x.shape[1:]
conv2d_output = Reshape((rows or -1, cols * channels), name="flatten_channels")(x)
if channel_dropout_rate > 0:
conv2d_input = SpatialDropout1D(channel_dropout_rate, name="channel_dropout_{:.2f}".format(channel_dropout_rate))(conv2d_input)
x = Concatenate(name="conv2d_skip")([conv2d_input, conv2d_output])
x = FrameLayer(512, 5, 1, name="frame1")(x)
x = FrameLayer(512, 3, 2, name="frame2")(x)
x = FrameLayer(512, 3, 3, name="frame3")(x)
x = FrameLayer(512, 1, 1, name="frame4")(x)
x = FrameLayer(1500, 1, 1, name="frame5")(x)
x = GlobalMeanStddevPooling1D(name="stats_pooling")(x)
x = SegmentLayer(512, name="segment1")(x)
x = SegmentLayer(512, name="segment2")(x)
outputs = Dense(num_outputs, name="output", activation=None)(x)
if output_activation:
outputs = Activation(getattr(tf.nn, output_activation), name=str(output_activation))(outputs)
return Model(inputs=inputs, outputs=outputs, name="x-vector-2D-skip")
def build_lstm_2_model(
X_train,
y_train,
X_val,
y_val,
embedding_matrix,
vocab_size,
embedding_dim,
learning_rate,
epochs=3
):
model = Sequential()
model.add(
Embedding(
vocab_size, embedding_dim, weights=[embedding_matrix], trainable=False
)
)
model.add(SpatialDropout1D(0.2))
model.add(LSTM(128, dropout=0.2, return_sequences=True))
model.add(LSTM(128, dropout=0.2))
model.add(Dense(1, activation="sigmoid"))
model.summary()
print(model.summary())
# train model
model.compile(
loss="binary_crossentropy",
optimizer=optimizers.Adam(lr=learning_rate),
metrics=["accuracy"],
)
history = model.fit(
X_train, y_train, epochs=epochs, validation_data=(X_val, y_val), batch_size=256
)
return model, history