def do_attention(self, x):
'''
add none or one of the following attention layers
:param x:
:param att_type:
:return:
'''
att_layer = None
if self.att_type == 'scaled_dot':
att_layer = scaled_dot_attention.ScaledDotProductAttention(
name='Attention')
x = att_layer(x)
# x = GlobalAveragePooling1D()(x)
x = GlobalMaxPooling1D()(x)
elif self.att_type == 'seq_self_attention':
att_layer = seq_self_attention.SeqSelfAttention(
attention_activation='sigmoid')
x = att_layer(x)
# x = GlobalAveragePooling1D()(x)
x = GlobalMaxPooling1D()(x)
elif self.att_type == 'seq_weighted_attention':
att_layer = seq_weighted_attention.SeqWeightedAttention()
x = att_layer(x)
# x = seq_weighted_attention.SeqWeightedAttention()(x)
elif self.att_type == 'attention_with_context':
att_layer = many_to_one_attention_with_context.AttentionWithContext(
)
x = att_layer(x)
return x, att_layer
def build_model2():
d = 0.2
model_input = Input(shape=(time_steps, input_dim))
x = Bidirectional(LSTM(128, return_sequences=True))(model_input)
x = Dropout(d)(x)
#x = Bidirectional(LSTM(64, return_sequences=False))(x)
#x = Dropout(d)(x)
a = Attention()([x, x])
out1 = GlobalMaxPooling1D()(x)
out2 = GlobalMaxPooling1D()(a)
merge = Concatenate()([out1, out2])
x = Dense(16, activation='relu')(merge)
x = Dense(1, activation='sigmoid')(x)
model = Model(model_input, x)
lossfn = tf.keras.losses.BinaryCrossentropy(
from_logits=False,
label_smoothing=0.0,
axis=-1,
reduction="auto",
name="binary_crossentropy",
)
# 二分类
model.compile(optimizer='rmsprop', loss=lossfn, metrics=['accuracy'])
return model
def create_cnn(num_classes: int = 2) -> tf.keras.Model:
x = Input(shape=(256, ), dtype="int64")
h = Embedding(en2vec.corpus_size + 1, 128, input_length=256)(x)
conv1 = Convolution1D(filters=256, kernel_size=10, activation="tanh")(h)
conv2 = Convolution1D(filters=256, kernel_size=7, activation="tanh")(h)
conv3 = Convolution1D(filters=256, kernel_size=5, activation="tanh")(h)
conv4 = Convolution1D(filters=256, kernel_size=3, activation="tanh")(h)
h = Concatenate()([
GlobalMaxPooling1D()(conv1),
GlobalMaxPooling1D()(conv2),
GlobalMaxPooling1D()(conv3),
GlobalMaxPooling1D()(conv4),
])
h = Dense(1024, activation="selu", kernel_initializer="lecun_normal")(h)
h = AlphaDropout(0.1)(h)
h = Dense(1024, activation="selu", kernel_initializer="lecun_normal")(h)
h = AlphaDropout(0.1)(h)
y = Dense(num_classes, activation="softmax")(h)
model = Model(x, y)
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy", AUC()])
return model
def build_model(lr=0.0,
lr_d=0.0,
units=0,
spatial_dr=0.0,
kernel_size1=3,
kernel_size2=2,
dense_units=128,
dr=0.1,
conv_size=32):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path,
monitor="val_loss",
verbose=1,
save_best_only=True,
mode="min")
early_stop = EarlyStopping(monitor="val_loss", mode="min", patience=3)
inp = Input(shape=(max_len, ))
x = Embedding(39457,
embed_size,
weights=[embedding_matrix],
trainable=False)(inp)
x1 = SpatialDropout1D(spatial_dr)(x)
x_gru = Bidirectional(LSTM(units, return_sequences=True))(x1)
x1 = Conv1D(conv_size,
kernel_size=kernel_size1,
padding='valid',
kernel_initializer='he_uniform')(x_gru)
avg_pool1_gru = GlobalAveragePooling1D()(x1)
max_pool1_gru = GlobalMaxPooling1D()(x1)
x_lstm = Bidirectional(LSTM(units, return_sequences=True))(x1)
x1 = Conv1D(conv_size,
kernel_size=kernel_size1,
padding='valid',
kernel_initializer='he_uniform')(x_lstm)
avg_pool1_lstm = GlobalAveragePooling1D()(x1)
max_pool1_lstm = GlobalMaxPooling1D()(x1)
x = concatenate(
[avg_pool1_gru, max_pool1_gru, avg_pool1_lstm, max_pool1_lstm])
x = BatchNormalization()(x)
x = Dropout(dr)(Dense(dense_units, activation='relu')(x))
x = Dense(2, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
model.compile(loss="binary_crossentropy",
optimizer=Adam(lr=lr, decay=lr_d),
metrics=["accuracy"])
history = model.fit(X_train,
y_ohe,
batch_size=128,
epochs=5,
validation_split=0.1,
verbose=1,
callbacks=[check_point, early_stop])
model = load_model(file_path)
return model
def BiLSTM2_CNN3_model(args, embedding_matrix):
VOCAB_SIZE = args.vocab_size
EMBEDDING_DIM = args.emb_dim
MAX_LENGTH = args.max_len
NUM_EPOCHS = args.num_epochs
LSTM_OUT = args.lstm_out
DROPOUT = args.dropout
FILTERS = args.num_filters
KERNEL_SIZE = args.kernel_size
STRIDES = args.strides
inp = Input((MAX_LENGTH, ))
emb = Embedding(VOCAB_SIZE,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_LENGTH)(inp)
emb = Dropout(DROPOUT)(emb)
conv1 = Conv1D(FILTERS,
KERNEL_SIZE,
padding="valid",
activation="relu",
strides=STRIDES)(emb)
pool1 = GlobalMaxPooling1D()(conv1)
conv2 = Conv1D(FILTERS,
KERNEL_SIZE + 1,
padding="valid",
activation="relu",
strides=STRIDES)(emb)
pool2 = GlobalMaxPooling1D()(conv2)
conv3 = Conv1D(FILTERS,
KERNEL_SIZE + 2,
padding="valid",
activation="relu",
strides=STRIDES)(emb)
pool3 = GlobalMaxPooling1D()(conv3)
lstm = Bidirectional(
LSTM(LSTM_OUT,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2))(emb)
drop = Dropout(DROPOUT)(lstm)
lstm2 = Bidirectional(LSTM(LSTM_OUT, dropout=0.2,
recurrent_dropout=0.2))(drop)
combined = concatenate([pool1, pool2, pool3, lstm2])
dense1 = Dense(512, activation="relu")(combined)
drop1 = Dropout(DROPOUT)(dense1)
dense2 = Dense(128, activation="relu")(drop1)
drop2 = Dropout(DROPOUT / 2)(dense2)
dense3 = Dense(32, activation="relu")(drop2)
output = Dense(1, activation="sigmoid")(dense3)
return Model(inp, output)
def build_model(
hparams
):
input_layer = Input(shape=(hparams["max_sequence_length"], ))
embedding_layer_static = get_w2v('').get_keras_embedding(train_embeddings=False)(input_layer)
embedding_layer = get_w2v('').get_keras_embedding(train_embeddings=True)(input_layer)
submodels = []
kernel_sizes = hparams['kernel_sizes'].split('-')
for ks in kernel_sizes:
model = Sequential()
conv_1_d = Conv1D(
activation = 'relu',
filters = hparams["filters"],
kernel_size = int(ks),
kernel_constraint = max_norm(hparams["max_norm_value"])
)
conv_layer_static = conv_1_d(embedding_layer_static)
conv_layer = conv_1_d(embedding_layer)
max_pooling_static = GlobalMaxPooling1D()(conv_layer_static)
max_pooling = GlobalMaxPooling1D()(conv_layer)
concatenate_layer = Concatenate()([max_pooling_static, max_pooling])
submodels.append(concatenate_layer)
concat = Concatenate()(submodels)
dropout_layer_1 = Dropout(hparams['dropout_ratio'])(concat)
hidden_layer = Dense(
hparams['hidden_size'],
activation = 'relu',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(len(kernel_sizes) * 2* hparams['filters']),
maxval = 1 / np.sqrt(len(kernel_sizes) * 2 * hparams['filters'])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_1)
dropout_layer_2 = Dropout(hparams['dropout_ratio'])(hidden_layer)
output_layer = Dense(
2,
activation = 'sigmoid',
kernel_initializer = initializers.RandomUniform(
minval = - 1 / np.sqrt(hparams['hidden_size']),
maxval = 1 / np.sqrt(hparams['hidden_size'])
),
bias_initializer = initializers.Zeros(),
kernel_regularizer = regularizers.l2(hparams['l2_regularization'])
)(dropout_layer_2)
model = Model(inputs=[input_layer], outputs=[output_layer])
model.compile(
loss = dice_loss,
optimizer = Adam(learning_rate = hparams["learning_rate"]),
metrics = [f1_score]
)
from keras.utils.vis_utils import plot_model
plot_model(model, "model_cnn_multichannel.png", show_layer_names=False)
return model
def multichannel_cnn():
"""
Build the Multichannel CNN model
"""
feature_length = 600
input_layer = Input(shape=(feature_length, ))
# Load the embedding weights and set the embedding layer
weights = np.load(open('weights/100d.npy', 'rb'))
embedding_layer = Embedding(input_dim=weights.shape[0],
output_dim=weights.shape[1],
mask_zero=False,
weights=[weights],
trainable=False)
embedding = embedding_layer(input_layer)
embedding = Dropout(0.6)(embedding)
# channel 1
channel_1 = Conv1D(filters=1024,
kernel_size=2,
padding='valid',
activation='relu')(embedding)
channel_1 = GlobalMaxPooling1D()(channel_1)
# channel 2
channel_2 = Conv1D(filters=1024,
kernel_size=4,
padding='valid',
activation='relu')(embedding)
channel_2 = GlobalMaxPooling1D()(channel_2)
# Fully connected network
fully_connected = Concatenate()([channel_1, channel_2])
fully_connected = Dropout(0.4)(fully_connected)
fully_connected = Dense(128,
activation='relu',
kernel_constraint=unit_norm(),
bias_constraint=unit_norm())(fully_connected)
fully_connected = Dropout(0.4)(fully_connected)
output = Dense(1,
activation='sigmoid',
kernel_constraint=unit_norm(),
bias_constraint=unit_norm())(fully_connected)
model = Model(inputs=(input_layer), outputs=output)
# Model settings
metrics = [
WorkSavedOverSamplingAtRecall(recall=1, name='wss'),
WorkSavedOverSamplingAtRecall(recall=0.95, name='wss_95')
]
opt = optimizers.Adam(1e-4)
model.compile(optimizer=opt, loss='binary_crossentropy', metrics=metrics)
return model
def model(inputlen,
vocabulary,
vector_dim,
embedding_dim=128,
lstm_unit=100,
num_class=0,
drop_rate=0.5,
l2=0.01,
train=False,
init_emb_enable=False,
init_emb=None,
attention_enable=False):
input = Input(shape=(inputlen))
if train == True:
label = Input(shape=(num_class))
if init_emb_enable == False:
embeddings_init = 'uniform'
else:
embeddings_init = keras.initializers.constant(init_emb)
embedding = Embedding(input_dim=vocabulary,
output_dim=embedding_dim,
embeddings_initializer=embeddings_init,
input_length=inputlen)(input)
reshape = Reshape((inputlen, embedding_dim))(embedding)
if attention_enable == True:
lstm = Bidirectional(LSTM(lstm_unit,
return_sequences=True))(reshape)
attention_output = Attention()([lstm, lstm])
pool_output1 = GlobalMaxPooling1D()(lstm)
pool_output2 = GlobalMaxPooling1D()(attention_output)
lstm = Concatenate()([pool_output1, pool_output2])
else:
lstm = Bidirectional(LSTM(lstm_unit,
return_sequences=False))(reshape)
if drop_rate != 0:
dropout = Dropout(drop_rate)(lstm)
else:
dropout = lstm
dense = Dense(vector_dim,
activity_regularizer=regularizers.l2(l2))(dropout)
bn = BatchNormalization()(dense)
if train == True:
output = ArcFace(n_classes=num_class)([bn, label])
model = Model([input, label], output)
else:
model = Model(input, bn)
return model
def __call__(self, inputs):
a = inputs[0]
b = inputs[1]
a_avg = GlobalAveragePooling1D()(a)
a_max = GlobalMaxPooling1D()(a)
b_avg = GlobalAveragePooling1D()(b)
b_max = GlobalMaxPooling1D()(b)
return concatenate([a_avg, a_max, b_avg, b_max])
def brnn(self):
inputs = Input(shape=(image_size,image_size), dtype='float32', name='inputs')
brnn1 = Bidirectional(LSTM(lstm_dim, return_sequences=True))(inputs)
x1 = GlobalMaxPooling1D()(brnn1)
inputs_ = Lambda(lambda t: K.permute_dimensions(t, pattern=(0,2,1)))(inputs)
brnn2 = Bidirectional(LSTM(lstm_dim, return_sequences=True))(inputs_)
x2 = GlobalMaxPooling1D()(brnn2)
concat = Concatenate(axis=1)
x = concat([x1,x2])
self.model = Model(inputs, x)
def define_model(input_shape_text,
input_shape_emoji): #input_shape = embeddings dimensions
# TODO: adapt filters, add regularizers
# channel 1 : textual features
input1 = Input(shape=input_shape_text)
conv1 = Conv1D(filters=128, kernel_size=4,
activation='relu')(input1) #(embedding1) #filter size 4
#drop1 = Dropout(0.5)(conv1) # higher drop out = often slower and more consistent learning
pool1 = GlobalMaxPooling1D()(conv1)
drop1 = Dropout(0.2)(pool1)
flat1 = Flatten()(drop1)
#dense1 = Dense(256, activation = 'relu')
# channel 2: emoji features
input2 = Input(shape=input_shape_emoji)
conv2 = Conv1D(filters=64, kernel_size=2,
activation='relu')(input2) #(embedding2) #filter size 6
# drop2 = Dropout(0.5)(conv2)
pool2 = GlobalMaxPooling1D()(conv2)
drop2 = Dropout(0.2)(pool2)
flat2 = Flatten()(drop2)
#d ense2 = Dense(256, activation = 'relu')
# # channel 3: image features
# input3 = Input(shape=input_shape_img)
# conv3 = Conv2D(filters = 128, kernel_size = 1, activation = 'relu')(input3) #(embedding3) #filter size 8
# conv4 = Conv2D(filters = 128, kernel_size = 3, activation = 'relu')(conv3)
# # drop3 = Dropout(0.5)(input4)
# pool3 = GlobalMaxPooling2D()(conv4)
# drop3 = Dropout(0.2)(pool3)
# flat3 = Flatten()(drop3)
# #dense3 = Dense(256, activation = 'relu')
# merge + classification layer
merged = concatenate([flat1, flat2])
dense1 = Dense(128, activation='relu')(merged)
dropout = Dropout(0.5)(dense1)
outputs = Dense(3, activation='softmax')(dropout)
model = Model(inputs=[input1, input2], outputs=outputs)
# compile
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.RMSprop(learning_rate=0.001),
metrics=['accuracy'])
# summary
print(model.summary())
return model
def __init__(self,
maxlen,
max_features,
embedding_dims,
kernel_sizes=[1, 2, 3, 4, 5],
class_num=1,
last_activation='sigmoid'):
super(RCNNVariant, self).__init__()
self.maxlen = maxlen
self.max_features = max_features
self.embedding_dims = embedding_dims
self.kernel_sizes = kernel_sizes
self.class_num = class_num
self.last_activation = last_activation
self.embedding = Embedding(self.max_features,
self.embedding_dims,
input_length=self.maxlen)
self.bi_rnn = Bidirectional(LSTM(128, return_sequences=True))
self.concatenate = Concatenate()
self.convs = []
for kernel_size in self.kernel_sizes:
conv = Conv1D(128, kernel_size, activation='relu')
self.convs.append(conv)
self.avg_pooling = GlobalAveragePooling1D()
self.max_pooling = GlobalMaxPooling1D()
self.classifier = Dense(self.class_num,
activation=self.last_activation)
def __init__(self,
padding_size,
vocab_size,
embed_size,
filter_num,
num_classes,
kernel_sizes=[3,4,5],
kernel_regularizer=None,
last_activation='softmax'):
super(TextCNN,self).__init__()
self.padding_size=padding_size
self.kernel_sizes=kernel_sizes
self.num_classes=num_classes
embed_initer = tf.keras.initializers.RandomUniform(minval=-1, maxval=1)
self.embedding=Embedding(input_dim=vocab_size,
output_dim=embed_size,
input_length=padding_size,
embeddings_initializer=embed_initer,
name='embedding')
self.conv1s=[]
self.avgpools=[]
for kernel_size in kernel_sizes:
self.conv1s.append(Conv1D(filters=filter_num,kernel_size=kernel_size,activation='relu',kernel_regularizer=kernel_regularizer))
self.avgpools.append(GlobalMaxPooling1D())
self.classifier=Dense(num_classes,activation=last_activation,)
def model():
input_length = 428
input_dim = 203169
num_classes = 7
embedding_dim = 500
lstm_units = 128
lstm_dropout = 0.1
recurrent_dropout = 0.1
filters=64
kernel_size=3
input_layer = Input(shape=(input_length,))
output_layer = Embedding(
input_dim=input_dim,
output_dim=embedding_dim,
input_shape=(input_length,)
)(input_layer)
output_layer = Bidirectional(
LSTM(lstm_units, return_sequences=True,
dropout=lstm_dropout, recurrent_dropout=recurrent_dropout)
)(output_layer)
output_layer = Conv1D(filters, kernel_size=kernel_size, padding='valid',
kernel_initializer='glorot_uniform')(output_layer)
avg_pool = GlobalAveragePooling1D()(output_layer)
max_pool = GlobalMaxPooling1D()(output_layer)
output_layer = concatenate([avg_pool, max_pool])
output_layer = Dense(num_classes, activation='softmax')(output_layer)
model = Model(input_layer, output_layer)
file = os.path.join(THIS_FOLDER, 'static/model.h5')
model.load_weights(file)
return model
def __init__(self,
input_dim,
output_dim,
filters=250,
kernel_size=3,
emb_dim=300,
embeddings=None,
trainable=True):
self.input = Input(shape=(None, ), name='input')
if embeddings is None:
self.embedding = Embedding(input_dim=input_dim,
output_dim=emb_dim,
trainable=trainable,
name='embedding')
else:
self.embedding = Embedding(input_dim=embeddings.shape[0],
output_dim=embeddings.shape[1],
trainable=trainable,
weights=[embeddings],
name='embedding')
self.conv = Conv1D(filters,
kernel_size,
padding='valid',
activation='relu',
strides=1)
self.pool = GlobalMaxPooling1D()
self.fc = Dense(output_dim, activation='softmax')
def _build_sentence_model(self, input_name):
# add input layer as first
input = Input(shape=(self.max_text_len, ),
name=str.format(f'{input_name}_input'))
# we will load embedding values from corpus here.
embed = Embedding(input_dim=self.embedding_matrix.shape[0],
output_dim=self.embedding_matrix.shape[1],
input_length=self.max_text_len,
weights=[self.embedding_matrix],
name=str.format(f'{input_name}_embed'),
trainable=False)(input)
# add convolutional layer
conv = Conv1D(filters=self.no_conv_filters,
kernel_size=5,
strides=1,
padding='same',
activation='relu',
kernel_regularizer=regularizers.l2(1e-5),
name=str.format(f'{input_name}_conv'))(embed)
# add some dropout rate for prevent overfitting
conv = Dropout(self.dropout_rate)(conv)
x_pool = GlobalMaxPooling1D(
name=str.format(f'{input_name}_pool'))(conv)
return input, x_pool
def emotion_recognition_model(input_length,
input_dim,
num_classes,
embedding_layer,
embedding_dim=100,
dropout=0.1,
gru_units=128,
gru_dropout=0.1,
recurrent_dropout=0.1):
model = Sequential()
if embedding_layer:
model.add(embedding_layer)
else:
model.add(
Embedding(input_dim=input_dim,
output_dim=embedding_dim,
input_shape=(input_length, )))
model.add(
Bidirectional(
LSTM(gru_units,
return_sequences=True,
dropout=gru_dropout,
recurrent_dropout=recurrent_dropout)))
model.add(GlobalMaxPooling1D())
model.add(Dense(32, activation='relu'))
model.add(Dropout(dropout))
model.add(Dense(num_classes, activation='sigmoid'))
return model
def build_model(word_vocab_size, word_vector_size, word_embedding_matrix,
output_size, max_sequence_len, filters, filter_count,
reg_lambda, activation, padding):
word_embedding_layer = Embedding(word_vocab_size,
word_vector_size,
weights=[word_embedding_matrix],
input_length=max_sequence_len,
trainable=False)
word_sequence_input = Input(shape=(max_sequence_len, ))
word_embedded_sequences = word_embedding_layer(word_sequence_input)
layers = []
for filter_size in filters:
layer = Conv1D(filter_count,
filter_size,
activation=activation,
padding=padding)(word_embedded_sequences)
layer = GlobalMaxPooling1D()(layer)
layers.append(layer)
layer = Concatenate()(layers)
preds = Dense(output_size, activation="softmax")(layer)
model = Model(word_sequence_input, preds)
return model
def cnn_model(self, x_train, x_test, y_train, y_test):
classes_num = self.dataset.getParameters()["classes_num"]
model = Sequential()
model.add(Embedding(
self.VOCAB_SIZE, self.EMBEDING_DIM, input_length=self.INPUT_LENGTH))
model.add(Dropout(0.5))
model.add(Conv1D(128, 5, activation='relu'))
model.add(GlobalMaxPooling1D())
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(classes_num, activation=self.ACTIVATION))
model.compile(optimizer='adam',
loss=self.LOSSFUNC,
metrics=['accuracy'])
es_callback = EarlyStopping(
monitor='val_loss', patience=4)
model.summary()
history = model.fit(x_train, y_train,
epochs=self.EPOCHS,
verbose=1,
validation_data=(x_test, y_test),
batch_size=self.BATCH_SIZE, callbacks=[es_callback])
loss, accuracy = model.evaluate(x_train, y_train, verbose=1)
print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(x_test, y_test, verbose=1)
print("Testing Accuracy: {:.4f}".format(accuracy))
return history
def get_model(self, pre_embeddings, dp_rate=-1.0, filter_sizes=[2, 3, 4]):
"""
:param pre_embeddings:
:param dp_rate: drop out rate
:param filter_sizes: sizes of convolution kernels
:return: the model
"""
# Embedding part can try multichannel as same as origin paper
embedding_layer = Embedding(self.max_features, # 字典长度
self.embedding_dims, # 词向量维度
weights=[pre_embeddings], # 预训练的词向量
input_length=self.maxlen, # 每句话的最大长度
trainable=False # 是否在训练过程中更新词向量
)
input = Input((self.maxlen,))
embedding = embedding_layer(input)
convs = []
for kernel_size in filter_sizes:
c = Conv1D(NUM_FILTERS, kernel_size, activation='relu')(embedding)
c = GlobalMaxPooling1D()(c)
convs.append(c)
x = Concatenate()(convs)
if dp_rate > 0:
# 加dropout层
x = Dropout(dp_rate)(x)
output = Dense(self.class_num, activation=self.last_activation)(x)
model = Model(inputs=input, outputs=output)
return model
def model_5(filters, kernel_size, optimizer, strides):
# then we can go ahead and set the parameter space
model = Sequential()
# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
model.add(Embedding(max_features, embedding_dims, input_length=maxlen))
model.add(
Conv1D((filters), (kernel_size),
padding='valid',
activation='relu',
strides=strides))
# we use max pooling:
model.add(GlobalMaxPooling1D())
# We add a vanilla hidden layer:
model.add(Dense(hidden_dims))
model.add(Dropout(dropout_default), )
model.add(Activation('relu'))
# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(output_size))
model.add(Activation('sigmoid'))
#optimizer = 'adam'
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.summary()
return model
def get_model(self):
input_current = Input((self.maxlen, ))
input_left = Input((self.maxlen, ))
input_right = Input((self.maxlen, ))
embedder = Embedding(self.max_features,
self.embedding_dims,
input_length=self.maxlen)
embedding_current = embedder(input_current)
embedding_left = embedder(input_left)
embedding_right = embedder(input_right)
x_left = LSTM(128, return_sequences=True)(embedding_left)
x_right = LSTM(128, return_sequences=True,
go_backwards=True)(embedding_right)
x_right = Lambda(lambda x: K.reverse(x, axes=1))(x_right)
x = Concatenate(axis=2)([x_left, embedding_current, x_right])
x = Conv1D(64, kernel_size=1, activation='tanh')(x)
x = GlobalMaxPooling1D()(x)
output = Dense(self.class_num, activation=self.last_activation)(x)
model = Model(inputs=[input_current, input_left, input_right],
outputs=output)
return model
def keras_model_fn(_, config):
"""
Creating a CNN model for sentiment modeling
"""
print("Creating a model...")
embeddings_matrix = load_embeddings(
config["embeddings_path"], config["embeddings_dictionary_size"],
config["embeddings_vector_size"])
cnn_model = Sequential()
embeddings = Embedding(
config["embeddings_dictionary_size"], config["embeddings_vector_size"],
weights=[np.array(embeddings_matrix)], input_length=config["padding_size"], trainable=True)
cnn_model.add(embeddings)
cnn_model.add(Conv1D(filters=100, kernel_size=2, padding="valid", activation="relu", strides=1))
cnn_model.add(GlobalMaxPooling1D())
cnn_model.add(Dense(256, activation="relu"))
cnn_model.add(Dense(1, activation="sigmoid"))
cnn_model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
return cnn_model
def make_model(embed_size, embedding_matrix, max_features=20000, maxlen=50):
inp = Input(shape=(maxlen, ))
x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(
GRU(128,
return_sequences=True,
activation="tanh",
recurrent_activation="sigmoid",
use_bias="True",
reset_after="True",
unroll="False"))(x)
# For CuDNN implementation with tf2
x = Dropout(0.2)(x)
x = Conv1D(64,
kernel_size=3,
padding="valid",
kernel_initializer="glorot_uniform")(x)
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
x = concatenate([avg_pool, max_pool])
preds = Dense(6, activation="sigmoid")(x)
model = Model(inp, preds)
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-4),
metrics=['accuracy'])
return model
def get_model(embedding_matrix=embedding_matrix):
words = Input(shape=(MAX_LEN, ))
x = Embedding(*embedding_matrix.shape,
weights=[embedding_matrix],
trainable=False)(words)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True))(x)
x = Bidirectional(LSTM(LSTM_UNITS, return_sequences=True))(x)
hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
hidden = add(
[hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu')(hidden)])
result = Dense(1, activation='sigmoid')(hidden)
model = Model(inputs=words, outputs=result)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[f1_score])
return model
def create_model_DNN(input_dim,
embedding_dim,
embedding_matrix,
pad_len,
trainable=False,
n1=64,
n2=32,
kr=None,
br=None):
Simple = Sequential()
embedding_layer = Simple.add(
Embedding(input_dim=input_dim,
output_dim=embedding_dim,
weights=[embedding_matrix],
input_length=pad_len,
trainable=trainable))
Simple.add(GlobalMaxPooling1D())
Simple.add(
Dense(n1,
kernel_regularizer=kr,
bias_regularizer=br,
activation='relu'))
Simple.add(
Dense(n2,
kernel_regularizer=kr,
bias_regularizer=br,
activation='relu'))
Simple.add(Dense(4, activation='softmax'))
Simple.compile(optimizer='adam',
loss="sparse_categorical_crossentropy",
metrics=['acc'])
return Simple
def __init__(self,
feature_columns,
mode,
attention_hidden_unit=64,
embed_reg=1e-4):
"""
AFM architecture
:param feature_columns: dense_feature_columns and sparse_feature_columns
:param mode: 'max'(MAX Pooling) or 'avg'(Average Pooling) or 'att'(Attention)
:param attention_hidden_unit: hidden unit, if mode == 'att'
:param embed_reg: the regularizer of embedding
"""
super(AFM, self).__init__()
self.dense_feature_columns, self.sparse_feature_columns = feature_columns
self.mode = mode
self.embed_layers = {
'embed_' + str(i):
Embedding(input_dim=feat['feat_num'],
input_length=1,
output_dim=feat['embed_dim'],
embeddings_initializer='random_uniform',
embeddings_regularizer=l2(embed_reg))
for i, feat in enumerate(self.sparse_feature_columns)
}
if self.mode == 'max':
self.max = GlobalMaxPooling1D()
elif self.mode == 'avg':
self.avg = GlobalAveragePooling1D()
else:
self.attention_dense = Dense(units=attention_hidden_unit,
activation='relu')
self.attention_dense2 = Dense(units=1, activation=None)
self.dense = Dense(units=1, activation=None)
self.concat = Concatenate(axis=-1)
def create_model_CNN(input_dim,
embedding_dim,
embedding_matrix,
pad_len,
trainable,
n1,
k,
d=0.0,
kr=None,
br=None):
myCNN = Sequential()
myCNN.add(
Embedding(input_dim=input_dim,
output_dim=embedding_dim,
weights=[embedding_matrix],
input_length=pad_len,
trainable=trainable))
myCNN.add(
Conv1D(n1,
kernel_size=k,
kernel_regularizer=kr,
bias_regularizer=br,
activation='relu'))
myCNN.add(GlobalMaxPooling1D())
myCNN.add(Dropout(d))
myCNN.add(Dense(4, activation='softmax'))
myCNN.compile(optimizer='adam',
loss="sparse_categorical_crossentropy",
metrics=['acc'])
return myCNN
def create_model(num_labels, units_gru, dense_size):
dropout_rate = 0.5
rec_dropout_rate = 0.2
inp = Input(shape=(WORD_LIMIT, EMBEDDING_DIM))
output = Dropout(rate=dropout_rate)(inp)
output, state1, state2 = Bidirectional(
GRU(units_gru,
activation='tanh',
return_sequences=True,
return_state=True,
dropout=dropout_rate,
recurrent_dropout=rec_dropout_rate))(output)
output1 = GlobalMaxPooling1D()(output)
output2 = GlobalAveragePooling1D()(output)
output = concatenate([output1, output2, state1, state2])
output = Dropout(rate=dropout_rate)(output)
output = Dense(dense_size, activation=None)(output)
output = Activation('relu')(output)
output = Dropout(rate=dropout_rate)(output)
output = Dense(num_labels, activation=None)(output)
act_output = Activation("sigmoid")(output)
model = Model(inputs=inp, outputs=act_output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def build_model(sent_length, embeddings_weight, class_num):
content = Input(shape=(sent_length, ), dtype='int32')
embedding = Embedding(name="sentence_cuted",
input_dim=embeddings_weight.shape[0],
weights=[embeddings_weight],
output_dim=embeddings_weight.shape[1],
trainable=False)
x = SpatialDropout1D(0.2)(embedding(content))
x = Bidirectional(GRU(200, return_sequences=True))(x)
# x = Bidirectional(GRU(200, return_sequences=True))(x)
avg_pool = GlobalAveragePooling1D()(x) # 全军平均池化
max_pool = GlobalMaxPooling1D()(x) # 全局最大池化
conc = concatenate([avg_pool, max_pool]) # 特征放到一起，
x = Dense(1000)(conc) # 全连接层
x = BatchNormalization()(x)
x = Activation(activation="relu")(x)
x = Dropout(0.2)(x)
x = Dense(500)(x)
x = BatchNormalization()(x)
x = Activation(activation="relu")(x)
output = Dense(class_num, activation="softmax")(x)
model = tf.keras.models.Model(inputs=content, outputs=output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
