def model():
quest_input = Input(shape=(maxQLen, ), dtype='int32')
questC_input = Input(shape=(maxQLen, maxWordLen), dtype='int32')
# token embedding
embed_layer = Embedding(vocab_size + 2, embedding_dim, mask_zero=True)
quest_emb = SpatialDropout1D(0.8)(embed_layer(quest_input))
quest_emb = BatchNormalization()(quest_emb)
quest_emb = Masking()(quest_emb) # (None, 16)
# char embedding
char_input = Input(shape=(maxWordLen, ), dtype='int32') # (None, 5)
cembed_layer = Embedding(char_size + 2, char_embd_dim)(char_input)
c_emb = SpatialDropout1D(0.8)(cembed_layer)
cc = BatchNormalization()(c_emb)
cc = Conv1D(char_embd_dim, 3, padding='same')(cc)
cc = LeakyReLU()(cc)
cc = Lambda(lambda x: K.sum(x, 1), output_shape=lambda d:
(d[0], d[2]))(cc) # (None, 16)
char_model = Model(inputs=char_input, outputs=cc)
char_model.summary()
qc_emb = TimeDistributed(char_model)(questC_input)
quest_emb = concatenate([quest_emb, qc_emb]) # (None, 30, 32)
final = GlobalAveragePooling1DMasked()(quest_emb) # Polling1D只能处理2阶tensor
final = Dense(typelen, activation='softmax')(final)
mm = Model(inputs=[quest_input, questC_input], outputs=final)
mm.compile(AdamWithWeightnorm(0.01),
'sparse_categorical_crossentropy',
metrics=['accuracy'])
return mm
def run_model():
global auc
model = Sequential()
model.add(Embedding(TOP_WORDS, 32, input_length=max_story_length))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=5,
validation_split=0.2)
# evaluate
loss, acc = model.evaluate(X_test, Y_test, verbose=0)
y_true, y_pred = Y_test, model.predict(X_test).round()
fpr, tpr, thresholds = roc_curve(y_true, y_pred)
auc = auc(fpr, tpr)
print('Test Accuracy: %f' % (acc), 'Test AUC-score: %f' % (auc))
return model
def train_lstm(dropout):
args = locals()
logger.debug(f"Try to training with paras:{args}")
#global MAX_SENTENCE_LENGTH, EMBEDDING_SIZE, HIDDEN_LAYER_SIZE, BATCH_SIZE, NUM_EPOCHS, X, y, Xtest, ytest, model, history
MAX_SENTENCE_LENGTH = 40
EMBEDDING_SIZE = 128
HIDDEN_LAYER_SIZE = 64
BATCH_SIZE = 32
NUM_EPOCHS = 1000
df = get_lstm_feature()
X = df.X
y = df.y
vocab_size = df['para'].max()
print(
f'Test(raw) feature is ready{X.shape}, {y.shape},vocab_size:{vocab_size}'
)
print(y.shape)
y = np_utils.to_categorical(y, 22)
# Pad the sequences (left padded with zeros)
X = sequence.pad_sequences(X, maxlen=MAX_SENTENCE_LENGTH)
print('Test feature is ready')
# Split input into training and test
Xtrain, Xtest, ytrain, ytest = train_test_split(X,
y,
test_size=0.3,
random_state=42)
print(Xtrain.shape, Xtest.shape, ytrain.shape, ytest.shape)
# Build model
model = Sequential()
model.add(
Embedding(vocab_size, EMBEDDING_SIZE,
input_length=MAX_SENTENCE_LENGTH))
model.add(SpatialDropout1D(dropout))
model.add(
LSTM(HIDDEN_LAYER_SIZE, dropout=dropout, recurrent_dropout=dropout))
model.add(Dense(22))
model.add(Activation("softmax"))
model.compile(
loss='categorical_crossentropy',
optimizer='adam',
# loss="binary_crossentropy", optimizer="adam",
# metrics=["accuracy"]
)
# check_best = ModelCheckpoint(filepath=replace_invalid_filename_char(file_path),
# monitor='val_loss', verbose=1,
# save_best_only=True, mode='min')
early_stop = EarlyStopping(
monitor='val_loss',
verbose=1,
patience=20,
)
print('Begin training')
history = model.fit(Xtrain,
ytrain,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
callbacks=[early_stop],
validation_data=(Xtest, ytest))
print('End training')
return model, history, args
def run_model():
global auc
model = Sequential()
model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(2, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X_train,
Y_train,
epochs=epochs,
batch_size=64,
validation_split=0.1)
loss, acc = model.evaluate(X_test, Y_test, verbose=2)
y_true = pd.Series(Y_test.columns[np.where(Y_test != 0)[1]])
y_predict = model.predict(X_test).round()
y_pred = []
for i in y_predict:
y_pred.append(np.argmax(i))
fpr, tpr, thresholds = roc_curve(y_true, y_pred)
auc = auc(fpr, tpr)
print('Test Accuracy: %f' % (acc), 'Test AUC-score: %f' % (auc), "\n")
return model
def get_model(embeddings_layer, sequence_length, nb_classes):
word_input = Input(shape=(sequence_length,), name="X_input")
embedded = embeddings_layer(word_input)
embedded_d = SpatialDropout1D(0.4)(embedded)
conv1 = Convolution1D(filters=64, kernel_size=2, padding="same",
activation="relu")(embedded_d)
conv2 = Convolution1D(filters=64, kernel_size=4, padding="same",
activation="relu")(embedded_d)
conv3 = Convolution1D(filters=64, kernel_size=6, padding="same",
activation="relu")(embedded_d)
conv4 = Convolution1D(filters=64, kernel_size=8, padding="same",
activation="relu")(embedded_d)
conv5 = Convolution1D(filters=64, kernel_size=10, padding="same",
activation="relu")(embedded_d)
merge_layer = concatenate([conv1, conv2, conv3, conv4, conv5],
name="convolutional_concat")
blstm = Bidirectional(LSTM(units=100, return_sequences=True,
recurrent_dropout=0.5))(merge_layer)
blstm = Bidirectional(LSTM(units=100, return_sequences=True,
recurrent_dropout=0.5))(blstm)
predict = Dense(nb_classes, activation="softmax")(blstm)
model = Model(word_input, predict)
model.compile(
optimizer="Adam",
loss="categorical_crossentropy",
metrics=["accuracy"]
)
return model
def create_model(embeddings, config=get_config(), sentence_length=100):
config['sentence_length'] = sentence_length
embedding_layer = Embedding(
embeddings.shape[0],
embeddings.shape[1],
input_length=config['sentence_length'],
trainable=False,
weights=[embeddings],
)
input = Input(shape=(config['sentence_length'],), dtype='int32')
x = embedding_layer(input)
x = SpatialDropout1D(config['embedding_dropout'])(x)
x = Lambda(lambda x: K.sum(x, axis=1))(x)
if config['dense_layer']:
x = Dense(config['dense_layer'], activation='sigmoid')(x)
x = Dropout(config['dropout_prob'])(x)
output = Dense(1, activation='sigmoid',)(x)
model = Model(inputs=input, outputs=output)
return model, config
def trainAndSaveModel(Xtrain, Ytrain):
'''
根据数据与标签已经全局参数训练并存储模型
'''
print(MAX_LEN)
print(VOCAB_SIZE)
model = Sequential()
model.add(Embedding(VOCAB_SIZE, EMBED_SIZE, input_length=MAX_LEN))
model.add(SpatialDropout1D(0.2))
model.add(
Conv1D(filters=NUM_FILTERS,
kernel_size=NUM_WORDS,
padding="valid",
activation="relu"))
model.add(GlobalAveragePooling1D())
model.add(Dense(1, activation="sigmoid"))
# model.add(Dense(2, activation="softmax"))
# 编译模型，因为目标是二分类，所以选用binary_crossentropy作为我们损失函数
# 优化器选择adam
# 之后用训练集训练集模型，批大小设置为 BATCH_SIZE ，训练 NUM_EPOCHS 个周期
# model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.summary()
model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS)
model.save(MODEL_FILE)
def get_model():
global embedding_dim
embed_size = embedding_dim
inp = Input(shape=(maxlen, embedding_dim))
# x = Embedding(max_features, embed_size)(inp)
# x = Dropout(0.4)(x)
x = SpatialDropout1D(0.2)(inp)
x = Bidirectional(CuDNNGRU(50, return_sequences=True))(x)
# x = BatchNormalization()(x)
x = Activation('relu')(x)
# x = Dropout(0.4)(x)
# x = Bidirectional(CuDNNLSTM(50, return_sequences=True))(x)
A = AttentionWeightedAverage(name='attlayer', return_attention=False)(x)
# A = AttentionWithContext()(x)
# x = AttentionWithContext()(x)
# C = Crop(2,-2,-1)(x)
# C = Reshape([-1,])(C)
# print(C)
G = GlobalMaxPool1D()(x)
x = Concatenate()([A, G])
x = Dropout(0.1)(x)
x = Dense(50, activation=None)(x)
# x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.1)(x)
x = Dense(6, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
# model.layers[1].set_weights([embedding_matrix])
# model.layers[1].trainable = False
model.compile(loss='binary_crossentropy',
optimizer=Adam(amsgrad=True),
metrics=['accuracy'])
return model
def build_model(self):
data = self.data
inputs = Input(shape=(data.seq_length, ))
embeddings = Embedding(
data.max_feature,
data.embed_dim,
weights=[data.embed_matrix],
trainable=False)(inputs)
x = SpatialDropout1D(self.dropout)(embeddings)
x = Bidirectional(CuDNNLSTM(data.embed_dim, return_sequences=True))(x)
x = Bidirectional(CuDNNLSTM(data.embed_dim, return_sequences=True))(x)
ave_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
conc = concatenate([ave_pool, max_pool])
# bn = BatchNormalization()(pool)
x = Dense(self.dense_size, activation="relu")(conc)
# drop = Dropout(self.dropout)(x)
outputs = Dense(6, activation="sigmoid")(x)
model = Model(inputs=inputs, outputs=outputs)
optimizer = self.get_optimizer(self.lr, self.optim_name)
model.compile(
loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
self.model = model
def create_model(vocabularySize):
#model = Sequential()
#
#model.add(Embedding(vocabularySize, 32, input_length=MAX_LEN))
#model.add(Flatten())
#model.add(Dense(512, activation="relu"))
#model.add(Dense(256, activation="relu"))
#model.add(Dense(128, activation="relu"))
#model.add(Dense(1, activation="linear"))
#
#model.compile(loss='mse',
# optimizer='adadelta',
# metrics=['accuracy'])
model = Sequential()
model.add(Embedding(vocabularySize, 256, input_length=MAX_LEN))
model.add(SpatialDropout1D(0.2))
model.add(GRU(256, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='mse', optimizer='adadelta', metrics=['accuracy'])
return model
def set_recurrent_network(mdl_dict, reverse_dictionary):
"""
Create a RNN network and set its parameters
"""
dimensions = len(reverse_dictionary) + 1
lr, embedding_size, dropout, units, batch_size, loss, activation_recurrent, activation_output = get_best_parameters(
mdl_dict)
# define the architecture of the recurrent neural network
model = Sequential()
model.add(Embedding(dimensions, embedding_size, mask_zero=True))
model.add(SpatialDropout1D(dropout))
model.add(
GRU(units,
dropout=dropout,
recurrent_dropout=dropout,
return_sequences=True,
activation=activation_recurrent))
model.add(Dropout(dropout))
model.add(
GRU(units,
dropout=dropout,
recurrent_dropout=dropout,
return_sequences=False,
activation=activation_recurrent))
model.add(Dropout(dropout))
model.add(Dense(dimensions, activation=activation_output))
optimizer = RMSprop(lr=lr)
model.compile(loss=loss, optimizer=optimizer)
return model
def create_model(embeddings, config=get_config(), sentence_length=100):
config['sentence_length'] = sentence_length
# sentence attention
attention_input = Input(shape=(
config['sentence_length'],
config['embedding_size'],
),
dtype='float32')
x = Permute((2, 1))(attention_input)
x = Reshape((config['embedding_size'], config['sentence_length']))(x)
x = Dense(config['sentence_length'], activation='softmax', bias=True)(x)
x = Lambda(lambda x: K.mean(x, axis=1),
name='attention_vector_sentence')(x)
x = RepeatVector(config['embedding_size'])(x)
# x = Lambda(lambda x: x, name='attention_vector_sentence')(x)
attention_probabilities = Permute((2, 1))(x)
x = merge.multiply([attention_input, attention_probabilities],
name='attention_mul')
x = Lambda(lambda x: K.sum(x, axis=1))(x)
sentence_attention = Model(attention_input, x, name='sentence_attention')
embedding_layer = Embedding(
embeddings.shape[0],
embeddings.shape[1],
input_length=config['sentence_length'],
trainable=False,
weights=[embeddings],
)
input = Input(shape=(config['sentence_length'], ), dtype='int32')
x = embedding_layer(input)
x = SpatialDropout1D(config['embedding_dropout'])(x)
x = sentence_attention(x)
# x = Attention()(x)
if config['dense_layer']:
x = Dense(config['dense_layer'], activation='relu')(x)
x = Dropout(config['dropout_prob'])(x)
output = Dense(
1,
activation='sigmoid',
)(x)
model = Model(inputs=input, outputs=output)
return model, config
def create_truncated_model(trained_model):
model = Sequential()
model.add(Embedding(TOP_WORDS, 32, input_length=max_story_length))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
for i, layer in enumerate(model.layers):
layer.set_weights(trained_model.layers[i].get_weights())
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def get_hi_text_cnn_model():
drop = 0.60
# dropout_p = 0.5
learning_rate = 0.005 # 0.0001
inputs = Input(shape=(maxlen, ))
embed_layer = Embedding(len(word2index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)(inputs)
embed_layer = SpatialDropout1D(drop)(embed_layer)
# 第一条支路：
capsule1 = Capsule(
num_capsule=Num_capsule,
dim_capsule=Dim_capsule,
routings=Routings, # kernel_size=(3, 1),
share_weights=True)(embed_layer)
bn1 = BatchNormalization()(capsule1)
# 第二条支路：
capsule2 = Capsule(
num_capsule=Num_capsule,
dim_capsule=Dim_capsule,
routings=Routings, # kernel_size=(3, 1),
share_weights=True)(embed_layer)
bn2 = BatchNormalization()(capsule2)
capsule3 = Capsule(
num_capsule=Num_capsule,
dim_capsule=Dim_capsule,
routings=Routings, # kernel_size=(3, 1),
share_weights=True)(bn2)
bn3 = BatchNormalization()(capsule3)
# concat, fc+bn+relu
bn = Concatenate(axis=1)([bn1, bn3])
bn = Flatten()(bn)
fc = Dense(300)(bn)
bn = BatchNormalization()(fc)
bn = Activation('relu')(bn)
bn_dropout = Dropout(drop)(bn)
outputs = Dense(30, activation="sigmoid")(bn_dropout)
print(outputs.shape)
model = Model(inputs=inputs, outputs=outputs)
# model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def get_capsule_model():
maxlen = 100
learning_rate = 0.00001
input1 = Input(shape=(maxlen, ))
embed_layer = Embedding(len(word2index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=maxlen,
trainable=True)(input1)
embed_layer = SpatialDropout1D(rate_drop_dense)(embed_layer)
x = Bidirectional(
GRU(gru_len,
activation='relu',
dropout=dropout_p,
recurrent_dropout=dropout_p,
return_sequences=True))(embed_layer)
# capsule是一个卷积网络
capsule = Capsule(num_capsule=Num_capsule,
dim_capsule=Dim_capsule,
routings=Routings,
share_weights=True)(x)
# output_capsule = Lambda(lambda x: K.sqrt(K.sum(K.square(x), 2)))(capsule)
avg_pool = GlobalAveragePooling1D()(capsule)
max_pool = GlobalMaxPooling1D()(capsule)
capsule_concat = Concatenate()([avg_pool, max_pool])
capsule = Flatten()(capsule)
caspule = BatchNormalization()(capsule_concat)
capsule = Activation('relu')(capsule)
capsule = Dropout(dropout_p)(capsule)
output = Dense(30, activation='sigmoid')(capsule)
model = Model(inputs=input1, outputs=output)
# print(capsule.shape, output.shape)
adam = Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0)
sgd = SGD(lr=learning_rate, momentum=0.0, decay=0.0, nesterov=False)
rmsprop = RMSprop(lr=learning_rate, rho=0.9, epsilon=None, decay=0.0)
nadam = keras.optimizers.Nadam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=[f1_score])
return model
def create_model(params):
model = Sequential()
model.add(
Embedding(dimensions,
int(params["embedding_size"]),
mask_zero=True))
model.add(SpatialDropout1D(params["spatial_dropout"]))
model.add(
GRU(
int(params["units"]),
dropout=params["dropout"],
recurrent_dropout=params["recurrent_dropout"],
return_sequences=True,
activation="elu",
))
model.add(Dropout(params["dropout"]))
model.add(
GRU(
int(params["units"]),
dropout=params["dropout"],
recurrent_dropout=params["recurrent_dropout"],
return_sequences=False,
activation="elu",
))
model.add(Dropout(params["dropout"]))
model.add(Dense(2 * dimensions, activation="sigmoid"))
optimizer_rms = RMSprop(lr=params["learning_rate"])
batch_size = int(params["batch_size"])
model.compile(loss=utils.weighted_loss(class_weights),
optimizer=optimizer_rms)
print(model.summary())
model_fit = model.fit_generator(
utils.balanced_sample_generator(
train_data,
train_labels,
batch_size,
tool_tr_samples,
reverse_dictionary,
),
steps_per_epoch=len(train_data) // batch_size,
epochs=optimize_n_epochs,
callbacks=[early_stopping],
validation_data=(test_data, test_labels),
verbose=2,
shuffle=True,
)
return {
"loss": model_fit.history["val_loss"][-1],
"status": STATUS_OK,
"model": model,
}
def __call__(self, *args, **kwargs):
models = []
inputs = []
# 需要做embedding的输入
for i in self.col_max:
# print(i, self.col_max[i])
input_layer = Input([1],
name="{0}_embedding_input_layer".format(i))
emb_size = self.embedding_size.get(i, 50)
if emb_size is None:
print('embedding {0} size not defined'.format(i))
embed = Embedding(
self.col_max[i],
emb_size,
name="{0}_embedding_layer".format(i))(input_layer)
# flat = Flatten(name="{0}_flatten_layer".format(i))(embed)
inputs.append(input_layer)
models.append((embed))
# 无需embedding 的layer
print(f'sample cols is {self.sample_features}')
for i in self.sample_features:
input_layer = Input([1], name="{0}_num_layer".format(i))
inputs.append(input_layer)
dense_layer = Dense(1, name="{0}_num_dense".format(i))(input_layer)
models.append(dense_layer)
main_1 = concatenate(models)
s_dout = SpatialDropout1D(0.1)(main_1)
x = Flatten()(s_dout)
x = Dropout(0.2)(Dense(1000, activation='relu')(x))
x = Dropout(0.2)(Dense(1000, activation='relu')(x))
output = Dense(1, activation='sigmoid')(x)
exp_decay = lambda init, fin, steps: (init / fin)**(1 /
(steps - 1)) - 1
lr_init, lr_fin = 0.001, 0.0001
lr_decay = exp_decay(lr_init, lr_fin, self.decay_steps)
optimizer_adam = Adam(lr=0.001, decay=lr_decay)
udf_loss = wrapped_partial(bin_categorical_crossentropy,
e1=1.2,
e2=0.8)
model = Model(inputs=inputs, outputs=output)
parallel_model = multi_gpu_model(model, gpus=2)
parallel_model.compile(
# loss='binary_crossentropy',
loss=udf_loss,
optimizer=optimizer_adam,
metrics=[binary_accuracy, binary_FPA, binary_TPA, binary_auc])
return parallel_model
def conv_layer(inputs, n_kernels=32, kernel_size=3, dropout=0,spatial_dropout=0.1,
dilation_rate=1, padding='valid', strides=1):
out = Conv1D(n_kernels,
kernel_size=kernel_size,
strides=strides,
padding=padding,
kernel_regularizer=l2(1e-4),
dilation_rate=dilation_rate)(inputs)
out = BatchNormalization()(out)
out = Activation('relu')(out)
out = SpatialDropout1D(spatial_dropout)(out) if spatial_dropout > 1e-8 else out
if dropout > 1e-8:
out = Dropout(dropout)(out)
return out
def model_bilstm_pre(modelname_tosave):
start = timeit.default_timer()
model = Sequential()
# LSTM
model.add(Embedding(vocab_dim, embedding_dim, weights=[embedding_weights], input_length=maxlen))
model.add(SpatialDropout1D(0.2))
model.add(Bidirectional(LSTM(hidden_dims, dropout=0.2, recurrent_dropout=0.2, input_shape=(maxlen, embedding_dim))))
model.add(Dense(2))
model.add(Activation("sigmoid"))
model.compile(optimizer="adam", loss="categorical_crossentropy",
metrics=["accuracy"])
save_model='saved_models/'+modelname_tosave+'.hdf5'
callbacks = [
EarlyStopping(monitor='val_loss', patience=2, verbose=0),
ModelCheckpoint(filepath=save_model, monitor='val_loss', save_best_only=True, verbose=1),
]
history=model.fit(Xtrain, Ytrain, batch_size=batch_s,epochs=nb_epoch,
validation_data=(Xtest, Ytest),callbacks=callbacks, verbose=1)
accuracy_rslt=history.history["acc"]
val_accuracy_rslt=history.history["val_acc"]
loss_rslt=history.history["loss"]
val_loss_rslt=history.history["val_loss"]
plt.subplot(211)
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.plot(history.history["acc"], color="g", label="Train")
plt.plot(history.history["val_acc"], color="b", label="Validation")
plt.legend(loc="best")
plt.subplot(212)
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.plot(history.history["loss"], color="g", label="Train")
plt.plot(history.history["val_loss"], color="g", label="Validation")
plt.tight_layout()
plt.savefig('results/model_bilstm_pre_bs32.png')
plt.close()
score = model.evaluate(Xtest, Ytest, verbose=1)
print("Test score cnn: {:.3f}, accuracy: {:.3f}".format(score[0], score[1]))
stop = timeit.default_timer()
train_time=stop - start
with open("results/bilstm_results.txt", "a") as myfile:
myfile.write("(bilstm_model_pre(bs32,hl128,em100) \n accuracy={0}\n validation_accuracy={1}\n loss={2}\n validation_loss={3}\n test_score={4}, train_time={5}\n".format(
accuracy_rslt,val_accuracy_rslt,loss_rslt,val_loss_rslt,score[1],train_time))
def create_model_lstm(top_words,
embedding_vecor_length,
max_length,
number_of_classes,
embedding_matrix,
lstm_out=196):
model = Sequential()
model.add(
Embedding(top_words,
embedding_vecor_length,
weights=[embedding_matrix],
input_length=max_length,
trainable=False))
model.add(SpatialDropout1D(0.4))
model.add(LSTM(lstm_out, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(number_of_classes, activation='softmax'))
return model
def build_model(self):
data = self.data
inputs = Input(shape=(data.seq_length, ), dtype='int32')
embeddings = Embedding(
data.max_feature,
data.embed_dim,
weights=[data.embed_matrix],
trainable=False)(inputs)
x = SpatialDropout1D(self.dropout)(embeddings)
# kernel_size 1, 2, 3, 4, 5
cnn1 = Conv1D(256, 1, padding='same', activation='relu')(x)
cnn1 = Conv1D(256, 1, padding='same', activation='relu')(x)
cnn1_max = GlobalMaxPooling1D()(cnn1)
cnn1_avg = GlobalAveragePooling1D()(cnn1)
cnn2 = Conv1D(256, 2, padding='same', activation='relu')(x)
cnn2 = Conv1D(256, 2, padding='same', activation='relu')(x)
cnn2_max = GlobalMaxPooling1D()(cnn2)
cnn2_avg = GlobalAveragePooling1D()(cnn2)
cnn3 = Conv1D(2, 3, padding='same', activation='relu')(x)
cnn3 = Conv1D(2, 3, padding='same', activation='relu')(x)
cnn3_max = GlobalMaxPooling1D()(cnn3)
cnn3_avg = GlobalAveragePooling1D()(cnn3)
cnn4 = Conv1D(2, 4, padding='same', activation='relu')(x)
cnn4 = Conv1D(2, 4, padding='same', activation='relu')(x)
cnn4_max = GlobalMaxPooling1D()(cnn4)
cnn4_avg = GlobalAveragePooling1D()(cnn4)
cnn5 = Conv1D(2, 5, padding='same', activation='relu')(x)
cnn5 = Conv1D(2, 5, padding='same', activation='relu')(x)
cnn5_max = GlobalMaxPooling1D()(cnn5)
cnn5_avg = GlobalAveragePooling1D()(cnn5)
cnn = concatenate(
[
cnn1_max, cnn1_avg, cnn2_max, cnn2_avg, cnn3_max, cnn3_avg,
cnn4_max, cnn4_avg, cnn5_max, cnn5_avg
],
axis=-1)
x = Dense(self.dense_size, activation='relu')(cnn)
outputs = Dense(6, activation='sigmoid')(x)
model = Model(inputs=inputs, outputs=outputs)
optimizer = self.get_optimizer(self.lr, self.optim_name)
model.compile(
loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
self.model = model
def main():
train_X, train_y, word_index = load_and_prec()
train_y = to_categorical(train_y, NB_CLASSES) # 转换成独热编码的形式
train_X, test_X, train_y, test_y = train_test_split(train_X, train_y, test_size=0.2, random_state=0)
embedding_matrix = load_vocab(word_index) # 调用get_vocab()返回字典
HIDDEN_LAYER_SIZE = 64
BATCH_SIZE = 32
NUM_EPOCHS = 1
model = Sequential()
model.add(Embedding(MAX_FEATURES, EMBEDDING_SIZE, weights=[embedding_matrix], trainable=False,
input_length=MAX_SENTENCE_LENGTH))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(HIDDEN_LAYER_SIZE, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(NB_CLASSES))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy", optimizer="adam",
metrics=["accuracy"])
history = model.fit(train_X, train_y, batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_split=0.2)
# plot loss and accuracy
plt.subplot(211)
plt.title("Accuracy")
plt.plot(history.history["acc"], color="g", label="Train")
plt.plot(history.history["val_acc"], color="b", label="Validation")
plt.legend(loc="best")
plt.subplot(212)
plt.title("Loss")
plt.plot(history.history["loss"], color="g", label="Train")
plt.plot(history.history["val_loss"], color="b", label="Validation")
plt.legend(loc="best")
plt.tight_layout()
plt.show()
# 评分
score, acc = model.evaluate(test_X, test_y, batch_size=BATCH_SIZE)
print("Test score: %.3f, accuracy: %.3f" % (score, acc))
def get_text_cnn_model():
# 加残差或者是attention
# 3个串联 0.10
drop = 0.60 # 0.55
# dropout_p = 0.5
learning_rate = 0.005 # 0.0001
Dim_capsule = 30 # 128
gru_units = Dim_capsule >> 1
inputs = Input(shape=(maxlen, ))
embed_layer = Embedding(len(word2index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)(inputs)
embed_layer = SpatialDropout1D(drop)(embed_layer)
capsule1 = Capsule(
num_capsule=Num_capsule,
dim_capsule=Dim_capsule,
routings=Routings, # kernel_size=(3, 1),
share_weights=True)(embed_layer)
bn1 = BatchNormalization()(capsule1)
x1 = Bidirectional(
GRU(gru_units,
activation='relu',
dropout=dropout_p,
recurrent_dropout=dropout_p,
return_sequences=True))(embed_layer)
print(bn1.shape, x1.shape)
concat = Concatenate(axis=1)([bn1, x1])
bn = Flatten()(concat)
fc = Dense(300)(bn)
bn = BatchNormalization()(fc)
bn = Activation('relu')(bn)
bn_dropout = Dropout(drop)(bn)
outputs = Dense(30, activation="sigmoid")(bn_dropout)
model = Model(inputs=inputs, outputs=outputs)
# model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def build_model(self):
data = self.data
maxlen = data.seq_length
inputs = Input(shape=(data.seq_length, ), dtype="int32")
embeddings = Embedding(
data.max_feature,
data.embed_dim,
weights=[data.embed_matrix],
trainable=False)(inputs)
x = SpatialDropout1D(self.dropout)(embeddings)
def k_slice(x, start, end):
return x[:, start:end]
left_1 = Lambda(
k_slice, arguments={'start': maxlen - 1,
'end': maxlen})(x)
left_2 = Lambda(k_slice, arguments={'start': 0, 'end': maxlen - 1})(x)
l_embedding = concatenate([left_1, left_2], axis=1)
right_1 = Lambda(k_slice, arguments={'start': 1, 'end': maxlen})(x)
right_2 = Lambda(k_slice, arguments={'start': 0, 'end': 1})(x)
r_embedding = concatenate([right_1, right_2], axis=1)
forward = CuDNNGRU(data.embed_dim, return_sequences=True)(l_embedding)
backward = CuDNNGRU(
data.embed_dim, return_sequences=True,
go_backwards=True)(r_embedding)
together = concatenate([forward, x, backward], axis=2)
semantic = TimeDistributed(
Dense(self.dense_size, activation="tanh"))(together)
max_pool = GlobalMaxPooling1D()(semantic)
ave_pool = GlobalAveragePooling1D()(semantic)
conc = concatenate([ave_pool, max_pool])
outputs = Dense(6, activation="sigmoid")(conc)
model = Model(inputs=inputs, outputs=outputs)
optimizer = self.get_optimizer(self.lr, self.optim_name)
model.compile(
loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
self.model = model
def fast_text_model():
# 3个串联 0.10
drop = 0.7 # 0.55
# dropout_p = 0.5
learning_rate = 0.001 # 0.0001
gru_units = 128 # 100
maxlen = 100
inputs = Input(shape=(maxlen, ))
embed_layer = Embedding(len(word2index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=maxlen,
trainable=True)(inputs)
embed_layer = SpatialDropout1D(drop)(embed_layer)
"""x = LSTM(output_dim=100,activation='relu',inner_activation='relu', return_sequences=True)(x)"""
pool = GlobalAveragePooling1D()(embed_layer)
print(pool.shape)
fc = Dense(400, activation='relu')(pool)
bn = BatchNormalization()(fc)
bn_relu = Activation('relu')(bn)
bn_dropout = Dropout(drop)(bn_relu)
outputs = Dense(30, activation='sigmoid')(bn_dropout)
# print(outputs.shape)
model = Model(inputs=inputs, outputs=outputs)
adam = Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=1e-6)
sgd = SGD(lr=learning_rate, momentum=0.0, decay=0.0, nesterov=False)
rmsprop = RMSprop(lr=learning_rate, rho=0.9, epsilon=None, decay=0.0)
nadam = Nadam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[f1_score])
return model
def set_recurrent_network(mdl_dict, reverse_dictionary, class_weights):
"""
Create a RNN network and set its parameters
"""
dimensions = len(reverse_dictionary) + 1
model_params = get_best_parameters(mdl_dict)
# define the architecture of the neural network
model = Sequential()
model.add(Embedding(dimensions, model_params["embedding_size"], mask_zero=True))
model.add(SpatialDropout1D(model_params["spatial_dropout"]))
model.add(GRU(model_params["units"], dropout=model_params["spatial_dropout"], recurrent_dropout=model_params["recurrent_dropout"], activation=model_params["activation_recurrent"], return_sequences=True))
model.add(Dropout(model_params["dropout"]))
model.add(GRU(model_params["units"], dropout=model_params["spatial_dropout"], recurrent_dropout=model_params["recurrent_dropout"], activation=model_params["activation_recurrent"], return_sequences=False))
model.add(Dropout(model_params["dropout"]))
model.add(Dense(dimensions, activation=model_params["activation_output"]))
optimizer = RMSprop(lr=model_params["lr"])
model.compile(loss=weighted_loss(class_weights), optimizer=optimizer)
return model, model_params
def build_model(self, loss, P=None):
input = Input(shape=(self.maxlen, ))
x = Embedding(self.max_features, self.embedding_dims)(input)
x = SpatialDropout1D(0.8)(x)
x = Activation('relu')(x)
x = Flatten()(x)
output = Dense(self.classes, kernel_initializer='he_normal')(x)
if loss in yes_bound:
output = BatchNormalization(axis=1)(output)
if loss in yes_softmax:
output = Activation('softmax')(output)
model = Model(inputs=input, outputs=output)
self.compile(model, loss, P)
def baseline(x, output_channels):
for nb_filters in [10, 25, 35, 50]:
x = Conv1D(nb_filters, 50, padding="same")(x)
x = Activation('relu')(x)
x = BatchNormalization()(x)
x = MaxPooling1D(4)(x)
x = SpatialDropout1D(0.5)(x)
x = GlobalAveragePooling1D()(x)
x = Dense(10)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.7)(x)
predictions = Dense(output_channels)(x)
return predictions
def train_lstm(n_symbols, embedding_weights, x_train, y_train, x_test, y_test):
'''
定义网络结构
'''
print('Defining a Simple Keras Model...')
model = Sequential() # or Graph or whatever
model.add(
Embedding(n_symbols,
word_dim,
mask_zero=False,
weights=[embedding_weights],
input_length=max_len,
trainable=False)) # Adding Input Length
model.add(SpatialDropout1D(0.4))
model.add(Bidirectional(CuDNNLSTM(hidden_size, return_sequences=True)))
model.add(Bidirectional(CuDNNLSTM(hidden_size, return_sequences=True)))
model.add(GlobalMaxPooling1D())
model.add(Dense(128))
model.add(Dense(69, activation='softmax'))
model.summary()
print('Compiling the Model...')
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print("Train...")
model_chechpoint = ModelCheckpoint('out/czc_model_40.h5',
save_best_only=True,
save_weights_only=False)
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epoch,
verbose=1,
validation_data=(x_test, y_test),
callbacks=[model_chechpoint])
print("Evaluate...")
score = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
def loadModel():
print(MAX_LEN)
print(VOCAB_SIZE)
model = Sequential()
model.add(Embedding(VOCAB_SIZE, EMBED_SIZE, input_length=MAX_LEN))
model.add(SpatialDropout1D(0.2))
model.add(
Conv1D(filters=NUM_FILTERS,
kernel_size=NUM_WORDS,
padding="valid",
activation="relu"))
model.add(GlobalAveragePooling1D())
model.add(Dense(1, activation="sigmoid"))
# model.add(Dense(2, activation="softmax"))
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
# model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
# model.summary()
model.load_weights(MODEL_FILE)
return model