def models(self, input_length, vocab_size, embedding_dim, conv_layer_info, connected_layer_info, batch_size,epochs):
model_input = Input(shape = (input_length,), dtype = 'int64')
model = Embedding(vocab_size, embedding_dim, input_length = input_length)(model_input)
for i in range( len(conv_layer_info) ):
model = Convolution1D(filters= conv_layer_info[i][0], kernel_size=conv_layer_info[i][1],
padding="valid",
activation="relu",
strides=1)(model)
if len(conv_layer_info[i]) > 2:
model = MaxPooling1D(pool_size = conv_layer_info[i][2])(model)
model = Flatten()(model)
for i in range(len(connected_layer_info)):
model = Dense(connected_layer_info[i][0],activation = "relu")(model)
model = Dropout(connected_layer_info[i][1])(model)
model_output = Dense(self.num_of_classes, activation = "softmax")(model)
model =Model(inputs=model_input,outputs=model_output)
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
print("Started Training : ")
model.fit(self.x_train, self.y_train, batch_size = batch_size, epochs= epochs, validation_data=(self.x_test, self.y_test),
verbose=2)
print("Training Completed")
self.model = model
def cnn_non_static(self):
x_train, y_train, x_test, y_test, vocabolary_dict = data_load_and_preproccess(
self.word_size, self.sequence_length)
embedding_weights = word_to_vector(np.vstack(
(x_train, x_test)), vocabolary_dict, self.embedding_dim,
self.min_word_count,
self.context_window_size)
embedding_weights = np.array(
[value for value in embedding_weights.values()])
model_input = Input(shape=(self.sequence_length, ))
model = Embedding(len(vocabolary_dict),
self.embedding_dim,
weights=[embedding_weights],
input_length=self.sequence_length,
name="embedding")(model_input)
model = Dropout(self.drop_prob[0])(model)
multi_cnn_channel = []
for kernal in self.kernal_size:
conv_channel = Convolution1D(filters=self.filters,
kernel_size=kernal,
padding="valid",
activation="relu",
strides=1)(model)
conv_channel = MaxPooling1D(pool_size=2)(conv_channel)
conv_channel = Flatten()(conv_channel)
multi_cnn_channel.append(conv_channel)
model = Concatenate()(multi_cnn_channel)
model = Dropout(self.drop_prob[1])(model)
for dimension in self.hidden_dims:
model = Dense(dimension, activation="relu")(model)
model_output = Dense(1, activation="sigmoid")(model)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
print("Started Training : ")
model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.epochs,
validation_data=(x_test, y_test),
verbose=2)
print("Training Completed")
self.model = model
def train():
input = Input(shape=(max_len, ))
model = Embedding(input_dim=n_words + 1,
output_dim=100,
input_length=max_len,
mask_zero=True)(input) # 20-dim embedding
model = Bidirectional(
LSTM(units=50, return_sequences=True,
recurrent_dropout=0.1))(model) # variational biLSTM
model = TimeDistributed(Dense(50, activation="relu"))(
model) # a dense layer as suggested by neuralNer
crf = CRF(n_tags) # CRF layer
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.summary()
history = model.fit(x_train,
np.array(y_train),
batch_size=64,
epochs=5,
validation_split=0.1,
verbose=1)
save_load_utils.save_all_weights(model, filepath="models/bilstm-crf.h5")
hist = pd.DataFrame(history.history)
print(hist)
plt.figure(figsize=(12, 12))
plt.plot(hist["crf_viterbi_accuracy"])
plt.plot(hist["val_crf_viterbi_accuracy"])
plt.show()
def trainLSTM():
# fit a LSTM network with an embedding layer
from keras.models import Model, Input
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional
input = Input(shape=(max_len, ))
model = Embedding(input_dim=n_words, output_dim=50,
input_length=max_len)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(n_tags, activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss="categorical_crossentropy",
metrics=["accuracy"])
history = model.fit(X_tr,
np.array(y_tr),
batch_size=32,
epochs=5,
validation_split=0.1,
verbose=1)
hist = pd.DataFrame(history.history)
plt.figure(figsize=(12, 12))
plt.plot(hist["acc"])
plt.plot(hist["val_acc"])
plt.show()
return model
def train(self):
input = Input(shape=(self.max_len, ))
model = Embedding(input_dim=self.num_word + 1,
output_dim=20,
input_length=self.max_len,
mask_zero=True)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=50, return_sequences=True,
recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(50, activation="relu"))(
model) # softmax output layer
crf = CRF(self.num_tag)
out = crf(model)
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
self.pad_process()
train_X, test_X, train_y, test_y = self.train_test_split()
history = model.fit(train_X,
np.array(train_y),
batch_size=32,
epochs=1,
validation_split=0.1,
verbose=1)
self.make_plot(history)
return model
def run(X_train,
Y_train,
X_val,
Y_val,
embedding_matrix,
vocab_size,
maxlen=40,
emb_dim=300,
neg_ratio=0,
hidden_dim=300,
drop=0.2,
r_drop=0.1):
##build model
input = Input(shape=(maxlen, ))
model = Embedding(vocab_size,
emb_dim,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)(input)
model = Dropout(drop)(model)
model = Bidirectional(
LSTM(hidden_dim, return_sequences=True,
recurrent_dropout=r_drop))(model)
model = Dropout(drop)(model)
out = TimeDistributed(Dense(1, activation='sigmoid'))(model)
model = Model(input, out)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
earlyStop = [EarlyStopping(monitor='val_loss', patience=1)]
history = model.fit(X_train,
Y_train,
batch_size=64,
epochs=10,
validation_data=(X_val, Y_val),
callbacks=earlyStop)
pred = model.predict(X_val)
Y_pred = np.squeeze(pred)
test = [[1 if y >= threshold else 0 for y in x] for x in Y_pred]
test_arr = np.asarray(test)
test_arr = np.reshape(test_arr, (-1))
target = np.reshape(Y_val, (-1))
print(
metrics.precision_recall_fscore_support(target,
test_arr,
average=None,
labels=[0, 1]))
# Y_pred_ = [[1 if y>=threshold else 0 for y in x] for x in Y_pred]
Y_val_ = np.squeeze(Y_val)
print("Evaluate: dev seg exact")
pred_out_dir = out_dir + 'seg_' + str(neg_ratio) + 'neg'
gold_dir = '../../data/val_segs/' + 'seg_' + str(neg_ratio) + 'neg'
p, r, f = seg_exact_match(test, Y_val_, pred_out_dir, gold_dir)
return model, history, p, r, f
def train(X_tr, y_tr):
output_dim = 50
word_input = Input(shape=(MAXLEN, ))
model = Embedding(input_dim=VOCAB_SIZE,
output_dim=output_dim,
input_length=MAXLEN)(word_input)
model = SpatialDropout1D(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(len(tags), activation="softmax"))(model)
model = Model(word_input, out)
model.compile(optimizer="rmsprop",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
checkpointer = ModelCheckpoint(filepath='NER_model.h5',
verbose=0,
mode='auto',
save_best_only=True,
monitor='val_loss')
history = model.fit(X_tr,
y_tr.reshape(*y_tr.shape, 1),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
shuffle=True,
validation_split=VALIDATION_SPLIT,
verbose=1,
callbacks=[checkpointer])
return history
def cross_validate(self, X, y):
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.1)
input = Input(shape=(self.max_len, ))
model = Embedding(input_dim=self.n_words,
output_dim=50,
input_length=self.max_len)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True,
recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(self.n_labels, activation="softmax"))(
model) # softmax output layer
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss="categorical_crossentropy",
metrics=["accuracy"])
history = model.fit(X_tr,
np.array(y_tr),
batch_size=32,
epochs=1,
validation_split=0.1,
verbose=1)
p = model.predict(np.array([X_te[10]]))
p = np.argmax(p, axis=-1)
for w, pred in zip(X_te[10], p[0]):
if self.words[w] != 'PADGARBAGE':
print("{:15}: {}".format(self.words[w], self.labels[pred]))
def train():
input = Input(shape=(max_len, ))
model = Embedding(input_dim=n_words, output_dim=50,
input_length=max_len)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(n_tags, activation='softmax'))(
model) # softmax output layer
model = Model(input, out)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
# checkpoint
# filepath = "../result/bilstm-weights-{epoch:02d}-{val_acc:.2f}.hdf5"
# checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
# history=model.fit(X_train,np.array(y_train),batch_size=32,epochs=5,validation_split=0.1,verbose=1,callbacks=[checkpoint])
history = model.fit(X_train,
np.array(y_train),
batch_size=32,
epochs=5,
validation_split=0.1,
verbose=1)
# 保存模型
model.save(filepath="../result/bi-lstm.h5")
hist = pd.DataFrame(history.history)
plt.figure(figsize=(12, 12))
plt.plot(hist["acc"])
plt.plot(hist["val_acc"])
plt.show()
def lstm_crf(x, y, vocab_size, n_tags, batch_size, epochs):
output_dim = 30
hid_size = 50
dense1 = 50
seq_len = x.shape[1]
input_ = Input(shape=(seq_len, ))
model = Embedding(input_dim=vocab_size,
output_dim=output_dim,
input_length=seq_len,
mask_zero=True)(input_)
model = Bidirectional(
LSTM(units=hid_size, return_sequences=True,
recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(dense1, activation='relu'))(model)
crf = CRF(n_tags, learn_mode='marginal')
out = crf(model) # prob
model = Model(inputs=input_, outputs=out)
model.compile(optimizer='rmsprop', loss=crf_loss, metrics=[crf.accuracy])
model.summary()
history = model.fit(x,
np.array(y),
batch_size=batch_size,
epochs=epochs,
validation_split=0.1,
verbose=1)
return model, history
def run(X_train, Y_train, X_val, Y_val, embedding_matrix, vocab_size, maxlen=40, emb_dim=300, neg_ratio=0, hidden_dim=300, drop=0.2, r_drop=0.1):
##build model
# input = Input(shape=(maxlen,))
# model = Embedding(vocab_size, emb_dim, weights=[embedding_matrix], input_length=maxlen, trainable=False)(input)
# model = Dropout(drop)(model)
# model = Bidirectional(LSTM(hidden_dim, return_sequences=True, recurrent_dropout=r_drop))(model)
# model = Dropout(drop)(model)
# out = TimeDistributed(Dense(1, activation='sigmoid'))(model)
input = Input(shape=(maxlen,))
model = Embedding(vocab_size, emb_dim, weights=[embedding_matrix], input_length=maxlen, trainable=False)(input)
model = Bidirectional(LSTM(hidden_dim, return_sequences=True, recurrent_dropout=r_drop))(model)
model = TimeDistributed(Dense(hidden_dim//4, activation='relu'))(model)
model = TimeDistributed(Dropout(drop))(model)
##use CRF instead of Dense
crf = CRF(2)
out = crf(model)
model = Model(input, out)
Y_train_2 = keras.utils.to_categorical(Y_train)
Y_val_2 = keras.utils.to_categorical(Y_val)
model.compile(optimizer='adam', loss=crf.loss_function, metrics=[crf.accuracy])
earlyStop = [EarlyStopping(monitor='val_loss', patience=1)]
history = model.fit(X_train, Y_train_2, batch_size=64, epochs=10,
validation_data=(X_val, Y_val_2), callbacks=earlyStop)
preds = model.predict(X_val)
test = [[np.argmax(y) for y in x] for x in preds]
test_arr = np.asarray(test)
test_arr = np.reshape(test_arr, (-1))
print (metrics.precision_recall_fscore_support(np.reshape(Y_val,(-1)), test_arr, average=None,
labels=[0, 1]))
# Y_pred_ = [[1 if y>=threshold else 0 for y in x] for x in Y_pred]
Y_val_ = np.squeeze(Y_val)
print ("Evaluate: dev seg exact")
pred_out_dir = out_dir+'seg_'+str(neg_ratio)+'neg'
gold_dir = '../../data/val_segs/'+'seg_'+str(neg_ratio)+'neg'
p, r, f = seg_exact_match(test, Y_val_, pred_out_dir, gold_dir)
return model, history, p, r, f
def train(self):
input = Input(shape=(120,))
model = Embedding(input_dim=self.num_words, output_dim=50, input_length=120)(input)
model = Dropout(0.1)(model)
model = Bidirectional(LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(self.num_entities, activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"])
history = model.fit(x=self.X_train, y=np.array(self.Y_train), batch_size=64, epochs=10,
validation_data=(self.X_validation, self.Y_validation))
model.save("../models/ner_" + str(datetime.utcnow().microsecond))
test_eval = model.evaluate(self.X_test, self.Y_test, verbose=0)
print('Test loss:', test_eval[0])
print('Test accuracy:', test_eval[1])
return model, history
def run():
sentences = read_train_file(TRAIN_PATH)
word_map, tag_map = create_word_idx()
max_len = max([len(s) for s in sentences])
X = [[word_map[w[0]] for w in s] for s in sentences]
n_words = len(word_map)
n_tags = 9
X = pad_sequences(maxlen=max_len, sequences=X, padding="post", value=n_words - 1)
y = [[tag_map[w[2]] for w in s] for s in sentences]
y_testing = pad_sequences(maxlen=max_len, sequences=y, padding="post", value=tag_map["O"])
y = [to_categorical(i, num_classes=n_tags) for i in y_testing]
dev_sentences = read_train_file(DEV_PATH)
dev_max_len = max([len(s) for s in dev_sentences])
X_dev = [[word_map[w[0]] for w in s] for s in dev_sentences]
X_dev = pad_sequences(maxlen=max_len, sequences=X_dev, padding="post", value=n_words - 1)
y_dev = [[tag_map[w[2]] for w in s] for s in dev_sentences]
y_dev = pad_sequences(maxlen=max_len, sequences=y_dev, padding="post", value=tag_map["O"])
y_dev = [to_categorical(i, num_classes=9) for i in y_dev]
test_sentences = read_train_file(TEST_PATH)
test_max_len = 33
X_test = [[word_map[w[0]] for w in s] for s in test_sentences]
X_test = pad_sequences(maxlen=test_max_len, sequences=X_test, padding="post", value=n_words - 1)
y_test = [[tag_map[w[2]] for w in s] for s in test_sentences]
y_test = pad_sequences(maxlen=test_max_len, sequences=y_test, padding="post", value=tag_map["O"])
input = Input(shape=(max_len,))
model = Embedding(input_dim=n_words + 1, output_dim=20, input_length=max_len)(input)
model = Dropout(0.1)(model)
model = Bidirectional(LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(n_tags, activation="relu"))(model) # softmax output layer
crf = CRF(n_tags) # CRF layer
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy])
history = model.fit(X, np.array(y), batch_size=32, epochs=5, verbose=1)
model.save('simple_model.h5')
return model, tag_map, X_test, y_test
def train():
input = Input(shape=(input_max_len, ))
model = Embedding(vocab_size,
embedding_size,
weights=[glove_embedding_matrix()],
input_length=input_max_len,
trainable=False)(input)
model = Bidirectional(
LSTM(embedding_size,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))(model)
model = Bidirectional(
LSTM(2 * embedding_size,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))(model)
model = TimeDistributed(Dense(embedding_size, activation='sigmoid'))(model)
model = Flatten()(model)
model = Dense(input_max_len, activation='sigmoid')(model)
out = model
model = Model(input, out)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
print(model.summary())
history = model.fit(padded_features,
np.array(final_label_updated),
validation_split=validation_split,
epochs=epochs,
batch_size=batch_size,
verbose=logging,
shuffle=True)
model.save(model_name)
metrics(history, model)
input_length=max_len)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(n_tags, activation="softmax"))(
model) # softmax output layer
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss="categorical_crossentropy",
metrics=["accuracy"])
history = model.fit(X_train,
np.array(y_train),
batch_size=32,
epochs=3,
validation_split=0.1,
verbose=1)
hist = pd.DataFrame(history.history)
plt.figure(figsize=(12, 12))
plt.plot(hist["accuracy"])
plt.plot(hist["val_accuracy"])
plt.show()
i = random.randint(0, len(X_test))
p = model.predict(np.array([X_test[i]]))
p = np.argmax(p, axis=-1)
print("{:15} ({:5}): {}".format("Word", "True", "Pred"))
for w, pred in zip(X_test[i], p[0]):
print("{:15}: {}".format(words[w], tags[pred]))
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.summary()
checkpointer = ModelCheckpoint(filepath='model.h5',
verbose=0,
mode='auto',
save_best_only=True,
monitor='val_loss')
history = model.fit(X_train,
np.array(y_train),
batch_size=batch_size,
epochs=epochs,
validation_split=0.1,
callbacks=[checkpointer])
history.history.keys()
acc = history.history['crf_viterbi_accuracy']
val_acc = history.history['val_crf_viterbi_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
plt.figure(figsize=(8, 8))
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()
output_dim=32,
kernel_initializer=initializers.glorot_uniform(seed=1),
activation='relu')(model)
model = Dropout(0.5)(model)
model = Dense(name="output",
output_dim=2,
kernel_initializer=initializers.glorot_uniform(seed=1),
activation='relu')(model)
model = Model(inputs=inputs, outputs=model)
model.compile(loss='mae', optimizer='adam', metrics=['mse'])
model.summary()
epochs = 10
callback = model.fit(x=train_x,
y=train_y,
epochs=epochs,
validation_split=.3,
batch_size=200,
verbose=1).history
test_y = np.rint(model.predict(x=test_x, batch_size=200,
verbose=1)).astype('int')
seconds = str((datetime.datetime.now() - now).seconds)
print(seconds)
with open('test{seconds}.txt'.format(seconds=seconds), 'w') as file:
file.write('id,good,bad\n')
for index, data in enumerate(test_y):
file.write('{},{},{}\n'.format(index, data[0], data[1]))
model = MaxPooling1D(pool_size=2)(model)
model = Conv1D(filters=32, kernel_size=2, padding='same',
activation='relu')(model)
model = MaxPooling1D(pool_size=2)(model)
model = GRU(128)(model)
model = Dense(64, activation="relu")(model)
model = Dense(32, activation="relu")(model)
model = Dense(16, activation="relu")(model)
model = Dense(6, activation="sigmoid")(model)
model = Model(inputs=inp, outputs=model)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
filepath = "weight-improvement-{epoch:02d}-{loss:4f}.hd5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
period=1)
early = EarlyStopping(monitor="val_loss", mode="min", patience=2)
callbacks_list = [checkpoint, early]
model.fit(data,
targets,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
validation_split=VALIDATION_SPLIT,
callbacks=callbacks_list)
def train(word2Vec, train_df, test_df, max_length, filters, kernel_size,
pool_size, dense):
r = random.randint(1, 10000)
now = datetime.datetime.now()
tokenizer = Tokenizer()
tokenizer.fit_on_texts(word2Vec.wv.vocab.keys())
train_x = tokenizer.texts_to_sequences(train_df['text'])
train_x = pad_sequences(train_x, maxlen=max_length)
# train_y_good = train_df["good"]
# train_y_bad = train_df["bad"]
train_y = pd.DataFrame(train_df, columns=["good", "bad"])
test_x = tokenizer.texts_to_sequences(test_df['text'])
test_x = pad_sequences(test_x, maxlen=max_length)
word_index = tokenizer.word_index
embedding_matrix = np.zeros((len(word_index) + 1, word2Vec.vector_size))
for word, i in word_index.items():
try:
embedding_matrix[i] = word2Vec[word]
except:
continue
inputs = Input(shape=(max_length, ))
model = Embedding(name="embedding",
input_dim=len(word_index) + 1,
output_dim=word2Vec.vector_size,
weights=[embedding_matrix],
input_length=max_length)(inputs)
model = Conv1D(name="conv1D",
filters=filters,
kernel_size=kernel_size,
kernel_initializer=initializers.glorot_uniform(seed=1),
padding='same')(model)
model = MaxPooling1D(name="maxPooling1D",
pool_size=pool_size,
strides=1,
padding='same')(model)
model = Flatten(name="flatten")(model)
model = Dense(name="dense",
output_dim=dense,
kernel_initializer=initializers.glorot_uniform(seed=1),
activation='relu')(model)
model = Dense(name="output",
output_dim=2,
kernel_initializer=initializers.glorot_uniform(seed=1),
activation='relu')(model)
model = Model(inputs=inputs, outputs=model)
model.compile(loss='mae',
optimizer=optimizers.Adam(lr=.001),
metrics=['mse'])
model.summary()
epochs = 15
callback = model.fit(x=train_x,
y=train_y,
epochs=epochs,
validation_split=.3,
batch_size=20,
verbose=1).history
test_y = np.rint(model.predict(x=test_x, batch_size=10,
verbose=1)).astype('int')
seconds = str((datetime.datetime.now() - now).seconds)
with open('test{seconds}_{r}.txt'.format(seconds=seconds, r=r),
'w') as file:
file.write('id,good,bad\n')
for index, data in enumerate(test_y):
file.write('{},{},{}\n'.format(index, data[0], data[1]))
with open('record{seconds}_{r}.log'.format(seconds=seconds, r=r),
'w') as file:
file.write('result\t\n\n')
file.write('\t'.join(
['index', 'loss\t\t', 'mse\t\t\t', 'val_loss\t\t', 'val_mse\t']) +
'\n')
for index, loss, mse, val_loss, val_mse in zip(
range(1, epochs + 1), callback['loss'],
callback['mean_squared_error'], callback['val_loss'],
callback['val_mean_squared_error']):
file.write('\t'.join([
str(index) + '\t', '{:.12f}'.format(loss), '{:.12f}'.format(
mse), '{:.12f}'.format(val_loss), '{:.12f}'.format(val_mse)
]) + '\n')
file.write(
'\nmax_length={max_length}\nmin_count={min_count}, size=270, iter=10, sg=1, workers=10\n'
.format(max_length=max_length, min_count=min_count))
file.write('inputs = Input(shape=(max_length,)\n')
file.write(
'model = Embedding(name="embedding", input_dim=len(word_index)+1, output_dim=word2Vec.vector_size, weights=[embedding_matrix], input_length=max_length)(inputs)\n'
)
file.write(
'model = Conv1D(name="conv1D_good", filters={filters}, kernel_size={kernel_size}, kernel_initializer=initializers.glorot_uniform(seed=1), padding="same")(model)\n'
.format(filters=filters, kernel_size=kernel_size))
file.write(
'model = MaxPooling1D(name="maxPooling1D", pool_size={pool_size}, strides=1, padding="same")(model)\n'
.format(pool_size=pool_size))
file.write(
'model = Dense(name="dense", output_dim={dense}, kernel_initializer=initializers.glorot_uniform(seed=1), activation="relu")(model)\n'
.format(dense=dense))
file.write(
'model = Dense(name="output", output_dim=2, kernel_initializer=initializers.glorot_uniform(seed=1), activation="relu")(model)\n'
)
file.write('model = Model(inputs=inputs, outputs=model)\n')
file.write(
'model.compile(loss="mae", optimizer=optimizers.Adam(lr=.001), metrics=["mse"])\n'
)
import matplotlib.pyplot as plt
fig = plt.figure()
plt.grid(True)
plt.ylim(0, 40)
plt.plot(callback['loss'])
plt.plot(callback['mse'])
plt.plot(callback['val_loss'])
plt.plot(callback['val_mse'])
plt.title('model loss')
plt.ylabel('loss (mae)')
plt.xlabel('epoch')
plt.legend(['train_loss', 'train_mse', 'test_loss', 'test_mse'],
loc='upper right')
fig.savefig('{seconds}_{r}.png'.format(seconds=seconds, r=r), dpi=fig.dpi)
def create_model(x_train, y_train, x_test, y_test):
args = parse_args()
set_logger(args.log_path, args.log_level)
logging.debug('Args:')
logging.debug(args)
lang = construct_languages(args.train)
assert len(lang) == 1
lang = lang[0]
game = initialize_game(train_file=lang.train,
test_file=lang.test,
dev_file=lang.dev,
emb_file=lang.emb,
budget=args.budget,
max_seq_len=args.max_seq_len,
max_vocab_size=args.max_vocab_size,
emb_size=args.embedding_size,
model_name=args.model_name)
max_len = args.max_seq_len
input_dim = args.max_vocab_size
output_dim = args.embedding_size
embedding_matrix = game.w2v
logging.debug('building Keras model...')
input = Input(shape=(max_len, ))
model = Embedding(input_dim=input_dim,
output_dim=output_dim,
input_length=max_len,
weights=[embedding_matrix],
trainable=False)(input)
model = Dropout(0.1)(model)
n_units = 128
model = Bidirectional(
LSTM(units=n_units, return_sequences=True,
recurrent_dropout=0.1))(model)
n_tags = 5
out = TimeDistributed(Dense(n_tags, activation='softmax'))(model)
model = Model(input, out)
logging.debug('Model type: ')
logging.debug(type(model))
logging.debug('Model summary: ')
logging.debug(model.summary())
rmsprop = keras.optimizers.RMSprop(lr={{choice([0.0001])}})
model.compile(optimizer=rmsprop,
loss='categorical_crossentropy',
metrics=['accuracy'])
logging.debug('done building model...')
logging.debug('starting training...')
num_train_examples = len(x_train)
for i in range(num_train_examples):
print('i: ', i)
model.fit(x_train[:i],
y_train[:i],
batch_size=200,
epochs=20,
verbose=0)
logging.debug('done training...')
logging.debug('starting testing...')
num_samples = x_test.shape[0]
logging.debug('Number of samples: {}'.format(num_samples))
max_batch_size = 4096
batch_size = min(num_samples, max_batch_size)
predictions_probability = model.predict(x_test, batch_size=batch_size)
predictions = numpy.argmax(predictions_probability, axis=-1)
fscore = compute_fscore(Y_pred=predictions, Y_true=y_test)
logging.debug('done testing...')
return -fscore
# get_ipython().system_raw('./ngrok http 6016 &')
# ! curl -s http://localhost:4040/api/tunnels | python3 -c \
# "import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])"
# from keras.callbacks import TensorBoard
# tbCallBack = TensorBoard(log_dir='./log', histogram_freq=1,
# write_graph=True,
# write_grads=True,
# batch_size=32,
# write_images=True)
history = model.fit(X_train,
np.array(y_train),
batch_size=32,
epochs=5,
validation_split=0.1,
verbose=1) #, callbacks = [tbCallBack])
hist = pd.DataFrame(history.history)
hist
plt.style.use('ggplot')
plt.figure(figsize=(12, 12))
plt.plot(hist['crf_viterbi_accuracy'])
plt.plot(hist['val_crf_viterbi_accuracy'])
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
# pip install seqeval
from seqeval.metrics import precision_score, recall_score, f1_score, classification_report
def bilstm_crf(train_loc, test_loc):
train_pre = preprocess(train_loc)
test_pre = preprocess(test_loc)
cc_train = cuu(train_pre)
cc_test = cuu(test_pre)
words_all, tags_all = combine_all(cc_train, cc_test)
n_words = len(words_all)
n_tags = len(tags_all)
max_len = 130
word2idx = {w: i for i, w in enumerate(words_all)}
tag2idx = {t: i for i, t in enumerate(tags_all)}
X = [[word2idx[w[0]] for w in s] for s in cc_train]
X = pad_sequences(maxlen=max_len,
sequences=X,
padding="post",
value=n_words - 1)
X1 = [[word2idx[w[0]] for w in s] for s in cc_test]
X1 = pad_sequences(maxlen=max_len,
sequences=X1,
padding="post",
value=n_words - 1)
y = [[tag2idx[w[1]] for w in s] for s in cc_train]
y = pad_sequences(maxlen=max_len,
sequences=y,
padding="post",
value=tag2idx["O"])
y1 = [[tag2idx[w[1]] for w in s] for s in cc_test]
y1 = pad_sequences(maxlen=max_len,
sequences=y1,
padding="post",
value=tag2idx["O"])
y = [to_categorical(i, num_classes=n_tags) for i in y]
input = Input(shape=(max_len, ))
model = Embedding(input_dim=n_words + 1,
output_dim=50,
input_length=max_len,
mask_zero=True)(input) # 20-dim embedding
model = Bidirectional(
LSTM(units=250, return_sequences=True,
recurrent_dropout=0.2))(model) # variational biLSTM
model = TimeDistributed(Dense(50, activation="relu"))(
model) # a dense layer as suggested by neuralNer
crf = CRF(n_tags) # CRF layer
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer="adam",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.summary()
history = model.fit(X, np.array(y), batch_size=4, epochs=15, verbose=1)
test_pred = model.predict(X, verbose=1)
idx2tag = {i: w for w, i in tag2idx.items()}
pred_labels = pred2label(test_pred, idx2tag)
true_labels = pred2label(y, idx2tag)
f1_train = f1_score(true_labels, pred_labels)
precision_train = precision_score(true_labels, pred_labels)
recall_train = recall_score(true_labels, pred_labels)
train_scores = [f1_train, precision_train, recall_train]
y1 = [to_categorical(i, num_classes=n_tags) for i in y1]
test_pred1 = model.predict(X1, verbose=1)
pred_labels1 = pred2label(test_pred1, idx2tag)
true_labels1 = pred2label(y1, idx2tag)
f1_test = f1_score(true_labels1, pred_labels1)
precision_test = precision_score(true_labels1, pred_labels1)
recall_test = recall_score(true_labels1, pred_labels1)
test_scores = [f1_test, precision_test, recall_test]
print('Testing scores:', test_scores)
return test_scores
model = Bidirectional(
LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(50, activation="relu"))(model)
out = Dense(6, activation='softmax')(model)
#crf = CRF(n_tags+1)
#out = crf(model)
model = Model(input, out)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
history = model.fit(X_train,
numpy.array(y_train),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_split=0.2,
verbose=2)
#history = model.fit(X_train, numpy.array(y_train), batch_size = BATCH_SIZE, epochs = EPOCHS, validation_split = 0.2, verbose = 2)
pred_cat = model.predict(X_test)
pred = numpy.argmax(pred_cat, axis=-1)
y_test_true = numpy.argmax(y_test, -1)
from sklearn_crfsuite.metrics import flat_classification_report
pred_tag = [[idx2tag[i] for i in row] for row in pred]
y_test_true_tag = [[idx2tag[i] for i in row] for row in y_test_true]
#from sklearn.metrics import f1_score
#report = f1_score(y_test, pred_cat)
report = flat_classification_report(y_pred=pred_tag, y_true=y_test_true_tag)
print(report)
max_len = 80
EMBEDDING = 40
input_text = Input(shape=(max_len,))
model = Embedding(input_dim=n_words+2, output_dim=EMBEDDING, # n_words + 2 (PAD & UNK)
input_length=max_len, mask_zero=True)(input_text)
model = Bidirectional(LSTM(units=80, return_sequences=True,
recurrent_dropout=0.1))(model)
model = Bidirectional(LSTM(units=50, return_sequences=True,
recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(50, activation="relu"))(model)
crf = CRF(n_tags+1) # CRF layer, n_tags+1(PAD)
out = crf(model)
model = Model(input_text, out)
model.compile(optimizer="adam", loss=crf.loss_function, metrics=[crf.accuracy])
model.summary()
history = model.fit(X_tr, np.array(y_tr), batch_size=batch_size, epochs=epochs,validation_split=0.1, verbose=2)
pred_cat = model.predict(X_te)
pred = np.argmax(pred_cat, axis=-1)
y_te_true = np.argmax(y_te, -1)
from sklearn_crfsuite.metrics import flat_classification_report
# Convert the index to tag
pred_tag = [[idx2tag[i] for i in row] for row in pred]
y_te_true_tag = [[idx2tag[i] for i in row] for row in y_te_true]
report = flat_classification_report(y_pred=pred_tag, y_true=y_te_true_tag)
print(report)
output = Dense(len(classes) - (2 if arguments.cues else 0),
activation = "softmax")(model);
model = Model([forms_input, cues_input] if arguments.cues else forms_input,
output);
optimizer = optimizers.Adam(lr = arguments.lr);
model.compile(optimizer = optimizer, loss = "categorical_crossentropy",
metrics = ["accuracy"]);
print(model.summary());
monitor = EarlyStopping(monitor = "val_acc", min_delta = 0.0001,
patience = 3, verbose = 1, mode = "max");
board = TensorBoard(log_dir = "log/{}".format(arguments.id));
model.fit(([inputs, cues] if arguments.cues else inputs), outputs,
validation_split = arguments.vs,
batch_size = arguments.bs, epochs = arguments.epochs,
callbacks = [monitor, board],
verbose = 1);
if arguments.debug:
print("model.evaluate() on training: {}"
"".format(model.evaluate(([inputs, cues] if arguments.cues
else inputs),
outputs, verbose = 1)));
model.save(arguments.id + ".h5");
#
# in a few, rare circumstances, we allow ourselves to re-interpret variable
# names, as is the case of .inputs. and .outputs. here: now turning our focus
# to the evaluation data.
#
LSTM(units=50, return_sequences=True,
recurrent_dropout=0.1))(model) # variational biLSTM
model = TimeDistributed(Dense(50, activation="relu"))(
model) # a dense layer as suggested by neuralNer
crf = CRF(n_tags + 1) # CRF layer, n_tags+1(PAD)
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.summary()
history = model.fit(X_tr,
np.array(y_tr),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_split=0.1,
verbose=2)
# Eval
pred_cat = model.predict(X_te)
pred = np.argmax(pred_cat, axis=-1)
y_te_true = np.argmax(y_te, -1)
# Convert the index to tag
pred_tag = [[idx2tag[i] for i in row] for row in pred]
y_te_true_tag = [[idx2tag[i] for i in row] for row in y_te_true]
report = flat_classification_report(y_pred=pred_tag, y_true=y_te_true_tag)
print(report)
def hyperopt_train_test(params):
epsilon = 10**params['epsilon_exp']
optimizer = optimizers.adam(lr=params['learning_rate'], epsilon=epsilon)
if dmc_parameters["use_embedding_layer"]:
input = Input(shape=(dmc_parameters["max_seq_len"], ))
model = Embedding(input_dim=dmc_parameters["one_hot_vector_len"],
output_dim=params['embedding_layer_output'],
input_length=dmc_parameters["max_seq_len"])(input)
model = Dropout(rate=params['embedding_dropout'])(model)
else:
input = Input(shape=(dmc_parameters["max_seq_len"],
dmc_parameters["one_hot_vector_len"]))
model = input
if params['bi_lstm1_units'] > 0:
model = Bidirectional(
CuDNNLSTM(units=params['bi_lstm1_units'],
return_sequences=True))(model)
if params['bi_lstm2_units'] > 0:
model = Bidirectional(
CuDNNLSTM(units=params['bi_lstm2_units'],
return_sequences=True))(model)
if dmc_parameters["use_crf_layer"]:
crf = CRF(dmc_parameters["num_tags"]) # CRF layer
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer=optimizer,
loss=losses.crf_loss,
metrics=[metrics.crf_accuracy,
avg_proximity_metric()])
else:
out = TimeDistributed(
Dense(dmc_parameters["num_tags"], activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy", avg_proximity_metric()])
model.summary()
es = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=dmc_parameters["patience"],
verbose=False,
mode='min',
restore_best_weights=True)
history = model.fit(X_tr,
np.array(y_tr),
batch_size=dmc_parameters['batch_size'],
epochs=dmc_parameters["epochs"],
validation_data=(X_vl, np.array(y_vl)),
verbose=False,
shuffle=True,
callbacks=[es])
loss, acc, prox = model.evaluate(x=X_vl,
y=np.array(y_vl),
batch_size=dmc_parameters['batch_size'],
verbose=False)
validation_labels = deepMirCut.pred2label(y_vl, dmc_parameters)
validation_pred = model.predict(X_vl, verbose=False)
pred_labels = deepMirCut.pred2label(validation_pred, dmc_parameters)
fScore = f1_score(validation_labels, pred_labels)
return loss, acc, prox, fScore
model = Model(inputs=input, outputs=[output1, output2, output3, output4])
model.compile(optimizer="Adam",
loss="mean_squared_error",
metrics=[metrics.mae, metrics.binary_accuracy])
model.summary()
y1 = y_trainc
y2 = y_trainc.reshape(1, -1).T
y3 = y_trainc.reshape(1, -1).T
y4 = y_trainc.reshape(-1, 1, 1)
history = model.fit(Xtrc, [y4, y2, y3, y1],
batch_size=1,
epochs=2,
validation_split=0.05,
verbose=1)
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_recall_fscore_support
confE = []
confN = []
confT = []
confJ = []
#precal=[]
yactual = []
ypred = []
model.summary()
# Saving the best model only
filepath = "ner-bi-lstm-td-model-{val_accuracy:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [checkpoint]
# Fit the best model
history = model.fit(X_train,
np.array(y_train),
batch_size=256,
epochs=20,
validation_split=0.1,
verbose=1,
callbacks=callbacks_list)
# In[ ]:
def plot_history(history):
accuracy = history.history['accuracy']
val_accuracy = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
x = range(1, len(accuracy) + 1)
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
model = Bidirectional(
LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(50, activation="relu"))(model)
crf = CRF_2nd(no_tags)
out_layer = crf(model)
model = Model(input, out_layer)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.summary()
history = model.fit(train_X,
np.array(train_y),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_split=0.1,
verbose=2)
pred_y = model.predict(test_X)
pred_y = np.argmax(pred_y, axis=-1)
y_test_true = np.argmax(test_y, -1)
y_test_true = [[index_to_tag[i] for i in row] for row in y_test_true]
y_test_true = [[x for x in row if x != 'PADword'] for row in y_test_true]
pred_y = [[index_to_tag[i] for i in row] for row in pred_y]
pred_y = [[x.replace("PADword", "O")
for x in pred_y[index]][:len(y_test_true[index])]
for index in range(len(y_test_true))]
print('LSTM Classification Report\n',
