def create_model(train=True):
if train:
(train_x, train_y, train_max_len, train_length), (
test_x, test_y, test_max_len, test_length), \
(vocab, maxlen, chunk_tags, embedding_weights) = process_data.load_lstm_data()
else:
with open('model/chars-config.pkl', 'rb') as inp:
(vocab, chunk_tags, embedding_weights) = pickle.load(inp)
input = Input(shape=(train_max_len, ))
model = Embedding(len(vocab) + 1, EMBED_DIM, mask_zero=True)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=200, return_sequences=True, recurrent_dropout=0.1))(model)
model = Dropout(0.7)(model)
out = TimeDistributed(Dense(len(chunk_tags) + 1,
activation="softmax"))(model)
model = Model(input, out)
model.summary()
model.compile('adam',
loss="categorical_crossentropy",
metrics=["accuracy"])
if train:
return model, (train_x, train_y,
train_max_len), (test_x, test_y,
test_max_len), (vocab, chunk_tags)
else:
return model, (vocab, chunk_tags)
def create_model(number_of_words_total, number_of_tags, len_max,
embedding_size, lstm_units, dropout, recurrent_dropout):
# Input
input_layer = Input(shape=(len_max, ))
# Embedding Layer
model = Embedding(input_dim=number_of_words_total,
output_dim=embedding_size,
input_length=len_max)(input_layer)
# BI-LSTM Layer
model = Bidirectional(
LSTM(units=lstm_units,
return_sequences=True,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
kernel_initializer=keras.initializers.he_normal()))(model)
# TimeDistributed layer
model = TimeDistributed(Dense(number_of_tags, activation="relu"))(model)
# CRF Layer
crf = CRF(number_of_tags)
# Output
output_layer = crf(model)
model = Model(input_layer, output_layer)
# Optimiser
adam = Adam(lr=0.0005, beta_1=0.9, beta_2=0.999)
model.compile(optimizer=adam,
loss=crf.loss_function,
metrics=[crf.accuracy, 'accuracy'])
model.summary()
return model
def lstm_crf(x, y, vocab_size, n_tags, batch_size, epochs):
output_dim = 30
hid_size = 50
dense1 = 50
seq_len = x.shape[1]
input_ = Input(shape=(seq_len, ))
model = Embedding(input_dim=vocab_size,
output_dim=output_dim,
input_length=seq_len,
mask_zero=True)(input_)
model = Bidirectional(
LSTM(units=hid_size, return_sequences=True,
recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(dense1, activation='relu'))(model)
crf = CRF(n_tags, learn_mode='marginal')
out = crf(model) # prob
model = Model(inputs=input_, outputs=out)
model.compile(optimizer='rmsprop', loss=crf_loss, metrics=[crf.accuracy])
model.summary()
history = model.fit(x,
np.array(y),
batch_size=batch_size,
epochs=epochs,
validation_split=0.1,
verbose=1)
return model, history
def train():
input = Input(shape=(max_len, ))
model = Embedding(input_dim=n_words + 1,
output_dim=100,
input_length=max_len,
mask_zero=True)(input) # 20-dim embedding
model = Bidirectional(
LSTM(units=50, return_sequences=True,
recurrent_dropout=0.1))(model) # variational biLSTM
model = TimeDistributed(Dense(50, activation="relu"))(
model) # a dense layer as suggested by neuralNer
crf = CRF(n_tags) # CRF layer
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.summary()
history = model.fit(x_train,
np.array(y_train),
batch_size=64,
epochs=5,
validation_split=0.1,
verbose=1)
save_load_utils.save_all_weights(model, filepath="models/bilstm-crf.h5")
hist = pd.DataFrame(history.history)
print(hist)
plt.figure(figsize=(12, 12))
plt.plot(hist["crf_viterbi_accuracy"])
plt.plot(hist["val_crf_viterbi_accuracy"])
plt.show()
def train():
input = Input(shape=(max_len, ))
model = Embedding(input_dim=n_words, output_dim=50,
input_length=max_len)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(n_tags, activation='softmax'))(
model) # softmax output layer
model = Model(input, out)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
# checkpoint
# filepath = "../result/bilstm-weights-{epoch:02d}-{val_acc:.2f}.hdf5"
# checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
# history=model.fit(X_train,np.array(y_train),batch_size=32,epochs=5,validation_split=0.1,verbose=1,callbacks=[checkpoint])
history = model.fit(X_train,
np.array(y_train),
batch_size=32,
epochs=5,
validation_split=0.1,
verbose=1)
# äæåęØ”å
model.save(filepath="../result/bi-lstm.h5")
hist = pd.DataFrame(history.history)
plt.figure(figsize=(12, 12))
plt.plot(hist["acc"])
plt.plot(hist["val_acc"])
plt.show()
def define_model(self):
"""Define the Bi-LSTM/CRF model"""
input_layer = Input(shape=(self.MAX_SEQ_LEN, ))
model = Embedding(
input_dim=self.n_words,
output_dim=self.EMBEDDING, # actual n_words + 2 (PAD & UNK)
input_length=self.MAX_SEQ_LEN)(
input_layer) # default: 300-dim embedding
model = Bidirectional(
LSTM(units=self.LSTM_HIDDEN_UNITS,
return_sequences=True,
recurrent_dropout=0.1))(model) # variational biLSTM
model = TimeDistributed(Dense(self.LSTM_DENSE_DIM, activation="relu"))(
model) # a dense layer as suggested by neuralNer
crf = CRF(self.n_tags) # CRF layer, actual n_tags+1(PAD)
output_layer = crf(model) # output
model = Model(input_layer, output_layer)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.summary()
return model
def get_model():
""" Definition du model cnn utilise """
inp = Input(shape=(MAX_TEXT_LENGTH,))
model = Embedding(MAX_TEXT_LENGTH, EMBED_SIZE)(inp)
model = Conv1D(filters=64, kernel_size=7, padding='same', activation='relu')(model)
model = MaxPooling1D(pool_size=3)(model)
model = BatchNormalization(axis=1)(model)
model = Dropout(0.25)(model)
model = Conv1D(filters=128, kernel_size=5, padding='same', activation='relu')(model)
model = MaxPooling1D(pool_size=5)(model)
model = BatchNormalization(axis=1)(model)
model = Dropout(0.3)(model)
model = Flatten()(model)
model = Dense(1024, activation="relu")(model)
model = Dense(10, activation="softmax")(model)
model = Model(inputs=inp, outputs=model)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
model.summary()
return model
def get_model(units, dout, emb, mtl, n_out):
"""
Build Model
:param units: cells number of conv1D
:param dout: rate for dropout
:param emb: output size of embedding layer
:param mlt: input size
:param n_out: number of classes to dense layer
:return: model
"""
inp = Input(shape=(mtl, ))
model = Embedding(mtl, emb)(inp)
model = Dropout(dout)(model)
model = Conv1D(filters=units,
kernel_size=emb,
padding='same',
activation='relu')(model)
model = MaxPooling1D(pool_size=2)(model)
model = Flatten()(model)
model = Dense(n_out, activation="softmax")(model)
model = Model(inputs=inp, outputs=model)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
return model
def createArchitecture(parameters):
optimizer = 0
if parameters["optimizer"] == 'rmsprop':
optimizer = optimizers.rmsprop(lr=parameters["learning_rate"],
epsilon=parameters["epsilon"])
elif parameters["optimizer"] == 'adam':
optimizer = optimizers.adam(lr=parameters["learning_rate"],
epsilon=parameters["epsilon"])
elif parameters["optimizer"] == 'nadam':
optimizer = optimizers.nadam(lr=parameters["learning_rate"],
epsilon=parameters["epsilon"])
elif parameters["optimizer"] == 'sgd':
optimizer = optimizers.sgd(lr=parameters["learning_rate"])
#else:
# optimizer = parameters["optimizer"]
if parameters["use_embedding_layer"]:
input = Input(shape=(parameters["max_seq_len"], ))
model = Embedding(input_dim=parameters["one_hot_vector_len"],
output_dim=parameters["embedding_layer_output"],
input_length=parameters["max_seq_len"])(input)
if parameters["embedding_dropout"] > 0:
model = Dropout(rate=parameters["embedding_dropout"])(model)
else:
input = Input(shape=(parameters["max_seq_len"],
parameters["one_hot_vector_len"]))
model = input
if parameters["bi_lstm1_units"] > 0:
model = Bidirectional(
CuDNNLSTM(units=parameters["bi_lstm1_units"],
return_sequences=True))(model)
if parameters["bi_lstm2_units"] > 0:
model = Bidirectional(
CuDNNLSTM(units=parameters["bi_lstm2_units"],
return_sequences=True))(model)
if parameters["bi_lstm3_units"] > 0:
model = Bidirectional(
CuDNNLSTM(units=parameters["bi_lstm3_units"],
return_sequences=True))(model)
if parameters["use_crf_layer"]:
crf = CRF(parameters["num_tags"], learn_mode="marginal")
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer=optimizer,
loss=losses.crf_loss,
metrics=[metrics.crf_accuracy,
avg_proximity_metric()])
else:
out = TimeDistributed(
Dense(parameters["num_tags"], activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy", avg_proximity_metric()])
model.summary()
return model
def BUILD_MODEL(X,MAX,n_words,n_tags,embedding_matrix):
input_word = Input(shape = (MAX,))
model=Embedding(input_dim=n_words,input_length=X.shape[1], output_dim=embedding_matrix.shape[1], weights=[embedding_matrix],trainable=False)(input_word)
model=Bidirectional(LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(model)
model=Bidirectional(LSTM(32, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(model)
out=TimeDistributed(Dense(n_tags, activation ='softmax'))(model)
model = Model(input_word, out)
model.summary()
return model
def BUILD_MODEL(X,MAX,n_words,n_tags,embedding_matrix):
input_word = Input(shape = (MAX,))
model=Embedding(input_dim=n_words,input_length=X.shape[1], output_dim=embedding_matrix.shape[1], weights=[embedding_matrix],trainable=False)(input_word)
model=Bidirectional(LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(model)
model=TimeDistributed(Dense(32, activation ='relu'))(model)
crf = CRF(n_tags) # CRF layer
out = crf(model) # output
model = Model(input_word, out)
model.summary()
model.compile(optimizer='adam', loss=crf.loss_function, metrics=[crf.accuracy, 'accuracy'])
return model
def model_stuff():
global model
word_embedding_size = 100
input = Input(shape=(maxlen,))
model = Embedding(input_dim=n_words, output_dim=word_embedding_size, weights=[glove_embedding_matrix()], input_length=maxlen, trainable=False)(input)
model = Bidirectional(LSTM(units=word_embedding_size,return_sequences=True,dropout=0.5,recurrent_dropout=0.5,kernel_initializer=k.initializers.he_normal()))(model)
model = Bidirectional(LSTM(units=word_embedding_size*2,return_sequences=True,dropout=0.5,recurrent_dropout=0.5,kernel_initializer=k.initializers.he_normal()))(model)
model = TimeDistributed(Dense(n_tags, activation="relu"))(model)
crf = CRF(n_tags)
out = crf(model)
model = Model(input, out)
# adam = k.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999)
model.compile(optimizer='adam', loss=crf.loss_function, metrics=[crf.accuracy, 'accuracy'])
model.summary()
def build_keras_model(max_len, input_dim, output_dim, embedding_matrix):
logging.debug('building Keras model...')
input = Input(shape=(max_len, ))
model = Embedding(input_dim=input_dim,
output_dim=output_dim,
input_length=max_len,
weights=[embedding_matrix],
trainable=False)(input)
model = Dropout(0.1)(model)
n_units = 128
# model = Dense(n_units)(model)
# model = LSTM(units=n_units, return_sequences=True, recurrent_dropout=0.1)(model)
# TODO the model below is a stronger bi-directional model
model = Bidirectional(
LSTM(units=n_units, return_sequences=True,
recurrent_dropout=0.1))(model)
# TODO this should be a parameter
n_tags = 5
# TODO what does TimeDistributed do?
out = TimeDistributed(Dense(n_tags, activation='softmax'))(model)
model = Model(input, out)
logging.debug('Model type: ')
logging.debug(type(model))
logging.debug('Model summary: ')
logging.debug(model.summary())
logging.debug('done building model...')
return model
def create(self, train_file_path: str, output_summary: bool = False):
if not os.path.exists(train_file_path):
raise FileNotFoundError
with open(train_file_path, 'r') as train_file:
self.max_words_count_in_sentence = pd.read_csv(train_file).groupby(
"article_id").size().max()
input_layer = Input(shape=(self.max_words_count_in_sentence, ))
word_embedding_size = 150
model = Embedding(
input_dim=len(self.lang.vocab),
output_dim=word_embedding_size,
input_length=self.max_words_count_in_sentence)(input_layer)
model = Bidirectional(
LSTM(units=word_embedding_size,
return_sequences=True,
dropout=0.5,
recurrent_dropout=0.5,
kernel_initializer=keras.initializers.he_normal()))(model)
model = LSTM(units=word_embedding_size * 2,
return_sequences=True,
dropout=0.5,
recurrent_dropout=0.5,
kernel_initializer=keras.initializers.he_normal())(model)
model = TimeDistributed(Dense(len(self._tags),
activation="relu"))(model)
crf = CRF(len(self._tags))
model = Model(input_layer, crf(model))
model.compile(optimizer=keras.optimizers.Adam(lr=0.0005,
beta_1=0.9,
beta_2=0.999),
loss=crf.loss_function,
metrics=[crf.accuracy, 'accuracy'])
if output_summary:
model.summary()
self._model = model
def get_model():
embed_size = 300
inp = Input(shape=(MAX_TEXT_LENGTH,))
model = Embedding(MAX_FEATURES, embed_size)(inp)
model = Dropout(0.2)(model)
model = Conv1D(filters=32, kernel_size=2, padding='same', activation='relu')(model)
model = MaxPooling1D(pool_size=2)(model)
model = Conv1D(filters=32, kernel_size=2, padding='same', activation='relu')(model)
model = MaxPooling1D(pool_size=2)(model)
model = GRU(128)(model)
model = Dense(64, activation="relu")(model)
model = Dense(32, activation="relu")(model)
model = Dense(16, activation="relu")(model)
model = Dense(6, activation="sigmoid")(model)
model = Model(inputs=inp, outputs=model)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
return model
def get_model():
"""
Create the model for a CNN and Embedding.
"""
inputs = Input(shape=(MAX_TEXT_LENGTH, ))
model = Embedding(MAX_FEATURES, EMBED_SIZE,
input_length=MAX_TEXT_LENGTH)(inputs)
model = Dropout(0.5)(model)
model = Conv1D(NUM_FILTERS, 5, padding="same", activation="relu")(model)
model = MaxPooling1D(pool_size=30)(model)
model = Flatten()(model)
model = Dense(N_OUT, activation="softmax")(model)
model = Model(inputs=inputs, outputs=model)
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.summary()
return model
def create_model():
BATCH_SIZE = 128
EPOCHS = 15
MAX_LEN = 40
EMBEDDING = 40
input = Input(shape=(MAX_LEN,))
model = Embedding(input_dim=n_words+2, output_dim=EMBEDDING,
input_length=MAX_LEN, mask_zero=True)(input)
model = Bidirectional(LSTM(units=25, return_sequences=True,
recurrent_dropout=0.3))(model)
model = TimeDistributed(Dense(50, activation="relu"))(model)
out = TimeDistributed(Dense(n_tags+1, activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=[metrics.categorical_accuracy])
model.summary()
return model
def define_model(self, embedding_dim, hidden_dim, use_saved_model_if_found, path_to_saved_model):
input = Input(shape=(self.max_len,))
model = Embedding(input_dim=len(self.word2idx) + 1, output_dim=embedding_dim,
weights=[self.embedding_matrix], input_length=self.max_len,
mask_zero=True, trainable=False)(input)
model = Bidirectional(LSTM(units=hidden_dim, return_sequences=True,
recurrent_dropout=0.1))(model) # variational biLSTM
model = TimeDistributed(Dense(hidden_dim, activation="relu"))(model) # a dense layer as suggested by neuralNer
model = Dense(hidden_dim, activation="relu")(model)
out = Dense(len(self.tag2idx), activation="softmax")(model)
model = Model(input, out)
model.compile(optimizer="rmsprop", loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()
if use_saved_model_if_found:
my_file = Path(path_to_saved_model)
if my_file.is_file():
self.load_model(path_to_saved_model)
self.model = model
def get_model_first(embedding_weights, word_index, vocab_dim, max_length, print_summary=True):
inp = Input(shape=(max_length,))
model = Embedding(input_dim=len(word_index)+1,
output_dim=vocab_dim,
trainable=False,
weights=[embedding_weights])(inp)
model = LSTM(vocab_dim, return_sequences=True)(model)
model = Dropout(0.2)(model)
model = LSTM(vocab_dim, return_sequences=False)(model)
model = Dropout(0.1)(model)
model = Dense(int(vocab_dim/10), activation='relu')(model)
model = Dropout(0.1)(model)
model = Dense(5, activation='softmax')(model)
model = Model(inputs=inp, outputs=model)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
if print_summary:
model.summary()
return model
def create_keras_model(n_words, n_tags, units, maxlen, dropout):
input = Input(shape=(maxlen, ))
model = Embedding(input_dim=n_words,
output_dim=maxlen,
input_length=maxlen)(input)
model = Dropout(dropout)(model)
model = Bidirectional(
LSTM(units=units, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(n_tags, activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy"])
model.summary()
return model
def get_model():
inp = Input(shape=(MAX_TEXT_LENGTH, ))
model = Embedding(MAX_TEXT_LENGTH, EMBED_SIZE)(inp)
model = Dropout(0.2)(model)
model = Conv1D(filters=Filters,
kernel_size=kernel_sizes,
padding='same',
activation='relu')(model)
model = MaxPooling1D(pool_size=kernel_sizes)(model)
model = Flatten()(model)
model = Dense(10, activation="softmax")(model)
model = Model(inputs=inp, outputs=model)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
return model
def __init__(self,
model_pars=None,
data_pars=None,
compute_pars=None,
**kwargs):
### Model Structure ################################
if model_pars is None:
self.model = None
else:
data_set, internal_states = get_dataset(data_pars)
X_train, X_test, y_train, y_test = data_set
words = internal_states.get("word_count")
max_len = X_train.shape[1]
num_tags = y_train.shape[2]
# Model architecture
input = Input(shape=(max_len, ))
model = Embedding(
input_dim=words,
output_dim=model_pars["embedding"],
input_length=max_len,
)(input)
model = Bidirectional(
LSTM(units=50, return_sequences=True,
recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(50, activation="relu"))(model)
crf = CRF(num_tags) # CRF layer
out = crf(model) # output
model = KModel(input, out)
model.compile(
optimizer=model_pars["optimizer"],
loss=crf.loss_function,
metrics=[crf.accuracy],
)
model.summary()
self.model = model
def define_model():
input = Input(shape=(max_wrd_len, ))
mask = Masking(mask_value=0)(input)
model = Embedding(input_dim=chr_vocab_size,
output_dim=100,
input_length=max_wrd_len,
mask_zero=True)(mask)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=250, return_sequences=True, recurrent_dropout=0.1))(model)
model = Bidirectional(LSTM(units=250, recurrent_dropout=0.1))(model)
out = Dense(num_classes, activation="softmax")(model)
model = Model([input], out)
#load existing weight if exist
if os.path.isfile(outFileName + "-best.hdf5"):
model.load_weights(outFileName + "-best.hdf5")
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
plot_model(model, show_shapes=True, to_file=outFileName + '-plot.png')
return model
def train():
input = Input(shape=(input_max_len, ))
model = Embedding(vocab_size,
embedding_size,
weights=[glove_embedding_matrix()],
input_length=input_max_len,
trainable=False)(input)
model = Bidirectional(
LSTM(embedding_size,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))(model)
model = Bidirectional(
LSTM(2 * embedding_size,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))(model)
model = TimeDistributed(Dense(embedding_size, activation='sigmoid'))(model)
model = Flatten()(model)
model = Dense(input_max_len, activation='sigmoid')(model)
out = model
model = Model(input, out)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
print(model.summary())
history = model.fit(padded_features,
np.array(final_label_updated),
validation_split=validation_split,
epochs=epochs,
batch_size=batch_size,
verbose=logging,
shuffle=True)
model.save(model_name)
metrics(history, model)
model = Model(input, out)
# In[ ]:
from keras.callbacks import ModelCheckpoint
import matplotlib.pyplot as plt
#Optimiser
adam = k.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999)
# Compile model
model.compile(optimizer=adam,
loss=crf.loss_function,
metrics=[crf.accuracy, 'accuracy'])
model.summary()
# Saving the best model only
filepath = "ner-bi-lstm-td-model-{val_accuracy:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [checkpoint]
# Fit the best model
history = model.fit(X_train,
np.array(y_train),
batch_size=256,
epochs=20,
def hyperopt_train_test(params):
epsilon = 10**params['epsilon_exp']
optimizer = optimizers.adam(lr=params['learning_rate'], epsilon=epsilon)
if dmc_parameters["use_embedding_layer"]:
input = Input(shape=(dmc_parameters["max_seq_len"], ))
model = Embedding(input_dim=dmc_parameters["one_hot_vector_len"],
output_dim=params['embedding_layer_output'],
input_length=dmc_parameters["max_seq_len"])(input)
model = Dropout(rate=params['embedding_dropout'])(model)
else:
input = Input(shape=(dmc_parameters["max_seq_len"],
dmc_parameters["one_hot_vector_len"]))
model = input
if params['bi_lstm1_units'] > 0:
model = Bidirectional(
CuDNNLSTM(units=params['bi_lstm1_units'],
return_sequences=True))(model)
if params['bi_lstm2_units'] > 0:
model = Bidirectional(
CuDNNLSTM(units=params['bi_lstm2_units'],
return_sequences=True))(model)
if dmc_parameters["use_crf_layer"]:
crf = CRF(dmc_parameters["num_tags"]) # CRF layer
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer=optimizer,
loss=losses.crf_loss,
metrics=[metrics.crf_accuracy,
avg_proximity_metric()])
else:
out = TimeDistributed(
Dense(dmc_parameters["num_tags"], activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy", avg_proximity_metric()])
model.summary()
es = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=dmc_parameters["patience"],
verbose=False,
mode='min',
restore_best_weights=True)
history = model.fit(X_tr,
np.array(y_tr),
batch_size=dmc_parameters['batch_size'],
epochs=dmc_parameters["epochs"],
validation_data=(X_vl, np.array(y_vl)),
verbose=False,
shuffle=True,
callbacks=[es])
loss, acc, prox = model.evaluate(x=X_vl,
y=np.array(y_vl),
batch_size=dmc_parameters['batch_size'],
verbose=False)
validation_labels = deepMirCut.pred2label(y_vl, dmc_parameters)
validation_pred = model.predict(X_vl, verbose=False)
pred_labels = deepMirCut.pred2label(validation_pred, dmc_parameters)
fScore = f1_score(validation_labels, pred_labels)
return loss, acc, prox, fScore
def train(word2Vec, train_df, test_df, max_length, filters, kernel_size,
pool_size, dense):
r = random.randint(1, 10000)
now = datetime.datetime.now()
tokenizer = Tokenizer()
tokenizer.fit_on_texts(word2Vec.wv.vocab.keys())
train_x = tokenizer.texts_to_sequences(train_df['text'])
train_x = pad_sequences(train_x, maxlen=max_length)
# train_y_good = train_df["good"]
# train_y_bad = train_df["bad"]
train_y = pd.DataFrame(train_df, columns=["good", "bad"])
test_x = tokenizer.texts_to_sequences(test_df['text'])
test_x = pad_sequences(test_x, maxlen=max_length)
word_index = tokenizer.word_index
embedding_matrix = np.zeros((len(word_index) + 1, word2Vec.vector_size))
for word, i in word_index.items():
try:
embedding_matrix[i] = word2Vec[word]
except:
continue
inputs = Input(shape=(max_length, ))
model = Embedding(name="embedding",
input_dim=len(word_index) + 1,
output_dim=word2Vec.vector_size,
weights=[embedding_matrix],
input_length=max_length)(inputs)
model = Conv1D(name="conv1D",
filters=filters,
kernel_size=kernel_size,
kernel_initializer=initializers.glorot_uniform(seed=1),
padding='same')(model)
model = MaxPooling1D(name="maxPooling1D",
pool_size=pool_size,
strides=1,
padding='same')(model)
model = Flatten(name="flatten")(model)
model = Dense(name="dense",
output_dim=dense,
kernel_initializer=initializers.glorot_uniform(seed=1),
activation='relu')(model)
model = Dense(name="output",
output_dim=2,
kernel_initializer=initializers.glorot_uniform(seed=1),
activation='relu')(model)
model = Model(inputs=inputs, outputs=model)
model.compile(loss='mae',
optimizer=optimizers.Adam(lr=.001),
metrics=['mse'])
model.summary()
epochs = 15
callback = model.fit(x=train_x,
y=train_y,
epochs=epochs,
validation_split=.3,
batch_size=20,
verbose=1).history
test_y = np.rint(model.predict(x=test_x, batch_size=10,
verbose=1)).astype('int')
seconds = str((datetime.datetime.now() - now).seconds)
with open('test{seconds}_{r}.txt'.format(seconds=seconds, r=r),
'w') as file:
file.write('id,good,bad\n')
for index, data in enumerate(test_y):
file.write('{},{},{}\n'.format(index, data[0], data[1]))
with open('record{seconds}_{r}.log'.format(seconds=seconds, r=r),
'w') as file:
file.write('result\t\n\n')
file.write('\t'.join(
['index', 'loss\t\t', 'mse\t\t\t', 'val_loss\t\t', 'val_mse\t']) +
'\n')
for index, loss, mse, val_loss, val_mse in zip(
range(1, epochs + 1), callback['loss'],
callback['mean_squared_error'], callback['val_loss'],
callback['val_mean_squared_error']):
file.write('\t'.join([
str(index) + '\t', '{:.12f}'.format(loss), '{:.12f}'.format(
mse), '{:.12f}'.format(val_loss), '{:.12f}'.format(val_mse)
]) + '\n')
file.write(
'\nmax_length={max_length}\nmin_count={min_count}, size=270, iter=10, sg=1, workers=10\n'
.format(max_length=max_length, min_count=min_count))
file.write('inputs = Input(shape=(max_length,)\n')
file.write(
'model = Embedding(name="embedding", input_dim=len(word_index)+1, output_dim=word2Vec.vector_size, weights=[embedding_matrix], input_length=max_length)(inputs)\n'
)
file.write(
'model = Conv1D(name="conv1D_good", filters={filters}, kernel_size={kernel_size}, kernel_initializer=initializers.glorot_uniform(seed=1), padding="same")(model)\n'
.format(filters=filters, kernel_size=kernel_size))
file.write(
'model = MaxPooling1D(name="maxPooling1D", pool_size={pool_size}, strides=1, padding="same")(model)\n'
.format(pool_size=pool_size))
file.write(
'model = Dense(name="dense", output_dim={dense}, kernel_initializer=initializers.glorot_uniform(seed=1), activation="relu")(model)\n'
.format(dense=dense))
file.write(
'model = Dense(name="output", output_dim=2, kernel_initializer=initializers.glorot_uniform(seed=1), activation="relu")(model)\n'
)
file.write('model = Model(inputs=inputs, outputs=model)\n')
file.write(
'model.compile(loss="mae", optimizer=optimizers.Adam(lr=.001), metrics=["mse"])\n'
)
import matplotlib.pyplot as plt
fig = plt.figure()
plt.grid(True)
plt.ylim(0, 40)
plt.plot(callback['loss'])
plt.plot(callback['mse'])
plt.plot(callback['val_loss'])
plt.plot(callback['val_mse'])
plt.title('model loss')
plt.ylabel('loss (mae)')
plt.xlabel('epoch')
plt.legend(['train_loss', 'train_mse', 'test_loss', 'test_mse'],
loc='upper right')
fig.savefig('{seconds}_{r}.png'.format(seconds=seconds, r=r), dpi=fig.dpi)
def create_model(x_train, y_train, x_test, y_test):
args = parse_args()
set_logger(args.log_path, args.log_level)
logging.debug('Args:')
logging.debug(args)
lang = construct_languages(args.train)
assert len(lang) == 1
lang = lang[0]
game = initialize_game(train_file=lang.train,
test_file=lang.test,
dev_file=lang.dev,
emb_file=lang.emb,
budget=args.budget,
max_seq_len=args.max_seq_len,
max_vocab_size=args.max_vocab_size,
emb_size=args.embedding_size,
model_name=args.model_name)
max_len = args.max_seq_len
input_dim = args.max_vocab_size
output_dim = args.embedding_size
embedding_matrix = game.w2v
logging.debug('building Keras model...')
input = Input(shape=(max_len, ))
model = Embedding(input_dim=input_dim,
output_dim=output_dim,
input_length=max_len,
weights=[embedding_matrix],
trainable=False)(input)
model = Dropout(0.1)(model)
n_units = 128
model = Bidirectional(
LSTM(units=n_units, return_sequences=True,
recurrent_dropout=0.1))(model)
n_tags = 5
out = TimeDistributed(Dense(n_tags, activation='softmax'))(model)
model = Model(input, out)
logging.debug('Model type: ')
logging.debug(type(model))
logging.debug('Model summary: ')
logging.debug(model.summary())
rmsprop = keras.optimizers.RMSprop(lr={{choice([0.0001])}})
model.compile(optimizer=rmsprop,
loss='categorical_crossentropy',
metrics=['accuracy'])
logging.debug('done building model...')
logging.debug('starting training...')
num_train_examples = len(x_train)
for i in range(num_train_examples):
print('i: ', i)
model.fit(x_train[:i],
y_train[:i],
batch_size=200,
epochs=20,
verbose=0)
logging.debug('done training...')
logging.debug('starting testing...')
num_samples = x_test.shape[0]
logging.debug('Number of samples: {}'.format(num_samples))
max_batch_size = 4096
batch_size = min(num_samples, max_batch_size)
predictions_probability = model.predict(x_test, batch_size=batch_size)
predictions = numpy.argmax(predictions_probability, axis=-1)
fscore = compute_fscore(Y_pred=predictions, Y_true=y_test)
logging.debug('done testing...')
return -fscore
LSTM(units=50, return_sequences=True,
recurrent_dropout=0.1))(model) # variational biLSTM
model = Dropout(0.1)(model)
model = TimeDistributed(Dense(50, activation="relu"))(
model) # a dense layer as suggested by neuralNer
crf = CRF(n_tags) # CRF layer
out = crf(model) # output
# input.shape
model = Model(input, out)
model.compile(optimizer="rmsprop",
loss=crf.loss_function,
metrics=[crf.accuracy])
print(model.summary())
# LOG_DIR = './log'
# get_ipython().system_raw(
# 'tensorboard --logdir {} --host 0.0.0.0 --port 6016 &'
# .format(LOG_DIR))
# get_ipython().system_raw('./ngrok http 6016 &')
# ! curl -s http://localhost:4040/api/tunnels | python3 -c \
# "import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])"
# from keras.callbacks import TensorBoard
# tbCallBack = TensorBoard(log_dir='./log', histogram_freq=1,
# write_graph=True,
# write_grads=True,
def bilstm_crf(train_loc, test_loc):
train_pre = preprocess(train_loc)
test_pre = preprocess(test_loc)
cc_train = cuu(train_pre)
cc_test = cuu(test_pre)
words_all, tags_all = combine_all(cc_train, cc_test)
n_words = len(words_all)
n_tags = len(tags_all)
max_len = 130
word2idx = {w: i for i, w in enumerate(words_all)}
tag2idx = {t: i for i, t in enumerate(tags_all)}
X = [[word2idx[w[0]] for w in s] for s in cc_train]
X = pad_sequences(maxlen=max_len,
sequences=X,
padding="post",
value=n_words - 1)
X1 = [[word2idx[w[0]] for w in s] for s in cc_test]
X1 = pad_sequences(maxlen=max_len,
sequences=X1,
padding="post",
value=n_words - 1)
y = [[tag2idx[w[1]] for w in s] for s in cc_train]
y = pad_sequences(maxlen=max_len,
sequences=y,
padding="post",
value=tag2idx["O"])
y1 = [[tag2idx[w[1]] for w in s] for s in cc_test]
y1 = pad_sequences(maxlen=max_len,
sequences=y1,
padding="post",
value=tag2idx["O"])
y = [to_categorical(i, num_classes=n_tags) for i in y]
input = Input(shape=(max_len, ))
model = Embedding(input_dim=n_words + 1,
output_dim=50,
input_length=max_len,
mask_zero=True)(input) # 20-dim embedding
model = Bidirectional(
LSTM(units=250, return_sequences=True,
recurrent_dropout=0.2))(model) # variational biLSTM
model = TimeDistributed(Dense(50, activation="relu"))(
model) # a dense layer as suggested by neuralNer
crf = CRF(n_tags) # CRF layer
out = crf(model) # output
model = Model(input, out)
model.compile(optimizer="adam",
loss=crf.loss_function,
metrics=[crf.accuracy])
model.summary()
history = model.fit(X, np.array(y), batch_size=4, epochs=15, verbose=1)
test_pred = model.predict(X, verbose=1)
idx2tag = {i: w for w, i in tag2idx.items()}
pred_labels = pred2label(test_pred, idx2tag)
true_labels = pred2label(y, idx2tag)
f1_train = f1_score(true_labels, pred_labels)
precision_train = precision_score(true_labels, pred_labels)
recall_train = recall_score(true_labels, pred_labels)
train_scores = [f1_train, precision_train, recall_train]
y1 = [to_categorical(i, num_classes=n_tags) for i in y1]
test_pred1 = model.predict(X1, verbose=1)
pred_labels1 = pred2label(test_pred1, idx2tag)
true_labels1 = pred2label(y1, idx2tag)
f1_test = f1_score(true_labels1, pred_labels1)
precision_test = precision_score(true_labels1, pred_labels1)
recall_test = recall_score(true_labels1, pred_labels1)
test_scores = [f1_test, precision_test, recall_test]
print('Testing scores:', test_scores)
return test_scores
