def main():
X, y, words, tags = load_dataset(DATA_PATH)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15)
inp = Input(shape=(MAX_LEN, ))
model = Embedding(input_dim=len(words) + 2,
output_dim=EMBEDDING_SIZE,
input_length=MAX_LEN,
mask_zero=True)(inp)
model = Bidirectional(
LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(50, activation="relu"))(model)
crf = CRF(len(tags) + 1) # CRF layer
out = crf(model) # output
model = Model(inp, out)
model.compile(optimizer="rmsprop", loss=crf.loss, metrics=[crf.accuracy])
model.summary()
checkpointer = ModelCheckpoint(filepath='model.h5',
verbose=0,
mode='auto',
save_best_only=True,
monitor='val_loss')
history = model.fit(X_train,
np.array(y_train),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_split=0.1,
callbacks=[checkpointer])
def create_model(n_words, n_tags, MAX_LEN=MAX_LEN, EMBEDDING=EMBEDDING):
input = Input(shape=(MAX_LEN, ))
model = Embedding(input_dim=n_words + 2,
output_dim=EMBEDDING,
input_length=MAX_LEN,
mask_zero=True)(input)
model = Bidirectional(
LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(50, activation="relu"))(model)
out = Dense(n_tags + 1, activation="softmax")(model)
model = Model(input, out)
model.compile(optimizer="adam", loss="mse", metrics=["accuracy"])
return model
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
from tensorflow.keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional
import tensorflow.keras.layers as layers
print(X_test[0].shape)
input = layers.Input(shape=(140, ))
model = Embedding(input_dim=n_words, output_dim=140, input_length=140)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(n_tags, activation="softmax"))(
model) # softmax output layer
model = Model(input, out)
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy"])
#p = model.predict(np.array([X_test[0]]))
history = model.fit(X_train,
np.array(y_train),
batch_size=32,
epochs=1,
validation_split=0.2,
verbose=1)
i = 0
p = model.predict(np.array([new_sent]))
p = np.argmax(p, axis=-1)
print("{:14} ({:5}): {}".format("Word", "True", "Pred"))
for w, pred in zip(new_sent, p[0]):
print("{:14}: {}".format(words[w], tags[pred]))
def fit_predict(self, testIds):
print("START PREPROCESSING")
vocabs = PreProcessor.getVocabs(self.data)
train_data = dict()
test_data = dict()
for k, v in self.data.items():
if k in testIds:
test_data[k] = v
else:
train_data[k] = v
train_data = PreProcessor.preprocess(train_data, self.extraction_of)
test_data = PreProcessor.preprocess(test_data, self.extraction_of)
words = list(set(vocabs))
words.append("ENDPAD")
n_words = len(words)
tags = ['O', self.extraction_of]
n_tags = len(tags)
word2idx = {w: i for i, w in enumerate(words)}
tag2idx = {t: i for i, t in enumerate(tags)}
getter_train = SentenceGetter(train_data)
sentences_train = getter_train.sentences
getter_test = SentenceGetter(test_data)
sentences_test = getter_test.sentences
X_train = [[word2idx[w[0]] for w in s] for s in sentences_train]
X_train = pad_sequences(maxlen=140,
sequences=X_train,
padding="post",
value=n_words - 1)
X_test = [[word2idx[w[0]] for w in s] for s in sentences_test]
X_test = pad_sequences(maxlen=140,
sequences=X_test,
padding="post",
value=n_words - 1)
y_train = [[tag2idx[w[1]] for w in s] for s in sentences_train]
y_train = pad_sequences(maxlen=140,
sequences=y_train,
padding="post",
value=tag2idx["O"])
y_train = [to_categorical(i, num_classes=n_tags) for i in y_train]
y_test = [[tag2idx[w[1]] for w in s] for s in sentences_test]
y_test = pad_sequences(maxlen=140,
sequences=y_test,
padding="post",
value=tag2idx["O"])
y_test = [to_categorical(i, num_classes=n_tags) for i in y_test]
input = Input(shape=(140, ))
model = Embedding(input_dim=n_words, output_dim=50,
input_length=140)(input)
model = Dropout(0.1)(model)
model = Bidirectional(
LSTM(units=100, return_sequences=True,
recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(n_tags, activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=["accuracy"])
history = model.fit(X_train,
np.array(y_train),
batch_size=32,
epochs=3,
validation_split=0.2,
verbose=1)
y_true = []
y_pred = []
for i in tqdm(range(0, len(X_test))):
p = model.predict(np.array(X_test[i]))
p = np.argmax(p, axis=-1)
t = y_test[i]
for wordIdx, trueTagIdx, predTagIdx in zip(X_test[i], t, p[0]):
tag_true = tags[np.argmax(trueTagIdx, axis=-1)]
tag_pred = tags[predTagIdx]
y_true.append(tag_true)
y_pred.append(tag_pred)
return y_true, y_pred
# creating the model
input_data = Input(shape=(max_len, ))
model = Embedding(input_dim=len(words), output_dim=50,
input_length=max_len)(input_data)
model = SpatialDropout1D(rate=0.1)(model)
model = Bidirectional(
LSTM(units=128, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(units=len(tags), activation='softmax'))(model)
model = Model(input_data, out)
model.summary()
# In[32]:
model.compile(optimizer='adam',
metrics=['accuracy'],
loss='sparse_categorical_crossentropy')
# In[33]:
model.fit(x=X_train,
y=Y_train,
batch_size=32,
validation_data=(X_test, Y_test),
epochs=3,
verbose=1)
# In[34]:
model.save('./')
def model_akty(seq_len_middle, tag_index, embedding_dim, word_index,
word_index_morf, embedding_matrix_1, embedding_matrix_2):
## main model
input = Input(shape=(seq_len_middle, ))
model = Embedding(
len(word_index) + 1,
embedding_dim,
embeddings_initializer=initializers.Constant(embedding_matrix_1),
input_length=seq_len_middle,
trainable=False)(input)
# model = Bidirectional (LSTM (units=85, return_sequences=True, dropout=drop), merge_mode="mul") (model)
# model = TimeDistributed (Dense (100, activation="relu")) (model)
model = Bidirectional(
LSTM(128, return_sequences=True, dropout=0.2,
recurrent_dropout=0.2))(model)
model = LSTM(128,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)(model)
model = GRU(64,
dropout=0.1,
recurrent_dropout=0.5,
activation='tanh',
recurrent_activation='sigmoid')(model)
model = Flatten()(model)
input3 = Input(shape=(seq_len_middle, ))
model3 = Embedding(
len(word_index_morf) + 1,
embedding_dim,
embeddings_initializer=initializers.Constant(embedding_matrix_2),
input_length=seq_len_middle,
trainable=True)(input3)
model3 = Bidirectional(
LSTM(128, return_sequences=True, dropout=0.2,
recurrent_dropout=0.2))(model3)
model3 = LSTM(128,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)(model3)
model3 = GRU(64,
dropout=0.1,
recurrent_dropout=0.5,
activation='tanh',
recurrent_activation='sigmoid')(model3)
model3 = Flatten()(model3)
merged = tf.keras.layers.concatenate([model, model3])
model = Dense(64, activation='relu')(merged)
# model = Dropout(0.20) (model)
# model = Dense (10, activation='relu') (model)
# out = (Dense (2, activation='softmax')) (model)
out = (Dense(len(tag_index) + 1, activation='softmax'))(model)
# model = Model (input, out)
model = Model(inputs=[input, input3], outputs=out)
# model.compile (loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# model.compile (loss='binary_crossentropy', optimizer='Adadelta', metrics=['accuracy'])
model.compile(loss='mse', optimizer=rmsprop, metrics=['accuracy'])
model.summary()
return model