def main(): X, y, words, tags = load_dataset(DATA_PATH) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15) inp = Input(shape=(MAX_LEN, )) model = Embedding(input_dim=len(words) + 2, output_dim=EMBEDDING_SIZE, input_length=MAX_LEN, mask_zero=True)(inp) model = Bidirectional( LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(50, activation="relu"))(model) crf = CRF(len(tags) + 1) # CRF layer out = crf(model) # output model = Model(inp, out) model.compile(optimizer="rmsprop", loss=crf.loss, metrics=[crf.accuracy]) model.summary() checkpointer = ModelCheckpoint(filepath='model.h5', verbose=0, mode='auto', save_best_only=True, monitor='val_loss') history = model.fit(X_train, np.array(y_train), batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.1, callbacks=[checkpointer])
def create_model(n_words, n_tags, MAX_LEN=MAX_LEN, EMBEDDING=EMBEDDING): input = Input(shape=(MAX_LEN, )) model = Embedding(input_dim=n_words + 2, output_dim=EMBEDDING, input_length=MAX_LEN, mask_zero=True)(input) model = Bidirectional( LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(50, activation="relu"))(model) out = Dense(n_tags + 1, activation="softmax")(model) model = Model(input, out) model.compile(optimizer="adam", loss="mse", metrics=["accuracy"]) return model
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) from tensorflow.keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional import tensorflow.keras.layers as layers print(X_test[0].shape) input = layers.Input(shape=(140, )) model = Embedding(input_dim=n_words, output_dim=140, input_length=140)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(n_tags, activation="softmax"))( model) # softmax output layer model = Model(input, out) model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) #p = model.predict(np.array([X_test[0]])) history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=1, validation_split=0.2, verbose=1) i = 0 p = model.predict(np.array([new_sent])) p = np.argmax(p, axis=-1) print("{:14} ({:5}): {}".format("Word", "True", "Pred")) for w, pred in zip(new_sent, p[0]): print("{:14}: {}".format(words[w], tags[pred]))
def fit_predict(self, testIds): print("START PREPROCESSING") vocabs = PreProcessor.getVocabs(self.data) train_data = dict() test_data = dict() for k, v in self.data.items(): if k in testIds: test_data[k] = v else: train_data[k] = v train_data = PreProcessor.preprocess(train_data, self.extraction_of) test_data = PreProcessor.preprocess(test_data, self.extraction_of) words = list(set(vocabs)) words.append("ENDPAD") n_words = len(words) tags = ['O', self.extraction_of] n_tags = len(tags) word2idx = {w: i for i, w in enumerate(words)} tag2idx = {t: i for i, t in enumerate(tags)} getter_train = SentenceGetter(train_data) sentences_train = getter_train.sentences getter_test = SentenceGetter(test_data) sentences_test = getter_test.sentences X_train = [[word2idx[w[0]] for w in s] for s in sentences_train] X_train = pad_sequences(maxlen=140, sequences=X_train, padding="post", value=n_words - 1) X_test = [[word2idx[w[0]] for w in s] for s in sentences_test] X_test = pad_sequences(maxlen=140, sequences=X_test, padding="post", value=n_words - 1) y_train = [[tag2idx[w[1]] for w in s] for s in sentences_train] y_train = pad_sequences(maxlen=140, sequences=y_train, padding="post", value=tag2idx["O"]) y_train = [to_categorical(i, num_classes=n_tags) for i in y_train] y_test = [[tag2idx[w[1]] for w in s] for s in sentences_test] y_test = pad_sequences(maxlen=140, sequences=y_test, padding="post", value=tag2idx["O"]) y_test = [to_categorical(i, num_classes=n_tags) for i in y_test] input = Input(shape=(140, )) model = Embedding(input_dim=n_words, output_dim=50, input_length=140)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(n_tags, activation="softmax"))(model) model = Model(input, out) model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=3, validation_split=0.2, verbose=1) y_true = [] y_pred = [] for i in tqdm(range(0, len(X_test))): p = model.predict(np.array(X_test[i])) p = np.argmax(p, axis=-1) t = y_test[i] for wordIdx, trueTagIdx, predTagIdx in zip(X_test[i], t, p[0]): tag_true = tags[np.argmax(trueTagIdx, axis=-1)] tag_pred = tags[predTagIdx] y_true.append(tag_true) y_pred.append(tag_pred) return y_true, y_pred
# creating the model input_data = Input(shape=(max_len, )) model = Embedding(input_dim=len(words), output_dim=50, input_length=max_len)(input_data) model = SpatialDropout1D(rate=0.1)(model) model = Bidirectional( LSTM(units=128, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(units=len(tags), activation='softmax'))(model) model = Model(input_data, out) model.summary() # In[32]: model.compile(optimizer='adam', metrics=['accuracy'], loss='sparse_categorical_crossentropy') # In[33]: model.fit(x=X_train, y=Y_train, batch_size=32, validation_data=(X_test, Y_test), epochs=3, verbose=1) # In[34]: model.save('./')
def model_akty(seq_len_middle, tag_index, embedding_dim, word_index, word_index_morf, embedding_matrix_1, embedding_matrix_2): ## main model input = Input(shape=(seq_len_middle, )) model = Embedding( len(word_index) + 1, embedding_dim, embeddings_initializer=initializers.Constant(embedding_matrix_1), input_length=seq_len_middle, trainable=False)(input) # model = Bidirectional (LSTM (units=85, return_sequences=True, dropout=drop), merge_mode="mul") (model) # model = TimeDistributed (Dense (100, activation="relu")) (model) model = Bidirectional( LSTM(128, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(model) model = LSTM(128, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(model) model = GRU(64, dropout=0.1, recurrent_dropout=0.5, activation='tanh', recurrent_activation='sigmoid')(model) model = Flatten()(model) input3 = Input(shape=(seq_len_middle, )) model3 = Embedding( len(word_index_morf) + 1, embedding_dim, embeddings_initializer=initializers.Constant(embedding_matrix_2), input_length=seq_len_middle, trainable=True)(input3) model3 = Bidirectional( LSTM(128, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(model3) model3 = LSTM(128, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(model3) model3 = GRU(64, dropout=0.1, recurrent_dropout=0.5, activation='tanh', recurrent_activation='sigmoid')(model3) model3 = Flatten()(model3) merged = tf.keras.layers.concatenate([model, model3]) model = Dense(64, activation='relu')(merged) # model = Dropout(0.20) (model) # model = Dense (10, activation='relu') (model) # out = (Dense (2, activation='softmax')) (model) out = (Dense(len(tag_index) + 1, activation='softmax'))(model) # model = Model (input, out) model = Model(inputs=[input, input3], outputs=out) # model.compile (loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # model.compile (loss='binary_crossentropy', optimizer='Adadelta', metrics=['accuracy']) model.compile(loss='mse', optimizer=rmsprop, metrics=['accuracy']) model.summary() return model