def models(self, input_length, vocab_size, embedding_dim, conv_layer_info, connected_layer_info, batch_size,epochs): model_input = Input(shape = (input_length,), dtype = 'int64') model = Embedding(vocab_size, embedding_dim, input_length = input_length)(model_input) for i in range( len(conv_layer_info) ): model = Convolution1D(filters= conv_layer_info[i][0], kernel_size=conv_layer_info[i][1], padding="valid", activation="relu", strides=1)(model) if len(conv_layer_info[i]) > 2: model = MaxPooling1D(pool_size = conv_layer_info[i][2])(model) model = Flatten()(model) for i in range(len(connected_layer_info)): model = Dense(connected_layer_info[i][0],activation = "relu")(model) model = Dropout(connected_layer_info[i][1])(model) model_output = Dense(self.num_of_classes, activation = "softmax")(model) model =Model(inputs=model_input,outputs=model_output) model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) print("Started Training : ") model.fit(self.x_train, self.y_train, batch_size = batch_size, epochs= epochs, validation_data=(self.x_test, self.y_test), verbose=2) print("Training Completed") self.model = model
def cnn_non_static(self): x_train, y_train, x_test, y_test, vocabolary_dict = data_load_and_preproccess( self.word_size, self.sequence_length) embedding_weights = word_to_vector(np.vstack( (x_train, x_test)), vocabolary_dict, self.embedding_dim, self.min_word_count, self.context_window_size) embedding_weights = np.array( [value for value in embedding_weights.values()]) model_input = Input(shape=(self.sequence_length, )) model = Embedding(len(vocabolary_dict), self.embedding_dim, weights=[embedding_weights], input_length=self.sequence_length, name="embedding")(model_input) model = Dropout(self.drop_prob[0])(model) multi_cnn_channel = [] for kernal in self.kernal_size: conv_channel = Convolution1D(filters=self.filters, kernel_size=kernal, padding="valid", activation="relu", strides=1)(model) conv_channel = MaxPooling1D(pool_size=2)(conv_channel) conv_channel = Flatten()(conv_channel) multi_cnn_channel.append(conv_channel) model = Concatenate()(multi_cnn_channel) model = Dropout(self.drop_prob[1])(model) for dimension in self.hidden_dims: model = Dense(dimension, activation="relu")(model) model_output = Dense(1, activation="sigmoid")(model) model = Model(model_input, model_output) model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) print("Started Training : ") model.fit(x_train, y_train, batch_size=self.batch_size, epochs=self.epochs, validation_data=(x_test, y_test), verbose=2) print("Training Completed") self.model = model
def train(): input = Input(shape=(max_len, )) model = Embedding(input_dim=n_words + 1, output_dim=100, input_length=max_len, mask_zero=True)(input) # 20-dim embedding model = Bidirectional( LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) # variational biLSTM model = TimeDistributed(Dense(50, activation="relu"))( model) # a dense layer as suggested by neuralNer crf = CRF(n_tags) # CRF layer out = crf(model) # output model = Model(input, out) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() history = model.fit(x_train, np.array(y_train), batch_size=64, epochs=5, validation_split=0.1, verbose=1) save_load_utils.save_all_weights(model, filepath="models/bilstm-crf.h5") hist = pd.DataFrame(history.history) print(hist) plt.figure(figsize=(12, 12)) plt.plot(hist["crf_viterbi_accuracy"]) plt.plot(hist["val_crf_viterbi_accuracy"]) plt.show()
def trainLSTM(): # fit a LSTM network with an embedding layer from keras.models import Model, Input from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional input = Input(shape=(max_len, )) model = Embedding(input_dim=n_words, output_dim=50, input_length=max_len)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(n_tags, activation="softmax"))(model) model = Model(input, out) model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"]) history = model.fit(X_tr, np.array(y_tr), batch_size=32, epochs=5, validation_split=0.1, verbose=1) hist = pd.DataFrame(history.history) plt.figure(figsize=(12, 12)) plt.plot(hist["acc"]) plt.plot(hist["val_acc"]) plt.show() return model
def train(self): input = Input(shape=(self.max_len, )) model = Embedding(input_dim=self.num_word + 1, output_dim=20, input_length=self.max_len, mask_zero=True)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(50, activation="relu"))( model) # softmax output layer crf = CRF(self.num_tag) out = crf(model) model = Model(input, out) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) self.pad_process() train_X, test_X, train_y, test_y = self.train_test_split() history = model.fit(train_X, np.array(train_y), batch_size=32, epochs=1, validation_split=0.1, verbose=1) self.make_plot(history) return model
def run(X_train, Y_train, X_val, Y_val, embedding_matrix, vocab_size, maxlen=40, emb_dim=300, neg_ratio=0, hidden_dim=300, drop=0.2, r_drop=0.1): ##build model input = Input(shape=(maxlen, )) model = Embedding(vocab_size, emb_dim, weights=[embedding_matrix], input_length=maxlen, trainable=False)(input) model = Dropout(drop)(model) model = Bidirectional( LSTM(hidden_dim, return_sequences=True, recurrent_dropout=r_drop))(model) model = Dropout(drop)(model) out = TimeDistributed(Dense(1, activation='sigmoid'))(model) model = Model(input, out) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc']) earlyStop = [EarlyStopping(monitor='val_loss', patience=1)] history = model.fit(X_train, Y_train, batch_size=64, epochs=10, validation_data=(X_val, Y_val), callbacks=earlyStop) pred = model.predict(X_val) Y_pred = np.squeeze(pred) test = [[1 if y >= threshold else 0 for y in x] for x in Y_pred] test_arr = np.asarray(test) test_arr = np.reshape(test_arr, (-1)) target = np.reshape(Y_val, (-1)) print( metrics.precision_recall_fscore_support(target, test_arr, average=None, labels=[0, 1])) # Y_pred_ = [[1 if y>=threshold else 0 for y in x] for x in Y_pred] Y_val_ = np.squeeze(Y_val) print("Evaluate: dev seg exact") pred_out_dir = out_dir + 'seg_' + str(neg_ratio) + 'neg' gold_dir = '../../data/val_segs/' + 'seg_' + str(neg_ratio) + 'neg' p, r, f = seg_exact_match(test, Y_val_, pred_out_dir, gold_dir) return model, history, p, r, f
def train(X_tr, y_tr): output_dim = 50 word_input = Input(shape=(MAXLEN, )) model = Embedding(input_dim=VOCAB_SIZE, output_dim=output_dim, input_length=MAXLEN)(word_input) model = SpatialDropout1D(0.1)(model) model = Bidirectional( LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(len(tags), activation="softmax"))(model) model = Model(word_input, out) model.compile(optimizer="rmsprop", loss="sparse_categorical_crossentropy", metrics=["accuracy"]) checkpointer = ModelCheckpoint(filepath='NER_model.h5', verbose=0, mode='auto', save_best_only=True, monitor='val_loss') history = model.fit(X_tr, y_tr.reshape(*y_tr.shape, 1), batch_size=BATCH_SIZE, epochs=EPOCHS, shuffle=True, validation_split=VALIDATION_SPLIT, verbose=1, callbacks=[checkpointer]) return history
def cross_validate(self, X, y): X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.1) input = Input(shape=(self.max_len, )) model = Embedding(input_dim=self.n_words, output_dim=50, input_length=self.max_len)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(self.n_labels, activation="softmax"))( model) # softmax output layer model = Model(input, out) model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"]) history = model.fit(X_tr, np.array(y_tr), batch_size=32, epochs=1, validation_split=0.1, verbose=1) p = model.predict(np.array([X_te[10]])) p = np.argmax(p, axis=-1) for w, pred in zip(X_te[10], p[0]): if self.words[w] != 'PADGARBAGE': print("{:15}: {}".format(self.words[w], self.labels[pred]))
def train(): input = Input(shape=(max_len, )) model = Embedding(input_dim=n_words, output_dim=50, input_length=max_len)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(n_tags, activation='softmax'))( model) # softmax output layer model = Model(input, out) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() # checkpoint # filepath = "../result/bilstm-weights-{epoch:02d}-{val_acc:.2f}.hdf5" # checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max') # history=model.fit(X_train,np.array(y_train),batch_size=32,epochs=5,validation_split=0.1,verbose=1,callbacks=[checkpoint]) history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=5, validation_split=0.1, verbose=1) # 保存模型 model.save(filepath="../result/bi-lstm.h5") hist = pd.DataFrame(history.history) plt.figure(figsize=(12, 12)) plt.plot(hist["acc"]) plt.plot(hist["val_acc"]) plt.show()
def lstm_crf(x, y, vocab_size, n_tags, batch_size, epochs): output_dim = 30 hid_size = 50 dense1 = 50 seq_len = x.shape[1] input_ = Input(shape=(seq_len, )) model = Embedding(input_dim=vocab_size, output_dim=output_dim, input_length=seq_len, mask_zero=True)(input_) model = Bidirectional( LSTM(units=hid_size, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(dense1, activation='relu'))(model) crf = CRF(n_tags, learn_mode='marginal') out = crf(model) # prob model = Model(inputs=input_, outputs=out) model.compile(optimizer='rmsprop', loss=crf_loss, metrics=[crf.accuracy]) model.summary() history = model.fit(x, np.array(y), batch_size=batch_size, epochs=epochs, validation_split=0.1, verbose=1) return model, history
def run(X_train, Y_train, X_val, Y_val, embedding_matrix, vocab_size, maxlen=40, emb_dim=300, neg_ratio=0, hidden_dim=300, drop=0.2, r_drop=0.1): ##build model # input = Input(shape=(maxlen,)) # model = Embedding(vocab_size, emb_dim, weights=[embedding_matrix], input_length=maxlen, trainable=False)(input) # model = Dropout(drop)(model) # model = Bidirectional(LSTM(hidden_dim, return_sequences=True, recurrent_dropout=r_drop))(model) # model = Dropout(drop)(model) # out = TimeDistributed(Dense(1, activation='sigmoid'))(model) input = Input(shape=(maxlen,)) model = Embedding(vocab_size, emb_dim, weights=[embedding_matrix], input_length=maxlen, trainable=False)(input) model = Bidirectional(LSTM(hidden_dim, return_sequences=True, recurrent_dropout=r_drop))(model) model = TimeDistributed(Dense(hidden_dim//4, activation='relu'))(model) model = TimeDistributed(Dropout(drop))(model) ##use CRF instead of Dense crf = CRF(2) out = crf(model) model = Model(input, out) Y_train_2 = keras.utils.to_categorical(Y_train) Y_val_2 = keras.utils.to_categorical(Y_val) model.compile(optimizer='adam', loss=crf.loss_function, metrics=[crf.accuracy]) earlyStop = [EarlyStopping(monitor='val_loss', patience=1)] history = model.fit(X_train, Y_train_2, batch_size=64, epochs=10, validation_data=(X_val, Y_val_2), callbacks=earlyStop) preds = model.predict(X_val) test = [[np.argmax(y) for y in x] for x in preds] test_arr = np.asarray(test) test_arr = np.reshape(test_arr, (-1)) print (metrics.precision_recall_fscore_support(np.reshape(Y_val,(-1)), test_arr, average=None, labels=[0, 1])) # Y_pred_ = [[1 if y>=threshold else 0 for y in x] for x in Y_pred] Y_val_ = np.squeeze(Y_val) print ("Evaluate: dev seg exact") pred_out_dir = out_dir+'seg_'+str(neg_ratio)+'neg' gold_dir = '../../data/val_segs/'+'seg_'+str(neg_ratio)+'neg' p, r, f = seg_exact_match(test, Y_val_, pred_out_dir, gold_dir) return model, history, p, r, f
def train(self): input = Input(shape=(120,)) model = Embedding(input_dim=self.num_words, output_dim=50, input_length=120)(input) model = Dropout(0.1)(model) model = Bidirectional(LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(self.num_entities, activation="softmax"))(model) model = Model(input, out) model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"]) history = model.fit(x=self.X_train, y=np.array(self.Y_train), batch_size=64, epochs=10, validation_data=(self.X_validation, self.Y_validation)) model.save("../models/ner_" + str(datetime.utcnow().microsecond)) test_eval = model.evaluate(self.X_test, self.Y_test, verbose=0) print('Test loss:', test_eval[0]) print('Test accuracy:', test_eval[1]) return model, history
def run(): sentences = read_train_file(TRAIN_PATH) word_map, tag_map = create_word_idx() max_len = max([len(s) for s in sentences]) X = [[word_map[w[0]] for w in s] for s in sentences] n_words = len(word_map) n_tags = 9 X = pad_sequences(maxlen=max_len, sequences=X, padding="post", value=n_words - 1) y = [[tag_map[w[2]] for w in s] for s in sentences] y_testing = pad_sequences(maxlen=max_len, sequences=y, padding="post", value=tag_map["O"]) y = [to_categorical(i, num_classes=n_tags) for i in y_testing] dev_sentences = read_train_file(DEV_PATH) dev_max_len = max([len(s) for s in dev_sentences]) X_dev = [[word_map[w[0]] for w in s] for s in dev_sentences] X_dev = pad_sequences(maxlen=max_len, sequences=X_dev, padding="post", value=n_words - 1) y_dev = [[tag_map[w[2]] for w in s] for s in dev_sentences] y_dev = pad_sequences(maxlen=max_len, sequences=y_dev, padding="post", value=tag_map["O"]) y_dev = [to_categorical(i, num_classes=9) for i in y_dev] test_sentences = read_train_file(TEST_PATH) test_max_len = 33 X_test = [[word_map[w[0]] for w in s] for s in test_sentences] X_test = pad_sequences(maxlen=test_max_len, sequences=X_test, padding="post", value=n_words - 1) y_test = [[tag_map[w[2]] for w in s] for s in test_sentences] y_test = pad_sequences(maxlen=test_max_len, sequences=y_test, padding="post", value=tag_map["O"]) input = Input(shape=(max_len,)) model = Embedding(input_dim=n_words + 1, output_dim=20, input_length=max_len)(input) model = Dropout(0.1)(model) model = Bidirectional(LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(n_tags, activation="relu"))(model) # softmax output layer crf = CRF(n_tags) # CRF layer out = crf(model) # output model = Model(input, out) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) history = model.fit(X, np.array(y), batch_size=32, epochs=5, verbose=1) model.save('simple_model.h5') return model, tag_map, X_test, y_test
def train(): input = Input(shape=(input_max_len, )) model = Embedding(vocab_size, embedding_size, weights=[glove_embedding_matrix()], input_length=input_max_len, trainable=False)(input) model = Bidirectional( LSTM(embedding_size, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True))(model) model = Bidirectional( LSTM(2 * embedding_size, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True))(model) model = TimeDistributed(Dense(embedding_size, activation='sigmoid'))(model) model = Flatten()(model) model = Dense(input_max_len, activation='sigmoid')(model) out = model model = Model(input, out) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) print(model.summary()) history = model.fit(padded_features, np.array(final_label_updated), validation_split=validation_split, epochs=epochs, batch_size=batch_size, verbose=logging, shuffle=True) model.save(model_name) metrics(history, model)
input_length=max_len)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(n_tags, activation="softmax"))( model) # softmax output layer model = Model(input, out) model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"]) history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=3, validation_split=0.1, verbose=1) hist = pd.DataFrame(history.history) plt.figure(figsize=(12, 12)) plt.plot(hist["accuracy"]) plt.plot(hist["val_accuracy"]) plt.show() i = random.randint(0, len(X_test)) p = model.predict(np.array([X_test[i]])) p = np.argmax(p, axis=-1) print("{:15} ({:5}): {}".format("Word", "True", "Pred")) for w, pred in zip(X_test[i], p[0]): print("{:15}: {}".format(words[w], tags[pred]))
model = Model(input, out) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() checkpointer = ModelCheckpoint(filepath='model.h5', verbose=0, mode='auto', save_best_only=True, monitor='val_loss') history = model.fit(X_train, np.array(y_train), batch_size=batch_size, epochs=epochs, validation_split=0.1, callbacks=[checkpointer]) history.history.keys() acc = history.history['crf_viterbi_accuracy'] val_acc = history.history['val_crf_viterbi_accuracy'] loss = history.history['loss'] val_loss = history.history['val_loss'] plt.figure(figsize=(8, 8)) epochs = range(1, len(acc) + 1) plt.plot(epochs, acc, 'bo', label='Training acc') plt.plot(epochs, val_acc, 'b', label='Validation acc') plt.title('Training and validation accuracy') plt.legend()
output_dim=32, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(model) model = Dropout(0.5)(model) model = Dense(name="output", output_dim=2, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(model) model = Model(inputs=inputs, outputs=model) model.compile(loss='mae', optimizer='adam', metrics=['mse']) model.summary() epochs = 10 callback = model.fit(x=train_x, y=train_y, epochs=epochs, validation_split=.3, batch_size=200, verbose=1).history test_y = np.rint(model.predict(x=test_x, batch_size=200, verbose=1)).astype('int') seconds = str((datetime.datetime.now() - now).seconds) print(seconds) with open('test{seconds}.txt'.format(seconds=seconds), 'w') as file: file.write('id,good,bad\n') for index, data in enumerate(test_y): file.write('{},{},{}\n'.format(index, data[0], data[1]))
model = MaxPooling1D(pool_size=2)(model) model = Conv1D(filters=32, kernel_size=2, padding='same', activation='relu')(model) model = MaxPooling1D(pool_size=2)(model) model = GRU(128)(model) model = Dense(64, activation="relu")(model) model = Dense(32, activation="relu")(model) model = Dense(16, activation="relu")(model) model = Dense(6, activation="sigmoid")(model) model = Model(inputs=inp, outputs=model) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) filepath = "weight-improvement-{epoch:02d}-{loss:4f}.hd5" checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) early = EarlyStopping(monitor="val_loss", mode="min", patience=2) callbacks_list = [checkpoint, early] model.fit(data, targets, batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=1, validation_split=VALIDATION_SPLIT, callbacks=callbacks_list)
def train(word2Vec, train_df, test_df, max_length, filters, kernel_size, pool_size, dense): r = random.randint(1, 10000) now = datetime.datetime.now() tokenizer = Tokenizer() tokenizer.fit_on_texts(word2Vec.wv.vocab.keys()) train_x = tokenizer.texts_to_sequences(train_df['text']) train_x = pad_sequences(train_x, maxlen=max_length) # train_y_good = train_df["good"] # train_y_bad = train_df["bad"] train_y = pd.DataFrame(train_df, columns=["good", "bad"]) test_x = tokenizer.texts_to_sequences(test_df['text']) test_x = pad_sequences(test_x, maxlen=max_length) word_index = tokenizer.word_index embedding_matrix = np.zeros((len(word_index) + 1, word2Vec.vector_size)) for word, i in word_index.items(): try: embedding_matrix[i] = word2Vec[word] except: continue inputs = Input(shape=(max_length, )) model = Embedding(name="embedding", input_dim=len(word_index) + 1, output_dim=word2Vec.vector_size, weights=[embedding_matrix], input_length=max_length)(inputs) model = Conv1D(name="conv1D", filters=filters, kernel_size=kernel_size, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same')(model) model = MaxPooling1D(name="maxPooling1D", pool_size=pool_size, strides=1, padding='same')(model) model = Flatten(name="flatten")(model) model = Dense(name="dense", output_dim=dense, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(model) model = Dense(name="output", output_dim=2, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(model) model = Model(inputs=inputs, outputs=model) model.compile(loss='mae', optimizer=optimizers.Adam(lr=.001), metrics=['mse']) model.summary() epochs = 15 callback = model.fit(x=train_x, y=train_y, epochs=epochs, validation_split=.3, batch_size=20, verbose=1).history test_y = np.rint(model.predict(x=test_x, batch_size=10, verbose=1)).astype('int') seconds = str((datetime.datetime.now() - now).seconds) with open('test{seconds}_{r}.txt'.format(seconds=seconds, r=r), 'w') as file: file.write('id,good,bad\n') for index, data in enumerate(test_y): file.write('{},{},{}\n'.format(index, data[0], data[1])) with open('record{seconds}_{r}.log'.format(seconds=seconds, r=r), 'w') as file: file.write('result\t\n\n') file.write('\t'.join( ['index', 'loss\t\t', 'mse\t\t\t', 'val_loss\t\t', 'val_mse\t']) + '\n') for index, loss, mse, val_loss, val_mse in zip( range(1, epochs + 1), callback['loss'], callback['mean_squared_error'], callback['val_loss'], callback['val_mean_squared_error']): file.write('\t'.join([ str(index) + '\t', '{:.12f}'.format(loss), '{:.12f}'.format( mse), '{:.12f}'.format(val_loss), '{:.12f}'.format(val_mse) ]) + '\n') file.write( '\nmax_length={max_length}\nmin_count={min_count}, size=270, iter=10, sg=1, workers=10\n' .format(max_length=max_length, min_count=min_count)) file.write('inputs = Input(shape=(max_length,)\n') file.write( 'model = Embedding(name="embedding", input_dim=len(word_index)+1, output_dim=word2Vec.vector_size, weights=[embedding_matrix], input_length=max_length)(inputs)\n' ) file.write( 'model = Conv1D(name="conv1D_good", filters={filters}, kernel_size={kernel_size}, kernel_initializer=initializers.glorot_uniform(seed=1), padding="same")(model)\n' .format(filters=filters, kernel_size=kernel_size)) file.write( 'model = MaxPooling1D(name="maxPooling1D", pool_size={pool_size}, strides=1, padding="same")(model)\n' .format(pool_size=pool_size)) file.write( 'model = Dense(name="dense", output_dim={dense}, kernel_initializer=initializers.glorot_uniform(seed=1), activation="relu")(model)\n' .format(dense=dense)) file.write( 'model = Dense(name="output", output_dim=2, kernel_initializer=initializers.glorot_uniform(seed=1), activation="relu")(model)\n' ) file.write('model = Model(inputs=inputs, outputs=model)\n') file.write( 'model.compile(loss="mae", optimizer=optimizers.Adam(lr=.001), metrics=["mse"])\n' ) import matplotlib.pyplot as plt fig = plt.figure() plt.grid(True) plt.ylim(0, 40) plt.plot(callback['loss']) plt.plot(callback['mse']) plt.plot(callback['val_loss']) plt.plot(callback['val_mse']) plt.title('model loss') plt.ylabel('loss (mae)') plt.xlabel('epoch') plt.legend(['train_loss', 'train_mse', 'test_loss', 'test_mse'], loc='upper right') fig.savefig('{seconds}_{r}.png'.format(seconds=seconds, r=r), dpi=fig.dpi)
def create_model(x_train, y_train, x_test, y_test): args = parse_args() set_logger(args.log_path, args.log_level) logging.debug('Args:') logging.debug(args) lang = construct_languages(args.train) assert len(lang) == 1 lang = lang[0] game = initialize_game(train_file=lang.train, test_file=lang.test, dev_file=lang.dev, emb_file=lang.emb, budget=args.budget, max_seq_len=args.max_seq_len, max_vocab_size=args.max_vocab_size, emb_size=args.embedding_size, model_name=args.model_name) max_len = args.max_seq_len input_dim = args.max_vocab_size output_dim = args.embedding_size embedding_matrix = game.w2v logging.debug('building Keras model...') input = Input(shape=(max_len, )) model = Embedding(input_dim=input_dim, output_dim=output_dim, input_length=max_len, weights=[embedding_matrix], trainable=False)(input) model = Dropout(0.1)(model) n_units = 128 model = Bidirectional( LSTM(units=n_units, return_sequences=True, recurrent_dropout=0.1))(model) n_tags = 5 out = TimeDistributed(Dense(n_tags, activation='softmax'))(model) model = Model(input, out) logging.debug('Model type: ') logging.debug(type(model)) logging.debug('Model summary: ') logging.debug(model.summary()) rmsprop = keras.optimizers.RMSprop(lr={{choice([0.0001])}}) model.compile(optimizer=rmsprop, loss='categorical_crossentropy', metrics=['accuracy']) logging.debug('done building model...') logging.debug('starting training...') num_train_examples = len(x_train) for i in range(num_train_examples): print('i: ', i) model.fit(x_train[:i], y_train[:i], batch_size=200, epochs=20, verbose=0) logging.debug('done training...') logging.debug('starting testing...') num_samples = x_test.shape[0] logging.debug('Number of samples: {}'.format(num_samples)) max_batch_size = 4096 batch_size = min(num_samples, max_batch_size) predictions_probability = model.predict(x_test, batch_size=batch_size) predictions = numpy.argmax(predictions_probability, axis=-1) fscore = compute_fscore(Y_pred=predictions, Y_true=y_test) logging.debug('done testing...') return -fscore
# get_ipython().system_raw('./ngrok http 6016 &') # ! curl -s http://localhost:4040/api/tunnels | python3 -c \ # "import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])" # from keras.callbacks import TensorBoard # tbCallBack = TensorBoard(log_dir='./log', histogram_freq=1, # write_graph=True, # write_grads=True, # batch_size=32, # write_images=True) history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=5, validation_split=0.1, verbose=1) #, callbacks = [tbCallBack]) hist = pd.DataFrame(history.history) hist plt.style.use('ggplot') plt.figure(figsize=(12, 12)) plt.plot(hist['crf_viterbi_accuracy']) plt.plot(hist['val_crf_viterbi_accuracy']) plt.xlabel('Epochs') plt.ylabel('Accuracy') # pip install seqeval from seqeval.metrics import precision_score, recall_score, f1_score, classification_report
def bilstm_crf(train_loc, test_loc): train_pre = preprocess(train_loc) test_pre = preprocess(test_loc) cc_train = cuu(train_pre) cc_test = cuu(test_pre) words_all, tags_all = combine_all(cc_train, cc_test) n_words = len(words_all) n_tags = len(tags_all) max_len = 130 word2idx = {w: i for i, w in enumerate(words_all)} tag2idx = {t: i for i, t in enumerate(tags_all)} X = [[word2idx[w[0]] for w in s] for s in cc_train] X = pad_sequences(maxlen=max_len, sequences=X, padding="post", value=n_words - 1) X1 = [[word2idx[w[0]] for w in s] for s in cc_test] X1 = pad_sequences(maxlen=max_len, sequences=X1, padding="post", value=n_words - 1) y = [[tag2idx[w[1]] for w in s] for s in cc_train] y = pad_sequences(maxlen=max_len, sequences=y, padding="post", value=tag2idx["O"]) y1 = [[tag2idx[w[1]] for w in s] for s in cc_test] y1 = pad_sequences(maxlen=max_len, sequences=y1, padding="post", value=tag2idx["O"]) y = [to_categorical(i, num_classes=n_tags) for i in y] input = Input(shape=(max_len, )) model = Embedding(input_dim=n_words + 1, output_dim=50, input_length=max_len, mask_zero=True)(input) # 20-dim embedding model = Bidirectional( LSTM(units=250, return_sequences=True, recurrent_dropout=0.2))(model) # variational biLSTM model = TimeDistributed(Dense(50, activation="relu"))( model) # a dense layer as suggested by neuralNer crf = CRF(n_tags) # CRF layer out = crf(model) # output model = Model(input, out) model.compile(optimizer="adam", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() history = model.fit(X, np.array(y), batch_size=4, epochs=15, verbose=1) test_pred = model.predict(X, verbose=1) idx2tag = {i: w for w, i in tag2idx.items()} pred_labels = pred2label(test_pred, idx2tag) true_labels = pred2label(y, idx2tag) f1_train = f1_score(true_labels, pred_labels) precision_train = precision_score(true_labels, pred_labels) recall_train = recall_score(true_labels, pred_labels) train_scores = [f1_train, precision_train, recall_train] y1 = [to_categorical(i, num_classes=n_tags) for i in y1] test_pred1 = model.predict(X1, verbose=1) pred_labels1 = pred2label(test_pred1, idx2tag) true_labels1 = pred2label(y1, idx2tag) f1_test = f1_score(true_labels1, pred_labels1) precision_test = precision_score(true_labels1, pred_labels1) recall_test = recall_score(true_labels1, pred_labels1) test_scores = [f1_test, precision_test, recall_test] print('Testing scores:', test_scores) return test_scores
model = Bidirectional( LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(50, activation="relu"))(model) out = Dense(6, activation='softmax')(model) #crf = CRF(n_tags+1) #out = crf(model) model = Model(input, out) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() history = model.fit(X_train, numpy.array(y_train), batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.2, verbose=2) #history = model.fit(X_train, numpy.array(y_train), batch_size = BATCH_SIZE, epochs = EPOCHS, validation_split = 0.2, verbose = 2) pred_cat = model.predict(X_test) pred = numpy.argmax(pred_cat, axis=-1) y_test_true = numpy.argmax(y_test, -1) from sklearn_crfsuite.metrics import flat_classification_report pred_tag = [[idx2tag[i] for i in row] for row in pred] y_test_true_tag = [[idx2tag[i] for i in row] for row in y_test_true] #from sklearn.metrics import f1_score #report = f1_score(y_test, pred_cat) report = flat_classification_report(y_pred=pred_tag, y_true=y_test_true_tag) print(report)
max_len = 80 EMBEDDING = 40 input_text = Input(shape=(max_len,)) model = Embedding(input_dim=n_words+2, output_dim=EMBEDDING, # n_words + 2 (PAD & UNK) input_length=max_len, mask_zero=True)(input_text) model = Bidirectional(LSTM(units=80, return_sequences=True, recurrent_dropout=0.1))(model) model = Bidirectional(LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(50, activation="relu"))(model) crf = CRF(n_tags+1) # CRF layer, n_tags+1(PAD) out = crf(model) model = Model(input_text, out) model.compile(optimizer="adam", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() history = model.fit(X_tr, np.array(y_tr), batch_size=batch_size, epochs=epochs,validation_split=0.1, verbose=2) pred_cat = model.predict(X_te) pred = np.argmax(pred_cat, axis=-1) y_te_true = np.argmax(y_te, -1) from sklearn_crfsuite.metrics import flat_classification_report # Convert the index to tag pred_tag = [[idx2tag[i] for i in row] for row in pred] y_te_true_tag = [[idx2tag[i] for i in row] for row in y_te_true] report = flat_classification_report(y_pred=pred_tag, y_true=y_te_true_tag) print(report)
output = Dense(len(classes) - (2 if arguments.cues else 0), activation = "softmax")(model); model = Model([forms_input, cues_input] if arguments.cues else forms_input, output); optimizer = optimizers.Adam(lr = arguments.lr); model.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics = ["accuracy"]); print(model.summary()); monitor = EarlyStopping(monitor = "val_acc", min_delta = 0.0001, patience = 3, verbose = 1, mode = "max"); board = TensorBoard(log_dir = "log/{}".format(arguments.id)); model.fit(([inputs, cues] if arguments.cues else inputs), outputs, validation_split = arguments.vs, batch_size = arguments.bs, epochs = arguments.epochs, callbacks = [monitor, board], verbose = 1); if arguments.debug: print("model.evaluate() on training: {}" "".format(model.evaluate(([inputs, cues] if arguments.cues else inputs), outputs, verbose = 1))); model.save(arguments.id + ".h5"); # # in a few, rare circumstances, we allow ourselves to re-interpret variable # names, as is the case of .inputs. and .outputs. here: now turning our focus # to the evaluation data. #
LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) # variational biLSTM model = TimeDistributed(Dense(50, activation="relu"))( model) # a dense layer as suggested by neuralNer crf = CRF(n_tags + 1) # CRF layer, n_tags+1(PAD) out = crf(model) # output model = Model(input, out) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() history = model.fit(X_tr, np.array(y_tr), batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.1, verbose=2) # Eval pred_cat = model.predict(X_te) pred = np.argmax(pred_cat, axis=-1) y_te_true = np.argmax(y_te, -1) # Convert the index to tag pred_tag = [[idx2tag[i] for i in row] for row in pred] y_te_true_tag = [[idx2tag[i] for i in row] for row in y_te_true] report = flat_classification_report(y_pred=pred_tag, y_true=y_te_true_tag) print(report)
def hyperopt_train_test(params): epsilon = 10**params['epsilon_exp'] optimizer = optimizers.adam(lr=params['learning_rate'], epsilon=epsilon) if dmc_parameters["use_embedding_layer"]: input = Input(shape=(dmc_parameters["max_seq_len"], )) model = Embedding(input_dim=dmc_parameters["one_hot_vector_len"], output_dim=params['embedding_layer_output'], input_length=dmc_parameters["max_seq_len"])(input) model = Dropout(rate=params['embedding_dropout'])(model) else: input = Input(shape=(dmc_parameters["max_seq_len"], dmc_parameters["one_hot_vector_len"])) model = input if params['bi_lstm1_units'] > 0: model = Bidirectional( CuDNNLSTM(units=params['bi_lstm1_units'], return_sequences=True))(model) if params['bi_lstm2_units'] > 0: model = Bidirectional( CuDNNLSTM(units=params['bi_lstm2_units'], return_sequences=True))(model) if dmc_parameters["use_crf_layer"]: crf = CRF(dmc_parameters["num_tags"]) # CRF layer out = crf(model) # output model = Model(input, out) model.compile(optimizer=optimizer, loss=losses.crf_loss, metrics=[metrics.crf_accuracy, avg_proximity_metric()]) else: out = TimeDistributed( Dense(dmc_parameters["num_tags"], activation="softmax"))(model) model = Model(input, out) model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy", avg_proximity_metric()]) model.summary() es = EarlyStopping(monitor='val_loss', min_delta=0, patience=dmc_parameters["patience"], verbose=False, mode='min', restore_best_weights=True) history = model.fit(X_tr, np.array(y_tr), batch_size=dmc_parameters['batch_size'], epochs=dmc_parameters["epochs"], validation_data=(X_vl, np.array(y_vl)), verbose=False, shuffle=True, callbacks=[es]) loss, acc, prox = model.evaluate(x=X_vl, y=np.array(y_vl), batch_size=dmc_parameters['batch_size'], verbose=False) validation_labels = deepMirCut.pred2label(y_vl, dmc_parameters) validation_pred = model.predict(X_vl, verbose=False) pred_labels = deepMirCut.pred2label(validation_pred, dmc_parameters) fScore = f1_score(validation_labels, pred_labels) return loss, acc, prox, fScore
model = Model(inputs=input, outputs=[output1, output2, output3, output4]) model.compile(optimizer="Adam", loss="mean_squared_error", metrics=[metrics.mae, metrics.binary_accuracy]) model.summary() y1 = y_trainc y2 = y_trainc.reshape(1, -1).T y3 = y_trainc.reshape(1, -1).T y4 = y_trainc.reshape(-1, 1, 1) history = model.fit(Xtrc, [y4, y2, y3, y1], batch_size=1, epochs=2, validation_split=0.05, verbose=1) from sklearn.metrics import confusion_matrix from sklearn.metrics import precision_recall_fscore_support confE = [] confN = [] confT = [] confJ = [] #precal=[] yactual = [] ypred = []
model.summary() # Saving the best model only filepath = "ner-bi-lstm-td-model-{val_accuracy:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] # Fit the best model history = model.fit(X_train, np.array(y_train), batch_size=256, epochs=20, validation_split=0.1, verbose=1, callbacks=callbacks_list) # In[ ]: def plot_history(history): accuracy = history.history['accuracy'] val_accuracy = history.history['val_accuracy'] loss = history.history['loss'] val_loss = history.history['val_loss'] x = range(1, len(accuracy) + 1) plt.figure(figsize=(12, 5)) plt.subplot(1, 2, 1)
model = Bidirectional( LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(50, activation="relu"))(model) crf = CRF_2nd(no_tags) out_layer = crf(model) model = Model(input, out_layer) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() history = model.fit(train_X, np.array(train_y), batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.1, verbose=2) pred_y = model.predict(test_X) pred_y = np.argmax(pred_y, axis=-1) y_test_true = np.argmax(test_y, -1) y_test_true = [[index_to_tag[i] for i in row] for row in y_test_true] y_test_true = [[x for x in row if x != 'PADword'] for row in y_test_true] pred_y = [[index_to_tag[i] for i in row] for row in pred_y] pred_y = [[x.replace("PADword", "O") for x in pred_y[index]][:len(y_test_true[index])] for index in range(len(y_test_true))] print('LSTM Classification Report\n',