def create_model(train=True): if train: (train_x, train_y, train_max_len, train_length), ( test_x, test_y, test_max_len, test_length), \ (vocab, maxlen, chunk_tags, embedding_weights) = process_data.load_lstm_data() else: with open('model/chars-config.pkl', 'rb') as inp: (vocab, chunk_tags, embedding_weights) = pickle.load(inp) input = Input(shape=(train_max_len, )) model = Embedding(len(vocab) + 1, EMBED_DIM, mask_zero=True)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=200, return_sequences=True, recurrent_dropout=0.1))(model) model = Dropout(0.7)(model) out = TimeDistributed(Dense(len(chunk_tags) + 1, activation="softmax"))(model) model = Model(input, out) model.summary() model.compile('adam', loss="categorical_crossentropy", metrics=["accuracy"]) if train: return model, (train_x, train_y, train_max_len), (test_x, test_y, test_max_len), (vocab, chunk_tags) else: return model, (vocab, chunk_tags)
def create_model(number_of_words_total, number_of_tags, len_max, embedding_size, lstm_units, dropout, recurrent_dropout): # Input input_layer = Input(shape=(len_max, )) # Embedding Layer model = Embedding(input_dim=number_of_words_total, output_dim=embedding_size, input_length=len_max)(input_layer) # BI-LSTM Layer model = Bidirectional( LSTM(units=lstm_units, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout, kernel_initializer=keras.initializers.he_normal()))(model) # TimeDistributed layer model = TimeDistributed(Dense(number_of_tags, activation="relu"))(model) # CRF Layer crf = CRF(number_of_tags) # Output output_layer = crf(model) model = Model(input_layer, output_layer) # Optimiser adam = Adam(lr=0.0005, beta_1=0.9, beta_2=0.999) model.compile(optimizer=adam, loss=crf.loss_function, metrics=[crf.accuracy, 'accuracy']) model.summary() return model
def lstm_crf(x, y, vocab_size, n_tags, batch_size, epochs): output_dim = 30 hid_size = 50 dense1 = 50 seq_len = x.shape[1] input_ = Input(shape=(seq_len, )) model = Embedding(input_dim=vocab_size, output_dim=output_dim, input_length=seq_len, mask_zero=True)(input_) model = Bidirectional( LSTM(units=hid_size, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(dense1, activation='relu'))(model) crf = CRF(n_tags, learn_mode='marginal') out = crf(model) # prob model = Model(inputs=input_, outputs=out) model.compile(optimizer='rmsprop', loss=crf_loss, metrics=[crf.accuracy]) model.summary() history = model.fit(x, np.array(y), batch_size=batch_size, epochs=epochs, validation_split=0.1, verbose=1) return model, history
def train(): input = Input(shape=(max_len, )) model = Embedding(input_dim=n_words + 1, output_dim=100, input_length=max_len, mask_zero=True)(input) # 20-dim embedding model = Bidirectional( LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) # variational biLSTM model = TimeDistributed(Dense(50, activation="relu"))( model) # a dense layer as suggested by neuralNer crf = CRF(n_tags) # CRF layer out = crf(model) # output model = Model(input, out) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() history = model.fit(x_train, np.array(y_train), batch_size=64, epochs=5, validation_split=0.1, verbose=1) save_load_utils.save_all_weights(model, filepath="models/bilstm-crf.h5") hist = pd.DataFrame(history.history) print(hist) plt.figure(figsize=(12, 12)) plt.plot(hist["crf_viterbi_accuracy"]) plt.plot(hist["val_crf_viterbi_accuracy"]) plt.show()
def train(): input = Input(shape=(max_len, )) model = Embedding(input_dim=n_words, output_dim=50, input_length=max_len)(input) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(n_tags, activation='softmax'))( model) # softmax output layer model = Model(input, out) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() # checkpoint # filepath = "../result/bilstm-weights-{epoch:02d}-{val_acc:.2f}.hdf5" # checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max') # history=model.fit(X_train,np.array(y_train),batch_size=32,epochs=5,validation_split=0.1,verbose=1,callbacks=[checkpoint]) history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=5, validation_split=0.1, verbose=1) # 保存模型 model.save(filepath="../result/bi-lstm.h5") hist = pd.DataFrame(history.history) plt.figure(figsize=(12, 12)) plt.plot(hist["acc"]) plt.plot(hist["val_acc"]) plt.show()
def define_model(self): """Define the Bi-LSTM/CRF model""" input_layer = Input(shape=(self.MAX_SEQ_LEN, )) model = Embedding( input_dim=self.n_words, output_dim=self.EMBEDDING, # actual n_words + 2 (PAD & UNK) input_length=self.MAX_SEQ_LEN)( input_layer) # default: 300-dim embedding model = Bidirectional( LSTM(units=self.LSTM_HIDDEN_UNITS, return_sequences=True, recurrent_dropout=0.1))(model) # variational biLSTM model = TimeDistributed(Dense(self.LSTM_DENSE_DIM, activation="relu"))( model) # a dense layer as suggested by neuralNer crf = CRF(self.n_tags) # CRF layer, actual n_tags+1(PAD) output_layer = crf(model) # output model = Model(input_layer, output_layer) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() return model
def get_model(): """ Definition du model cnn utilise """ inp = Input(shape=(MAX_TEXT_LENGTH,)) model = Embedding(MAX_TEXT_LENGTH, EMBED_SIZE)(inp) model = Conv1D(filters=64, kernel_size=7, padding='same', activation='relu')(model) model = MaxPooling1D(pool_size=3)(model) model = BatchNormalization(axis=1)(model) model = Dropout(0.25)(model) model = Conv1D(filters=128, kernel_size=5, padding='same', activation='relu')(model) model = MaxPooling1D(pool_size=5)(model) model = BatchNormalization(axis=1)(model) model = Dropout(0.3)(model) model = Flatten()(model) model = Dense(1024, activation="relu")(model) model = Dense(10, activation="softmax")(model) model = Model(inputs=inp, outputs=model) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) model.summary() return model
def get_model(units, dout, emb, mtl, n_out): """ Build Model :param units: cells number of conv1D :param dout: rate for dropout :param emb: output size of embedding layer :param mlt: input size :param n_out: number of classes to dense layer :return: model """ inp = Input(shape=(mtl, )) model = Embedding(mtl, emb)(inp) model = Dropout(dout)(model) model = Conv1D(filters=units, kernel_size=emb, padding='same', activation='relu')(model) model = MaxPooling1D(pool_size=2)(model) model = Flatten()(model) model = Dense(n_out, activation="softmax")(model) model = Model(inputs=inp, outputs=model) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() return model
def createArchitecture(parameters): optimizer = 0 if parameters["optimizer"] == 'rmsprop': optimizer = optimizers.rmsprop(lr=parameters["learning_rate"], epsilon=parameters["epsilon"]) elif parameters["optimizer"] == 'adam': optimizer = optimizers.adam(lr=parameters["learning_rate"], epsilon=parameters["epsilon"]) elif parameters["optimizer"] == 'nadam': optimizer = optimizers.nadam(lr=parameters["learning_rate"], epsilon=parameters["epsilon"]) elif parameters["optimizer"] == 'sgd': optimizer = optimizers.sgd(lr=parameters["learning_rate"]) #else: # optimizer = parameters["optimizer"] if parameters["use_embedding_layer"]: input = Input(shape=(parameters["max_seq_len"], )) model = Embedding(input_dim=parameters["one_hot_vector_len"], output_dim=parameters["embedding_layer_output"], input_length=parameters["max_seq_len"])(input) if parameters["embedding_dropout"] > 0: model = Dropout(rate=parameters["embedding_dropout"])(model) else: input = Input(shape=(parameters["max_seq_len"], parameters["one_hot_vector_len"])) model = input if parameters["bi_lstm1_units"] > 0: model = Bidirectional( CuDNNLSTM(units=parameters["bi_lstm1_units"], return_sequences=True))(model) if parameters["bi_lstm2_units"] > 0: model = Bidirectional( CuDNNLSTM(units=parameters["bi_lstm2_units"], return_sequences=True))(model) if parameters["bi_lstm3_units"] > 0: model = Bidirectional( CuDNNLSTM(units=parameters["bi_lstm3_units"], return_sequences=True))(model) if parameters["use_crf_layer"]: crf = CRF(parameters["num_tags"], learn_mode="marginal") out = crf(model) # output model = Model(input, out) model.compile(optimizer=optimizer, loss=losses.crf_loss, metrics=[metrics.crf_accuracy, avg_proximity_metric()]) else: out = TimeDistributed( Dense(parameters["num_tags"], activation="softmax"))(model) model = Model(input, out) model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy", avg_proximity_metric()]) model.summary() return model
def BUILD_MODEL(X,MAX,n_words,n_tags,embedding_matrix): input_word = Input(shape = (MAX,)) model=Embedding(input_dim=n_words,input_length=X.shape[1], output_dim=embedding_matrix.shape[1], weights=[embedding_matrix],trainable=False)(input_word) model=Bidirectional(LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(model) model=Bidirectional(LSTM(32, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(model) out=TimeDistributed(Dense(n_tags, activation ='softmax'))(model) model = Model(input_word, out) model.summary() return model
def BUILD_MODEL(X,MAX,n_words,n_tags,embedding_matrix): input_word = Input(shape = (MAX,)) model=Embedding(input_dim=n_words,input_length=X.shape[1], output_dim=embedding_matrix.shape[1], weights=[embedding_matrix],trainable=False)(input_word) model=Bidirectional(LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(model) model=TimeDistributed(Dense(32, activation ='relu'))(model) crf = CRF(n_tags) # CRF layer out = crf(model) # output model = Model(input_word, out) model.summary() model.compile(optimizer='adam', loss=crf.loss_function, metrics=[crf.accuracy, 'accuracy']) return model
def model_stuff(): global model word_embedding_size = 100 input = Input(shape=(maxlen,)) model = Embedding(input_dim=n_words, output_dim=word_embedding_size, weights=[glove_embedding_matrix()], input_length=maxlen, trainable=False)(input) model = Bidirectional(LSTM(units=word_embedding_size,return_sequences=True,dropout=0.5,recurrent_dropout=0.5,kernel_initializer=k.initializers.he_normal()))(model) model = Bidirectional(LSTM(units=word_embedding_size*2,return_sequences=True,dropout=0.5,recurrent_dropout=0.5,kernel_initializer=k.initializers.he_normal()))(model) model = TimeDistributed(Dense(n_tags, activation="relu"))(model) crf = CRF(n_tags) out = crf(model) model = Model(input, out) # adam = k.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999) model.compile(optimizer='adam', loss=crf.loss_function, metrics=[crf.accuracy, 'accuracy']) model.summary()
def build_keras_model(max_len, input_dim, output_dim, embedding_matrix): logging.debug('building Keras model...') input = Input(shape=(max_len, )) model = Embedding(input_dim=input_dim, output_dim=output_dim, input_length=max_len, weights=[embedding_matrix], trainable=False)(input) model = Dropout(0.1)(model) n_units = 128 # model = Dense(n_units)(model) # model = LSTM(units=n_units, return_sequences=True, recurrent_dropout=0.1)(model) # TODO the model below is a stronger bi-directional model model = Bidirectional( LSTM(units=n_units, return_sequences=True, recurrent_dropout=0.1))(model) # TODO this should be a parameter n_tags = 5 # TODO what does TimeDistributed do? out = TimeDistributed(Dense(n_tags, activation='softmax'))(model) model = Model(input, out) logging.debug('Model type: ') logging.debug(type(model)) logging.debug('Model summary: ') logging.debug(model.summary()) logging.debug('done building model...') return model
def create(self, train_file_path: str, output_summary: bool = False): if not os.path.exists(train_file_path): raise FileNotFoundError with open(train_file_path, 'r') as train_file: self.max_words_count_in_sentence = pd.read_csv(train_file).groupby( "article_id").size().max() input_layer = Input(shape=(self.max_words_count_in_sentence, )) word_embedding_size = 150 model = Embedding( input_dim=len(self.lang.vocab), output_dim=word_embedding_size, input_length=self.max_words_count_in_sentence)(input_layer) model = Bidirectional( LSTM(units=word_embedding_size, return_sequences=True, dropout=0.5, recurrent_dropout=0.5, kernel_initializer=keras.initializers.he_normal()))(model) model = LSTM(units=word_embedding_size * 2, return_sequences=True, dropout=0.5, recurrent_dropout=0.5, kernel_initializer=keras.initializers.he_normal())(model) model = TimeDistributed(Dense(len(self._tags), activation="relu"))(model) crf = CRF(len(self._tags)) model = Model(input_layer, crf(model)) model.compile(optimizer=keras.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999), loss=crf.loss_function, metrics=[crf.accuracy, 'accuracy']) if output_summary: model.summary() self._model = model
def get_model(): embed_size = 300 inp = Input(shape=(MAX_TEXT_LENGTH,)) model = Embedding(MAX_FEATURES, embed_size)(inp) model = Dropout(0.2)(model) model = Conv1D(filters=32, kernel_size=2, padding='same', activation='relu')(model) model = MaxPooling1D(pool_size=2)(model) model = Conv1D(filters=32, kernel_size=2, padding='same', activation='relu')(model) model = MaxPooling1D(pool_size=2)(model) model = GRU(128)(model) model = Dense(64, activation="relu")(model) model = Dense(32, activation="relu")(model) model = Dense(16, activation="relu")(model) model = Dense(6, activation="sigmoid")(model) model = Model(inputs=inp, outputs=model) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() return model
def get_model(): """ Create the model for a CNN and Embedding. """ inputs = Input(shape=(MAX_TEXT_LENGTH, )) model = Embedding(MAX_FEATURES, EMBED_SIZE, input_length=MAX_TEXT_LENGTH)(inputs) model = Dropout(0.5)(model) model = Conv1D(NUM_FILTERS, 5, padding="same", activation="relu")(model) model = MaxPooling1D(pool_size=30)(model) model = Flatten()(model) model = Dense(N_OUT, activation="softmax")(model) model = Model(inputs=inputs, outputs=model) model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) model.summary() return model
def create_model(): BATCH_SIZE = 128 EPOCHS = 15 MAX_LEN = 40 EMBEDDING = 40 input = Input(shape=(MAX_LEN,)) model = Embedding(input_dim=n_words+2, output_dim=EMBEDDING, input_length=MAX_LEN, mask_zero=True)(input) model = Bidirectional(LSTM(units=25, return_sequences=True, recurrent_dropout=0.3))(model) model = TimeDistributed(Dense(50, activation="relu"))(model) out = TimeDistributed(Dense(n_tags+1, activation="softmax"))(model) model = Model(input, out) model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=[metrics.categorical_accuracy]) model.summary() return model
def define_model(self, embedding_dim, hidden_dim, use_saved_model_if_found, path_to_saved_model): input = Input(shape=(self.max_len,)) model = Embedding(input_dim=len(self.word2idx) + 1, output_dim=embedding_dim, weights=[self.embedding_matrix], input_length=self.max_len, mask_zero=True, trainable=False)(input) model = Bidirectional(LSTM(units=hidden_dim, return_sequences=True, recurrent_dropout=0.1))(model) # variational biLSTM model = TimeDistributed(Dense(hidden_dim, activation="relu"))(model) # a dense layer as suggested by neuralNer model = Dense(hidden_dim, activation="relu")(model) out = Dense(len(self.tag2idx), activation="softmax")(model) model = Model(input, out) model.compile(optimizer="rmsprop", loss='categorical_crossentropy', metrics=['accuracy']) model.summary() if use_saved_model_if_found: my_file = Path(path_to_saved_model) if my_file.is_file(): self.load_model(path_to_saved_model) self.model = model
def get_model_first(embedding_weights, word_index, vocab_dim, max_length, print_summary=True): inp = Input(shape=(max_length,)) model = Embedding(input_dim=len(word_index)+1, output_dim=vocab_dim, trainable=False, weights=[embedding_weights])(inp) model = LSTM(vocab_dim, return_sequences=True)(model) model = Dropout(0.2)(model) model = LSTM(vocab_dim, return_sequences=False)(model) model = Dropout(0.1)(model) model = Dense(int(vocab_dim/10), activation='relu')(model) model = Dropout(0.1)(model) model = Dense(5, activation='softmax')(model) model = Model(inputs=inp, outputs=model) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) if print_summary: model.summary() return model
def create_keras_model(n_words, n_tags, units, maxlen, dropout): input = Input(shape=(maxlen, )) model = Embedding(input_dim=n_words, output_dim=maxlen, input_length=maxlen)(input) model = Dropout(dropout)(model) model = Bidirectional( LSTM(units=units, return_sequences=True, recurrent_dropout=0.1))(model) out = TimeDistributed(Dense(n_tags, activation="softmax"))(model) model = Model(input, out) model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) model.summary() return model
def get_model(): inp = Input(shape=(MAX_TEXT_LENGTH, )) model = Embedding(MAX_TEXT_LENGTH, EMBED_SIZE)(inp) model = Dropout(0.2)(model) model = Conv1D(filters=Filters, kernel_size=kernel_sizes, padding='same', activation='relu')(model) model = MaxPooling1D(pool_size=kernel_sizes)(model) model = Flatten()(model) model = Dense(10, activation="softmax")(model) model = Model(inputs=inp, outputs=model) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() return model
def __init__(self, model_pars=None, data_pars=None, compute_pars=None, **kwargs): ### Model Structure ################################ if model_pars is None: self.model = None else: data_set, internal_states = get_dataset(data_pars) X_train, X_test, y_train, y_test = data_set words = internal_states.get("word_count") max_len = X_train.shape[1] num_tags = y_train.shape[2] # Model architecture input = Input(shape=(max_len, )) model = Embedding( input_dim=words, output_dim=model_pars["embedding"], input_length=max_len, )(input) model = Bidirectional( LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) model = TimeDistributed(Dense(50, activation="relu"))(model) crf = CRF(num_tags) # CRF layer out = crf(model) # output model = KModel(input, out) model.compile( optimizer=model_pars["optimizer"], loss=crf.loss_function, metrics=[crf.accuracy], ) model.summary() self.model = model
def define_model(): input = Input(shape=(max_wrd_len, )) mask = Masking(mask_value=0)(input) model = Embedding(input_dim=chr_vocab_size, output_dim=100, input_length=max_wrd_len, mask_zero=True)(mask) model = Dropout(0.1)(model) model = Bidirectional( LSTM(units=250, return_sequences=True, recurrent_dropout=0.1))(model) model = Bidirectional(LSTM(units=250, recurrent_dropout=0.1))(model) out = Dense(num_classes, activation="softmax")(model) model = Model([input], out) #load existing weight if exist if os.path.isfile(outFileName + "-best.hdf5"): model.load_weights(outFileName + "-best.hdf5") model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) plot_model(model, show_shapes=True, to_file=outFileName + '-plot.png') return model
def train(): input = Input(shape=(input_max_len, )) model = Embedding(vocab_size, embedding_size, weights=[glove_embedding_matrix()], input_length=input_max_len, trainable=False)(input) model = Bidirectional( LSTM(embedding_size, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True))(model) model = Bidirectional( LSTM(2 * embedding_size, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True))(model) model = TimeDistributed(Dense(embedding_size, activation='sigmoid'))(model) model = Flatten()(model) model = Dense(input_max_len, activation='sigmoid')(model) out = model model = Model(input, out) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) print(model.summary()) history = model.fit(padded_features, np.array(final_label_updated), validation_split=validation_split, epochs=epochs, batch_size=batch_size, verbose=logging, shuffle=True) model.save(model_name) metrics(history, model)
model = Model(input, out) # In[ ]: from keras.callbacks import ModelCheckpoint import matplotlib.pyplot as plt #Optimiser adam = k.optimizers.Adam(lr=0.0005, beta_1=0.9, beta_2=0.999) # Compile model model.compile(optimizer=adam, loss=crf.loss_function, metrics=[crf.accuracy, 'accuracy']) model.summary() # Saving the best model only filepath = "ner-bi-lstm-td-model-{val_accuracy:.2f}.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] # Fit the best model history = model.fit(X_train, np.array(y_train), batch_size=256, epochs=20,
def hyperopt_train_test(params): epsilon = 10**params['epsilon_exp'] optimizer = optimizers.adam(lr=params['learning_rate'], epsilon=epsilon) if dmc_parameters["use_embedding_layer"]: input = Input(shape=(dmc_parameters["max_seq_len"], )) model = Embedding(input_dim=dmc_parameters["one_hot_vector_len"], output_dim=params['embedding_layer_output'], input_length=dmc_parameters["max_seq_len"])(input) model = Dropout(rate=params['embedding_dropout'])(model) else: input = Input(shape=(dmc_parameters["max_seq_len"], dmc_parameters["one_hot_vector_len"])) model = input if params['bi_lstm1_units'] > 0: model = Bidirectional( CuDNNLSTM(units=params['bi_lstm1_units'], return_sequences=True))(model) if params['bi_lstm2_units'] > 0: model = Bidirectional( CuDNNLSTM(units=params['bi_lstm2_units'], return_sequences=True))(model) if dmc_parameters["use_crf_layer"]: crf = CRF(dmc_parameters["num_tags"]) # CRF layer out = crf(model) # output model = Model(input, out) model.compile(optimizer=optimizer, loss=losses.crf_loss, metrics=[metrics.crf_accuracy, avg_proximity_metric()]) else: out = TimeDistributed( Dense(dmc_parameters["num_tags"], activation="softmax"))(model) model = Model(input, out) model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy", avg_proximity_metric()]) model.summary() es = EarlyStopping(monitor='val_loss', min_delta=0, patience=dmc_parameters["patience"], verbose=False, mode='min', restore_best_weights=True) history = model.fit(X_tr, np.array(y_tr), batch_size=dmc_parameters['batch_size'], epochs=dmc_parameters["epochs"], validation_data=(X_vl, np.array(y_vl)), verbose=False, shuffle=True, callbacks=[es]) loss, acc, prox = model.evaluate(x=X_vl, y=np.array(y_vl), batch_size=dmc_parameters['batch_size'], verbose=False) validation_labels = deepMirCut.pred2label(y_vl, dmc_parameters) validation_pred = model.predict(X_vl, verbose=False) pred_labels = deepMirCut.pred2label(validation_pred, dmc_parameters) fScore = f1_score(validation_labels, pred_labels) return loss, acc, prox, fScore
def train(word2Vec, train_df, test_df, max_length, filters, kernel_size, pool_size, dense): r = random.randint(1, 10000) now = datetime.datetime.now() tokenizer = Tokenizer() tokenizer.fit_on_texts(word2Vec.wv.vocab.keys()) train_x = tokenizer.texts_to_sequences(train_df['text']) train_x = pad_sequences(train_x, maxlen=max_length) # train_y_good = train_df["good"] # train_y_bad = train_df["bad"] train_y = pd.DataFrame(train_df, columns=["good", "bad"]) test_x = tokenizer.texts_to_sequences(test_df['text']) test_x = pad_sequences(test_x, maxlen=max_length) word_index = tokenizer.word_index embedding_matrix = np.zeros((len(word_index) + 1, word2Vec.vector_size)) for word, i in word_index.items(): try: embedding_matrix[i] = word2Vec[word] except: continue inputs = Input(shape=(max_length, )) model = Embedding(name="embedding", input_dim=len(word_index) + 1, output_dim=word2Vec.vector_size, weights=[embedding_matrix], input_length=max_length)(inputs) model = Conv1D(name="conv1D", filters=filters, kernel_size=kernel_size, kernel_initializer=initializers.glorot_uniform(seed=1), padding='same')(model) model = MaxPooling1D(name="maxPooling1D", pool_size=pool_size, strides=1, padding='same')(model) model = Flatten(name="flatten")(model) model = Dense(name="dense", output_dim=dense, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(model) model = Dense(name="output", output_dim=2, kernel_initializer=initializers.glorot_uniform(seed=1), activation='relu')(model) model = Model(inputs=inputs, outputs=model) model.compile(loss='mae', optimizer=optimizers.Adam(lr=.001), metrics=['mse']) model.summary() epochs = 15 callback = model.fit(x=train_x, y=train_y, epochs=epochs, validation_split=.3, batch_size=20, verbose=1).history test_y = np.rint(model.predict(x=test_x, batch_size=10, verbose=1)).astype('int') seconds = str((datetime.datetime.now() - now).seconds) with open('test{seconds}_{r}.txt'.format(seconds=seconds, r=r), 'w') as file: file.write('id,good,bad\n') for index, data in enumerate(test_y): file.write('{},{},{}\n'.format(index, data[0], data[1])) with open('record{seconds}_{r}.log'.format(seconds=seconds, r=r), 'w') as file: file.write('result\t\n\n') file.write('\t'.join( ['index', 'loss\t\t', 'mse\t\t\t', 'val_loss\t\t', 'val_mse\t']) + '\n') for index, loss, mse, val_loss, val_mse in zip( range(1, epochs + 1), callback['loss'], callback['mean_squared_error'], callback['val_loss'], callback['val_mean_squared_error']): file.write('\t'.join([ str(index) + '\t', '{:.12f}'.format(loss), '{:.12f}'.format( mse), '{:.12f}'.format(val_loss), '{:.12f}'.format(val_mse) ]) + '\n') file.write( '\nmax_length={max_length}\nmin_count={min_count}, size=270, iter=10, sg=1, workers=10\n' .format(max_length=max_length, min_count=min_count)) file.write('inputs = Input(shape=(max_length,)\n') file.write( 'model = Embedding(name="embedding", input_dim=len(word_index)+1, output_dim=word2Vec.vector_size, weights=[embedding_matrix], input_length=max_length)(inputs)\n' ) file.write( 'model = Conv1D(name="conv1D_good", filters={filters}, kernel_size={kernel_size}, kernel_initializer=initializers.glorot_uniform(seed=1), padding="same")(model)\n' .format(filters=filters, kernel_size=kernel_size)) file.write( 'model = MaxPooling1D(name="maxPooling1D", pool_size={pool_size}, strides=1, padding="same")(model)\n' .format(pool_size=pool_size)) file.write( 'model = Dense(name="dense", output_dim={dense}, kernel_initializer=initializers.glorot_uniform(seed=1), activation="relu")(model)\n' .format(dense=dense)) file.write( 'model = Dense(name="output", output_dim=2, kernel_initializer=initializers.glorot_uniform(seed=1), activation="relu")(model)\n' ) file.write('model = Model(inputs=inputs, outputs=model)\n') file.write( 'model.compile(loss="mae", optimizer=optimizers.Adam(lr=.001), metrics=["mse"])\n' ) import matplotlib.pyplot as plt fig = plt.figure() plt.grid(True) plt.ylim(0, 40) plt.plot(callback['loss']) plt.plot(callback['mse']) plt.plot(callback['val_loss']) plt.plot(callback['val_mse']) plt.title('model loss') plt.ylabel('loss (mae)') plt.xlabel('epoch') plt.legend(['train_loss', 'train_mse', 'test_loss', 'test_mse'], loc='upper right') fig.savefig('{seconds}_{r}.png'.format(seconds=seconds, r=r), dpi=fig.dpi)
def create_model(x_train, y_train, x_test, y_test): args = parse_args() set_logger(args.log_path, args.log_level) logging.debug('Args:') logging.debug(args) lang = construct_languages(args.train) assert len(lang) == 1 lang = lang[0] game = initialize_game(train_file=lang.train, test_file=lang.test, dev_file=lang.dev, emb_file=lang.emb, budget=args.budget, max_seq_len=args.max_seq_len, max_vocab_size=args.max_vocab_size, emb_size=args.embedding_size, model_name=args.model_name) max_len = args.max_seq_len input_dim = args.max_vocab_size output_dim = args.embedding_size embedding_matrix = game.w2v logging.debug('building Keras model...') input = Input(shape=(max_len, )) model = Embedding(input_dim=input_dim, output_dim=output_dim, input_length=max_len, weights=[embedding_matrix], trainable=False)(input) model = Dropout(0.1)(model) n_units = 128 model = Bidirectional( LSTM(units=n_units, return_sequences=True, recurrent_dropout=0.1))(model) n_tags = 5 out = TimeDistributed(Dense(n_tags, activation='softmax'))(model) model = Model(input, out) logging.debug('Model type: ') logging.debug(type(model)) logging.debug('Model summary: ') logging.debug(model.summary()) rmsprop = keras.optimizers.RMSprop(lr={{choice([0.0001])}}) model.compile(optimizer=rmsprop, loss='categorical_crossentropy', metrics=['accuracy']) logging.debug('done building model...') logging.debug('starting training...') num_train_examples = len(x_train) for i in range(num_train_examples): print('i: ', i) model.fit(x_train[:i], y_train[:i], batch_size=200, epochs=20, verbose=0) logging.debug('done training...') logging.debug('starting testing...') num_samples = x_test.shape[0] logging.debug('Number of samples: {}'.format(num_samples)) max_batch_size = 4096 batch_size = min(num_samples, max_batch_size) predictions_probability = model.predict(x_test, batch_size=batch_size) predictions = numpy.argmax(predictions_probability, axis=-1) fscore = compute_fscore(Y_pred=predictions, Y_true=y_test) logging.debug('done testing...') return -fscore
LSTM(units=50, return_sequences=True, recurrent_dropout=0.1))(model) # variational biLSTM model = Dropout(0.1)(model) model = TimeDistributed(Dense(50, activation="relu"))( model) # a dense layer as suggested by neuralNer crf = CRF(n_tags) # CRF layer out = crf(model) # output # input.shape model = Model(input, out) model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy]) print(model.summary()) # LOG_DIR = './log' # get_ipython().system_raw( # 'tensorboard --logdir {} --host 0.0.0.0 --port 6016 &' # .format(LOG_DIR)) # get_ipython().system_raw('./ngrok http 6016 &') # ! curl -s http://localhost:4040/api/tunnels | python3 -c \ # "import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])" # from keras.callbacks import TensorBoard # tbCallBack = TensorBoard(log_dir='./log', histogram_freq=1, # write_graph=True, # write_grads=True,
def bilstm_crf(train_loc, test_loc): train_pre = preprocess(train_loc) test_pre = preprocess(test_loc) cc_train = cuu(train_pre) cc_test = cuu(test_pre) words_all, tags_all = combine_all(cc_train, cc_test) n_words = len(words_all) n_tags = len(tags_all) max_len = 130 word2idx = {w: i for i, w in enumerate(words_all)} tag2idx = {t: i for i, t in enumerate(tags_all)} X = [[word2idx[w[0]] for w in s] for s in cc_train] X = pad_sequences(maxlen=max_len, sequences=X, padding="post", value=n_words - 1) X1 = [[word2idx[w[0]] for w in s] for s in cc_test] X1 = pad_sequences(maxlen=max_len, sequences=X1, padding="post", value=n_words - 1) y = [[tag2idx[w[1]] for w in s] for s in cc_train] y = pad_sequences(maxlen=max_len, sequences=y, padding="post", value=tag2idx["O"]) y1 = [[tag2idx[w[1]] for w in s] for s in cc_test] y1 = pad_sequences(maxlen=max_len, sequences=y1, padding="post", value=tag2idx["O"]) y = [to_categorical(i, num_classes=n_tags) for i in y] input = Input(shape=(max_len, )) model = Embedding(input_dim=n_words + 1, output_dim=50, input_length=max_len, mask_zero=True)(input) # 20-dim embedding model = Bidirectional( LSTM(units=250, return_sequences=True, recurrent_dropout=0.2))(model) # variational biLSTM model = TimeDistributed(Dense(50, activation="relu"))( model) # a dense layer as suggested by neuralNer crf = CRF(n_tags) # CRF layer out = crf(model) # output model = Model(input, out) model.compile(optimizer="adam", loss=crf.loss_function, metrics=[crf.accuracy]) model.summary() history = model.fit(X, np.array(y), batch_size=4, epochs=15, verbose=1) test_pred = model.predict(X, verbose=1) idx2tag = {i: w for w, i in tag2idx.items()} pred_labels = pred2label(test_pred, idx2tag) true_labels = pred2label(y, idx2tag) f1_train = f1_score(true_labels, pred_labels) precision_train = precision_score(true_labels, pred_labels) recall_train = recall_score(true_labels, pred_labels) train_scores = [f1_train, precision_train, recall_train] y1 = [to_categorical(i, num_classes=n_tags) for i in y1] test_pred1 = model.predict(X1, verbose=1) pred_labels1 = pred2label(test_pred1, idx2tag) true_labels1 = pred2label(y1, idx2tag) f1_test = f1_score(true_labels1, pred_labels1) precision_test = precision_score(true_labels1, pred_labels1) recall_test = recall_score(true_labels1, pred_labels1) test_scores = [f1_test, precision_test, recall_test] print('Testing scores:', test_scores) return test_scores