def main(args): print('Loading dataset...') x_train, y_train = load_data_and_labels(args.train_data) x_valid, y_valid = load_data_and_labels(args.valid_data) x_test, y_test = load_data_and_labels(args.test_data) x_train = np.r_[x_train, x_valid] y_train = np.r_[y_train, y_valid] print('Transforming datasets...') p = ELMoTransformer() p.fit(x_train, y_train) print('Loading word embeddings...') embeddings = load_glove(EMBEDDING_PATH) embeddings = filter_embeddings(embeddings, p._word_vocab.vocab, 100) print('Building a model.') model = ELModel(char_embedding_dim=args.char_emb_size, word_embedding_dim=args.word_emb_size, char_lstm_size=args.char_lstm_units, word_lstm_size=args.word_lstm_units, char_vocab_size=p.char_vocab_size, word_vocab_size=p.word_vocab_size, num_labels=p.label_size, embeddings=embeddings, dropout=args.dropout) model, loss = model.build() model.compile(loss=loss, optimizer='adam') print('Training the model...') trainer = Trainer(model, preprocessor=p) trainer.train(x_train, y_train, x_test, y_test) print('Saving the model...') model.save(args.weights_file, args.params_file)
def main(args): print('Loading datasets...') X, y = load_data_and_labels(args.data_path) x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1, random_state=42) embeddings = KeyedVectors.load(args.embedding_path).wv print('Transforming datasets...') p = IndexTransformer() p.fit(X, y) embeddings = filter_embeddings(embeddings, p._word_vocab, embeddings.vector_size) print('Building a model...') model = BiLSTMCRF(char_vocab_size=p.char_vocab_size, word_vocab_size=p.word_vocab_size, num_labels=p.label_size, embeddings=embeddings, char_embedding_dim=50) model.build() print('Training the model...') trainer = Trainer(model, preprocessor=p) trainer.train(x_train, y_train, x_valid, y_valid) print('Saving the model...') model.save(args.weights_file, args.params_file) p.save(args.preprocessor_file)
def fit(self, x_train, y_train, x_valid=None, y_valid=None, epochs=1, batch_size=32, verbose=1, callbacks=None, shuffle=True): """Fit the model for a fixed number of epochs. Args: x_train: list of training data. y_train: list of training target (label) data. x_valid: list of validation data. y_valid: list of validation target (label) data. batch_size: Integer. Number of samples per gradient update. If unspecified, `batch_size` will default to 32. epochs: Integer. Number of epochs to train the model. verbose: Integer. 0, 1, or 2. Verbosity mode. 0 = silent, 1 = progress bar, 2 = one line per epoch. callbacks: List of `keras.callbacks.Callback` instances. List of callbacks to apply during training. shuffle: Boolean (whether to shuffle the training data before each epoch). `shuffle` will default to True. """ p = IndexTransformer(initial_vocab=self.initial_vocab, use_char=self.use_char) p.fit(x_train, y_train) embeddings = filter_embeddings(self.embeddings, p._word_vocab.vocab, self.word_embedding_dim) model = BiLSTMCRF(char_vocab_size=p.char_vocab_size, word_vocab_size=p.word_vocab_size, num_labels=p.label_size, word_embedding_dim=self.word_embedding_dim, char_embedding_dim=self.char_embedding_dim, word_lstm_size=self.word_lstm_size, char_lstm_size=self.char_lstm_size, fc_dim=self.fc_dim, dropout=self.dropout, embeddings=embeddings, use_char=self.use_char, use_crf=self.use_crf) model, loss = model.build() model.compile(loss=loss, optimizer=self.optimizer) trainer = Trainer(model, preprocessor=p) trainer.train(x_train, y_train, x_valid, y_valid, epochs=epochs, batch_size=batch_size, verbose=verbose, callbacks=callbacks, shuffle=shuffle) self.p = p self.model = model
def training(train, test): x_train = [x.split() for x in train['sentence'].tolist()] y_train = train['tag'].tolist() x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, train_size=0.8, random_state=233) print('Transforming datasets...') p = IndexTransformer(use_char=True) p.fit(x_train, y_train) embeddings = load_glove(config.glove_file) embeddings = filter_embeddings(embeddings, p._word_vocab.vocab, config.glove_size) model = BiLSTMCRF(char_vocab_size=p.char_vocab_size, word_vocab_size=p.word_vocab_size, num_labels=p.label_size, word_embedding_dim=300, char_embedding_dim=100, word_lstm_size=100, char_lstm_size=50, fc_dim=100, dropout=0.5, embeddings=embeddings, use_char=True, use_crf=True) opt = Adam(lr=0.001) model, loss = model.build() model.compile(loss=loss, optimizer=opt, metrics=[crf_viterbi_accuracy]) filepath = '../models/' + 'best_model' ckp = ModelCheckpoint(filepath + '.h5', monitor='val_crf_viterbi_accuracy', verbose=1, save_best_only=True, mode='max', save_weights_only=True) es = EarlyStopping(monitor='val_crf_viterbi_accuracy', min_delta=0.00001, patience=3, verbose=1, mode='max') rlr = ReduceLROnPlateau(monitor='val_crf_viterbi_accuracy', factor=0.2, patience=2, verbose=1, mode='max', min_delta=0.0001) callbacks = [ckp, es, rlr] train_seq = NERSequence(x_train, y_train, config.batch_size, p.transform) if x_val and y_val: valid_seq = NERSequence(x_val, y_val, config.batch_size, p.transform) f1 = F1score(valid_seq, preprocessor=p) callbacks.append(f1) model.fit_generator(generator=train_seq, validation_data=valid_seq, epochs=config.nepochs, callbacks=callbacks, verbose=True, shuffle=True, use_multiprocessing=True, workers=42)
print("got ", len(x_train), " entries for training and ", len(x_valid), " entries for testing") entities = set() for s in y_train: for w in s: entities.add(w) print("Defined entities are :", entities) preprocessor = IndexTransformer(use_char=True) x = x_train + x_valid y = y_train + y_valid preprocessor.fit(x, y) print(len(x_train), 'train sequences') print(len(x_valid), 'valid sequences') embeddings = filter_embeddings(wv_model, preprocessor._word_vocab.vocab, wv_model.vector_size) # Use pre-trained word embeddings model = anago.models.BiLSTMCRF( embeddings=embeddings, use_crf=False, use_char=True, num_labels=preprocessor.label_size, word_vocab_size=preprocessor.word_vocab_size, char_vocab_size=preprocessor.char_vocab_size, dropout=.5, word_lstm_size=120) model.build() model.compile(loss=model.get_loss(), optimizer='adam', metrics=["acc"]) model.summary()
def training(train, test, fold): x_train = [x.split() for x in train['sentence'].tolist()] y_train = train['tag'].tolist() x_test = [x.split() for x in test['sentence'].tolist()] print('Transforming datasets...') p = IndexTransformer(use_char=True) p.fit(x_train + x_test, y_train) skf = KFold(n_splits=config.nfolds, random_state=config.seed, shuffle=True) embeddings = load_glove(config.glove_file) # embeddings_fast = load_glove(config.glove_file) embeddings_wang = load_glove(config.wang_file) embeddings = filter_embeddings(embeddings, p._word_vocab.vocab, config.glove_size) # embeddings_fast = filter_embeddings(embeddings_fast, p._word_vocab.vocab, config.fasttext_size) embeddings_wang = filter_embeddings(embeddings_wang, p._word_vocab.vocab, config.wang_size) embeddings = np.concatenate((embeddings, embeddings_wang), axis=1) for n_fold, (train_indices, val_indices) in enumerate(skf.split(x_train)): if n_fold >= fold: print("Training fold: ", n_fold) x_val = list(np.array(x_train)[val_indices]) y_val = list(np.array(y_train)[val_indices]) x_train_spl = list(np.array(x_train)[train_indices]) y_train_spl = list(np.array(y_train)[train_indices]) model = BiLSTMCRF(char_vocab_size=p.char_vocab_size, word_vocab_size=p.word_vocab_size, num_labels=p.label_size, word_embedding_dim=1200, char_embedding_dim=50, word_lstm_size=300, char_lstm_size=300, fc_dim=50, dropout=0.5, embeddings=embeddings, use_char=True, use_crf=True) opt = Adam(lr=0.001) model, loss = model.build() model.compile(loss=loss, optimizer=opt, metrics=[crf_viterbi_accuracy]) es = EarlyStopping(monitor='val_crf_viterbi_accuracy', patience=3, verbose=1, mode='max', restore_best_weights=True) rlr = ReduceLROnPlateau(monitor='val_crf_viterbi_accuracy', factor=0.2, patience=2, verbose=1, mode='max') callbacks = [es, rlr] train_seq = NERSequence(x_train_spl, y_train_spl, config.batch_size, p.transform) if x_val and y_val: valid_seq = NERSequence(x_val, y_val, config.batch_size, p.transform) f1 = F1score(valid_seq, preprocessor=p, fold=n_fold) callbacks.append(f1) model.fit_generator(generator=train_seq, validation_data=valid_seq, epochs=config.nepochs, callbacks=callbacks, verbose=True, shuffle=True, use_multiprocessing=True, workers=12) p.save('../models/best_transform.it') model.load_weights('../models/best_model_' + str(n_fold) + '.h5') predict(model, p, x_test, n_fold)
def fit(self, X, y): """ Trains the NER model. Input is list of AnnotatedDocuments. Parameters ---------- X : list(list(str)) list of list of tokens y : list(list(str)) list of list of BIO tags Returns ------- self """ if self.embeddings is None and self.embeddings_file is None: raise ValueError( "Either embeddings or embeddings_file should be provided, exiting." ) log.info("Preprocessing dataset...") self.preprocessor_ = ELMoTransformer() self.preprocessor_.fit(X, y) if self.embeddings is None: self.embeddings = load_glove(self.embeddings_file) embeddings_dim != self.embeddings[list( self.embeddings.keys())[0]].shape[0] self.embeddings = filter_embeddings( self.embeddings, self.preprocessor_._word_vocab.vocab, embeddings_dim) log.info("Building model...") self.model_ = ELModel( char_embedding_dim=self.char_embedding_dim, word_embedding_dim=self.word_embedding_dim, char_lstm_size=self.char_lstm_size, word_lstm_size=self.word_lstm_size, char_vocab_size=self.preprocessor_.char_vocab_size, word_vocab_size=self.preprocessor_.word_vocab_size, num_labels=self.preprocessor_.label_size, embeddings=self.embeddings, dropout=self.dropout) self.model_, loss = self.model_.build() optimizer = Adam(lr=self.learning_rate) self.model_.compile(loss=loss, optimizer=optimizer) self.model_.summary() log.info('Training the model...') self.trainer_ = Trainer(self.model_, preprocessor=self.preprocessor_) x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1, random_state=42) self.trainer_.train(x_train, y_train, x_valid=x_valid, y_valid=y_valid, batch_size=self.batch_size, epochs=self.max_iter) self.tagger_ = Tagger(self.model_, preprocessor=self.preprocessor_) return self
def bertFitV2(self, x_train, y_train, x_valid=None, y_valid=None, epochs=1, batch_size=32, verbose=1, callbacks=None, shuffle=True): sess = tf.Session() bert_path = "https://tfhub.dev/google/bert_multi_cased_L-12_H-768_A-12/1" max_seq_length = self._bretMaxLen p = IndexTransformer(initial_vocab=self.initial_vocab, use_char=self.use_char) p.fit(x_train, y_train) embeddings = filter_embeddings(self.embeddings, p._word_vocab.vocab, self.word_embedding_dim) #tokenizer = create_tokenizer_from_hub_module() #print("tokenizar done") #train_examples = convert_text_to_examples(x_train, y_train) #(train_input_ids, train_input_masks, train_segment_ids, train_labels) = convert_examples_to_features(tokenizer,train_examples,max_seq_length=max_seq_length) model = ABM.BertBiLSTMCRF(num_labels=p.label_size, char_embedding_dim=self.char_embedding_dim, word_lstm_size=self.word_lstm_size, char_lstm_size=self.char_lstm_size, fc_dim=self.fc_dim, use_char=self.use_char, char_vocab_size=None, use_crf=self.use_crf, layer2Flag=self._layer2Flag, layerdropout=self._layerdropout, bretFlag=self._bretFlag, bretMaxLen=self._bretMaxLen, bert_path=self._bert_path) model, loss = model.build() # Instantiate variables ABM.initialize_vars(sess) model.compile(loss=loss, optimizer=self.optimizer) trainer = Trainer(model, preprocessor=p) trainer.train(x_train, y_train, x_valid, y_valid, epochs=epochs, batch_size=batch_size, verbose=verbose, callbacks=callbacks, shuffle=shuffle) self.p = p self.model = model
x_train, y_train = pickle.load(f) with open("data_valid.pkl", "rb") as f: x_valid, y_valid = pickle.load(f) with open("data_test.pkl", "rb") as f: x_test, y_test = pickle.load(f) x_train = np.r_[x_train, x_valid] y_train = np.r_[y_train, y_valid] print('Transforming datasets...') p = ELMoTransformer() p.fit(x_train, y_train) print('Loading word embeddings...') embeddings = load_glove(EMBEDDING_PATH) embeddings = filter_embeddings(embeddings, p._word_vocab.vocab, EMBEDDING_DIM) print('Building a model.') model = ELModel(char_embedding_dim=32, word_embedding_dim=EMBEDDING_DIM, char_lstm_size=32, word_lstm_size=EMBEDDING_DIM, char_vocab_size=p.char_vocab_size, word_vocab_size=p.word_vocab_size, num_labels=p.label_size, embeddings=embeddings) model, loss = model.build() model.compile(loss=loss, optimizer='adam') print('Training the model...') trainer = Trainer(model, preprocessor=p)