def memm_train_and_store(corpora): """ Trains MEMM model and stores all the data needed for using MEMM :return: nothing, stores everything in memm_model.pkl """ train_texts, dev_texts, test_texts, _, _, _, _ = preprocess(corpora) logreg, vec, idx_to_tag_dict = memm_train(train_texts, dev_texts) dump_object_to_file((logreg, vec, idx_to_tag_dict, test_texts), memm_path)
def hmm_train_and_store(corpora): """ Trains HMM model and stores all the data needed for using HMM :return: nothing, stores everything in hmm_model.pkl """ train_texts, dev_texts, test_texts, _, _, _, _ = preprocess(corpora) most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2 = \ hmm_train(train_texts, dev_texts) dump_object_to_file((most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2, test_texts), hmm_path)
def train(trainer, model, reader): """ Use trainer object to train the biLSTM model :param trainer: trainer object of the biLSTM model :param model: biLSTM model object :param reader: reader for the biLSTM :return: nothing """ trainer.train() predictor = SentenceTaggerPredictor(model, dataset_reader=reader) dump_object_to_file(predictor, predictor_path) dump_object_to_file(model, model_path)
def biLSTM_train_and_store(corpora): """ Trains biLSTM model and stores all the data needed for using biLSTM :return: nothing, stores everything in bilstm_model.pkl """ train_texts, dev_texts, test_texts, sign_to_id, _, _, id_to_tran = preprocess(corpora) model, vocab, train_dataset, validation_dataset, cuda_device, reader = prepare1() trainer, model, reader, vocab = prepare2(model, vocab, train_dataset, validation_dataset, cuda_device, reader) trainer.train() predictor = SentenceTaggerPredictor(model, dataset_reader=reader) dump_object_to_file((model, predictor, sign_to_id, id_to_tran, test_texts), bilstm_path)