def memm_train_and_store(corpora):
"""
Trains MEMM model and stores all the data needed for using MEMM
:return: nothing, stores everything in memm_model.pkl
"""
train_texts, dev_texts, test_texts, _, _, _, _ = preprocess(corpora)
logreg, vec, idx_to_tag_dict = memm_train(train_texts, dev_texts)
dump_object_to_file((logreg, vec, idx_to_tag_dict, test_texts), memm_path)
def hmm_train_and_store(corpora):
"""
Trains HMM model and stores all the data needed for using HMM
:return: nothing, stores everything in hmm_model.pkl
"""
train_texts, dev_texts, test_texts, _, _, _, _ = preprocess(corpora)
most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1, lambda2 = \
hmm_train(train_texts, dev_texts)
dump_object_to_file((most_common_tag, possible_tags, q, e, S, total_tokens, q_bi_counts, q_uni_counts, lambda1,
lambda2, test_texts), hmm_path)
def train(trainer, model, reader):
"""
Use trainer object to train the biLSTM model
:param trainer: trainer object of the biLSTM model
:param model: biLSTM model object
:param reader: reader for the biLSTM
:return: nothing
"""
trainer.train()
predictor = SentenceTaggerPredictor(model, dataset_reader=reader)
dump_object_to_file(predictor, predictor_path)
dump_object_to_file(model, model_path)
def biLSTM_train_and_store(corpora):
"""
Trains biLSTM model and stores all the data needed for using biLSTM
:return: nothing, stores everything in bilstm_model.pkl
"""
train_texts, dev_texts, test_texts, sign_to_id, _, _, id_to_tran = preprocess(corpora)
model, vocab, train_dataset, validation_dataset, cuda_device, reader = prepare1()
trainer, model, reader, vocab = prepare2(model, vocab, train_dataset, validation_dataset, cuda_device, reader)
trainer.train()
predictor = SentenceTaggerPredictor(model, dataset_reader=reader)
dump_object_to_file((model, predictor, sign_to_id, id_to_tran, test_texts), bilstm_path)