def train_model(training_data, validating_data, batch_size, max_pad): head = '%(asctime)-15s %(message)s' logging.basicConfig(level=logging.DEBUG, format=head) segmenter = CharacterSegmenter() train_corpus = SequencePairCorpus(source_with_unk=True, same_length=True) train_corpus.build(codecs.open(training_data, 'r', encoding="utf8"), segmenter, segmenter) logging.debug("Train corpus built : " + str(train_corpus.corpus_size())) unlabeled_tag_id = train_corpus.target_corpus.id("U") val_corpus = train_corpus.make( codecs.open(validating_data, 'r', encoding="utf8"), segmenter, segmenter) logging.debug("Validate corpus built") learning_param = LearnParam(num_epoch=25, learning_rate=0.05, momentum=0.0, batch_size=batch_size, max_pad=max_pad, device=None, nworker=None) lm = SequenceTaggingMachine(unlabeled_tag_id) logging.log(logging.INFO, "Begin to train ...") lm.train(train_corpus, val_corpus, learning_param)
def train_model(training_data, validating_data, batch_size, max_pad, dev, nworker): head = '%(asctime)-15s %(message)s' logging.basicConfig(level=logging.DEBUG, format=head) segmenter = CharacterSegmenter() corpus = SequencePairCorpus(source_with_unk=True, same_length=True) corpus.build(codecs.open(training_data, 'r', encoding = "utf8"), segmenter, segmenter) unlabeled_tag_id = corpus.target_corpus.id("U") problem = SequenceTaggingProblem(corpus) data_train = BucketIter(problem, batch_size, max_pad_num = max_pad) val_corpus = corpus.make(codecs.open(validating_data, 'r', encoding = "utf8"), segmenter, segmenter) val_problem = SequenceTaggingProblem(val_corpus) data_val = BucketIter(val_problem, batch_size, max_pad_num = max_pad) arch_param = ArchParam( num_hidden= 200, num_embed= 200, num_lstm_layer= 2, input_cell_num = corpus.source_cell_num(), output_cell_num= corpus.target_cell_num() ) learning_param = LearnParam( num_epoch=25,learning_rate=0.05, momentum=0.0, batch_size = batch_size, max_pad = max_pad, device=dev, nworker = nworker ) lm = PartialLabeledSenquenceTaggingModel(arch_param, unlabeled_tag_id) #lm.show_shape_info(data_train) logging.debug("O = {0}, S = {1}, B = {2}, I = {3}, E = {4} U = {5}".format( corpus.target_corpus.id("O"), corpus.target_corpus.id("S"), corpus.target_corpus.id("B"), corpus.target_corpus.id("I"), corpus.target_corpus.id("E"), corpus.target_corpus.id("U"))) logging.log(logging.INFO, "Begin to train ...") lm.train(data_train, data_val, learning_param)