def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0, gold_preproc=False, n_sents=0, corruption_level=0, beam_width=1, verbose=False, use_orig_arc_eager=False): if n_sents > 0: gold_tuples = gold_tuples[:n_sents] templates = Tagger.default_templates() nlp = Language(data_dir=model_dir, tagger=False) nlp.tagger = Tagger.blank(nlp.vocab, templates) print("Itn.\tP.Loss\tUAS\tNER F.\tTag %\tToken %") for itn in range(n_iter): scorer = Scorer() loss = 0 for raw_text, sents in gold_tuples: if gold_preproc: raw_text = None else: sents = _merge_sents(sents) for annot_tuples, ctnt in sents: words = annot_tuples[1] gold_tags = annot_tuples[2] score_model(scorer, nlp, raw_text, annot_tuples) if raw_text is None: tokens = nlp.tokenizer.tokens_from_list(words) else: tokens = nlp.tokenizer(raw_text) loss += nlp.tagger.train(tokens, gold_tags) random.shuffle(gold_tuples) print('%d:\t%d\t%.3f\t%.3f\t%.3f\t%.3f' % (itn, loss, scorer.uas, scorer.ents_f, scorer.tags_acc, scorer.token_acc)) nlp.end_training(model_dir)
def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0, gold_preproc=False, force_gold=False): dep_model_dir = path.join(model_dir, 'deps') pos_model_dir = path.join(model_dir, 'pos') if path.exists(dep_model_dir): shutil.rmtree(dep_model_dir) if path.exists(pos_model_dir): shutil.rmtree(pos_model_dir) os.mkdir(dep_model_dir) os.mkdir(pos_model_dir) Config.write(dep_model_dir, 'config', features=feat_set, seed=seed, labels=ArcEager.get_labels(gold_tuples)) nlp = Language(data_dir=model_dir, tagger=False, parser=False, entity=False) nlp.tagger = Tagger.blank(nlp.vocab, Tagger.default_templates()) nlp.parser = Parser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager) print("Itn.\tP.Loss\tUAS\tNER F.\tTag %\tToken %") for itn in range(n_iter): scorer = Scorer() loss = 0 for _, sents in gold_tuples: for annot_tuples, _ in sents: if len(annot_tuples[1]) == 1: continue score_model(scorer, nlp, None, annot_tuples, verbose=False) tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1]) nlp.tagger(tokens) gold = GoldParse(tokens, annot_tuples, make_projective=True) if not gold.is_projective: raise Exception( "Non-projective sentence in training, after we should " "have enforced projectivity: %s" % annot_tuples) loss += nlp.parser.train(tokens, gold) nlp.tagger.train(tokens, gold.tags) random.shuffle(gold_tuples) print('%d:\t%d\t%.3f\t%.3f\t%.3f' % (itn, loss, scorer.uas, scorer.tags_acc, scorer.token_acc)) print('end training') nlp.end_training(model_dir) print('done')
def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0, gold_preproc=False, force_gold=False): dep_model_dir = path.join(model_dir, 'deps') pos_model_dir = path.join(model_dir, 'pos') if path.exists(dep_model_dir): shutil.rmtree(dep_model_dir) if path.exists(pos_model_dir): shutil.rmtree(pos_model_dir) os.mkdir(dep_model_dir) os.mkdir(pos_model_dir) Config.write(dep_model_dir, 'config', features=feat_set, seed=seed, labels=ArcEager.get_labels(gold_tuples)) nlp = Language(data_dir=model_dir, tagger=False, parser=False, entity=False) nlp.tagger = Tagger.blank(nlp.vocab, Tagger.default_templates()) nlp.parser = Parser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager) print("Itn.\tP.Loss\tUAS\tNER F.\tTag %\tToken %") for itn in range(n_iter): scorer = Scorer() loss = 0 for _, sents in gold_tuples: for annot_tuples, _ in sents: if len(annot_tuples[1]) == 1: continue score_model(scorer, nlp, None, annot_tuples, verbose=False) tokens = nlp.tokenizer.tokens_from_list(annot_tuples[1]) nlp.tagger(tokens) gold = GoldParse(tokens, annot_tuples, make_projective=True) if not gold.is_projective: raise Exception( "Non-projective sentence in training, after we should " "have enforced projectivity: %s" % annot_tuples ) loss += nlp.parser.train(tokens, gold) nlp.tagger.train(tokens, gold.tags) random.shuffle(gold_tuples) print('%d:\t%d\t%.3f\t%.3f\t%.3f' % (itn, loss, scorer.uas, scorer.tags_acc, scorer.token_acc)) print('end training') nlp.end_training(model_dir) print('done')
def main(output_dir): ensure_dir(output_dir) ensure_dir(output_dir, "pos") ensure_dir(output_dir, "vocab") vocab = Vocab(tag_map=TAG_MAP) tokenizer = Tokenizer(vocab, {}, None, None, None) # The default_templates argument is where features are specified. See # spacy/tagger.pyx for the defaults. tagger = Tagger.blank(vocab, Tagger.default_templates()) for i in range(5): for words, tags in DATA: tokens = tokenizer.tokens_from_list(words) tagger.train(tokens, tags) random.shuffle(DATA) tagger.model.end_training(path.join(output_dir, 'pos', 'model')) vocab.strings.dump(path.join(output_dir, 'vocab', 'strings.txt'))
def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0, gold_preproc=False, n_sents=0, corruption_level=0, beam_width=1, verbose=False, use_orig_arc_eager=False): dep_model_dir = path.join(model_dir, 'deps') ner_model_dir = path.join(model_dir, 'ner') pos_model_dir = path.join(model_dir, 'pos') if path.exists(dep_model_dir): shutil.rmtree(dep_model_dir) if path.exists(ner_model_dir): shutil.rmtree(ner_model_dir) if path.exists(pos_model_dir): shutil.rmtree(pos_model_dir) os.mkdir(dep_model_dir) os.mkdir(ner_model_dir) os.mkdir(pos_model_dir) Config.write(dep_model_dir, 'config', features=feat_set, seed=seed, labels=ArcEager.get_labels(gold_tuples), beam_width=beam_width) Config.write(ner_model_dir, 'config', features='ner', seed=seed, labels=BiluoPushDown.get_labels(gold_tuples), beam_width=0) if n_sents > 0: gold_tuples = gold_tuples[:n_sents] nlp = Language(data_dir=model_dir, tagger=False, parser=False, entity=False) nlp.tagger = Tagger.blank(nlp.vocab, Tagger.default_templates()) nlp.parser = Parser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager) nlp.entity = Parser.from_dir(ner_model_dir, nlp.vocab.strings, BiluoPushDown) print("Itn.\tP.Loss\tUAS\tNER F.\tTag %\tToken %") for itn in range(n_iter): scorer = Scorer() loss = 0 for raw_text, sents in gold_tuples: if gold_preproc: raw_text = None else: sents = _merge_sents(sents) for annot_tuples, ctnt in sents: if len(annot_tuples[1]) == 1: continue score_model(scorer, nlp, raw_text, annot_tuples, verbose=verbose if itn >= 2 else False) if raw_text is None: words = add_noise(annot_tuples[1], corruption_level) tokens = nlp.tokenizer.tokens_from_list(words) else: raw_text = add_noise(raw_text, corruption_level) tokens = nlp.tokenizer(raw_text) nlp.tagger(tokens) gold = GoldParse(tokens, annot_tuples, make_projective=True) if not gold.is_projective: raise Exception( "Non-projective sentence in training, after we should " "have enforced projectivity: %s" % annot_tuples ) loss += nlp.parser.train(tokens, gold) nlp.entity.train(tokens, gold) nlp.tagger.train(tokens, gold.tags) random.shuffle(gold_tuples) print('%d:\t%d\t%.3f\t%.3f\t%.3f\t%.3f' % (itn, loss, scorer.uas, scorer.ents_f, scorer.tags_acc, scorer.token_acc)) print('end training') nlp.end_training(model_dir) print('done')
def train(Language, gold_tuples, model_dir, n_iter=15, feat_set=u'basic', seed=0, gold_preproc=False, n_sents=0, corruption_level=0, beam_width=1, verbose=False, use_orig_arc_eager=False, pseudoprojective=False): dep_model_dir = path.join(model_dir, 'deps') ner_model_dir = path.join(model_dir, 'ner') pos_model_dir = path.join(model_dir, 'pos') if path.exists(dep_model_dir): shutil.rmtree(dep_model_dir) if path.exists(ner_model_dir): shutil.rmtree(ner_model_dir) if path.exists(pos_model_dir): shutil.rmtree(pos_model_dir) os.mkdir(dep_model_dir) os.mkdir(ner_model_dir) os.mkdir(pos_model_dir) if pseudoprojective: # preprocess training data here before ArcEager.get_labels() is called gold_tuples = PseudoProjectivity.preprocess_training_data(gold_tuples) Config.write(dep_model_dir, 'config', features=feat_set, seed=seed, labels=ArcEager.get_labels(gold_tuples), beam_width=beam_width,projectivize=pseudoprojective) Config.write(ner_model_dir, 'config', features='ner', seed=seed, labels=BiluoPushDown.get_labels(gold_tuples), beam_width=0) if n_sents > 0: gold_tuples = gold_tuples[:n_sents] nlp = Language(data_dir=model_dir, tagger=False, parser=False, entity=False) if nlp.lang == 'de': nlp.vocab.morphology.lemmatizer = lambda string,pos: set([string]) nlp.tagger = Tagger.blank(nlp.vocab, Tagger.default_templates()) nlp.parser = Parser.from_dir(dep_model_dir, nlp.vocab.strings, ArcEager) nlp.entity = Parser.from_dir(ner_model_dir, nlp.vocab.strings, BiluoPushDown) print("Itn.\tP.Loss\tUAS\tNER F.\tTag %\tToken %") for itn in range(n_iter): scorer = Scorer() loss = 0 for raw_text, sents in gold_tuples: if gold_preproc: raw_text = None else: sents = _merge_sents(sents) for annot_tuples, ctnt in sents: if len(annot_tuples[1]) == 1: continue score_model(scorer, nlp, raw_text, annot_tuples, verbose=verbose if itn >= 2 else False) if raw_text is None: words = add_noise(annot_tuples[1], corruption_level) tokens = nlp.tokenizer.tokens_from_list(words) else: raw_text = add_noise(raw_text, corruption_level) tokens = nlp.tokenizer(raw_text) nlp.tagger(tokens) gold = GoldParse(tokens, annot_tuples) if not gold.is_projective: raise Exception("Non-projective sentence in training: %s" % annot_tuples[1]) loss += nlp.parser.train(tokens, gold) nlp.entity.train(tokens, gold) nlp.tagger.train(tokens, gold.tags) random.shuffle(gold_tuples) print('%d:\t%d\t%.3f\t%.3f\t%.3f\t%.3f' % (itn, loss, scorer.uas, scorer.ents_f, scorer.tags_acc, scorer.token_acc)) print('end training') nlp.end_training(model_dir) print('done')