def __call__(self, args): config = get_config(args.config_path) assert config.ucca.type in ["chart", "top-down", "global-chart"] with open(os.path.join(args.save_path, "config.json"), "w", encoding="utf-8") as f: json.dump(config, f, ensure_ascii=False, default=lambda o: o.__dict__, indent=4) print("save all files to %s" % (args.save_path)) # read training , dev file print("loading datasets and transforming to trees...") train = Corpus(args.train_path) dev = Corpus(args.dev_path) print(train, "\n", dev) # init vocab print("collecting words and labels in training dataset...") vocab = Vocab(train) print(vocab) # prepare pre-trained embedding if args.emb_path: print("reading pre-trained embedding...") pre_emb = Embedding.load(args.emb_path) print( "pre-trained words:%d, dim=%d in %s" % (len(pre_emb), pre_emb.dim, args.emb_path) ) else: pre_emb = None embedding = vocab.read_embedding(config.ucca.word_dim, pre_emb) vocab_path = os.path.join(args.save_path, "vocab.pt") torch.save(vocab, vocab_path) # init parser print("initializing model...") ucca_parser = UCCA_Parser(vocab, config.ucca, pre_emb=embedding) if torch.cuda.is_available(): ucca_parser = ucca_parser.cuda() # prepare data print("preparing input data...") train_loader = Data.DataLoader( dataset=train.generate_inputs(vocab, True), batch_size=config.ucca.batch_size, shuffle=True, collate_fn=collate_fn, ) dev_loader = Data.DataLoader( dataset=dev.generate_inputs(vocab, False), batch_size=10, shuffle=False, collate_fn=collate_fn, ) optimizer = optim.Adam(ucca_parser.parameters(), lr=config.ucca.lr) ucca_evaluator = UCCA_Evaluator( parser=ucca_parser, gold_dic=args.dev_path, ) trainer = Trainer( parser=ucca_parser, optimizer=optimizer, evaluator=ucca_evaluator, batch_size=config.ucca.batch_size, epoch=config.ucca.epoch, patience=config.ucca.patience, path=args.save_path, ) trainer.train(train_loader, dev_loader) # reload parser del ucca_parser torch.cuda.empty_cache() print("reloading the best parser for testing...") vocab_path = os.path.join(args.save_path, "vocab.pt") state_path = os.path.join(args.save_path, "parser.pt") config_path = os.path.join(args.save_path, "config.json") ucca_parser = UCCA_Parser.load(vocab_path, config_path, state_path) if args.test_id_path: print("evaluating test data : %s" % (args.test_id_path)) test = Corpus(args.test_id_path) print(test) test_loader = Data.DataLoader( dataset=test.generate_inputs(vocab, False), batch_size=10, shuffle=False, collate_fn=collate_fn, ) ucca_evaluator = UCCA_Evaluator( parser=ucca_parser, gold_dic=args.test_id_path, ) ucca_evaluator.compute_accuracy(test_loader) ucca_evaluator.remove_temp() if args.test_ood_path: print("evaluating test data : %s" % (args.test_ood_path)) test = Corpus(args.test_ood_path) print(test) test_loader = Data.DataLoader( dataset=test.generate_inputs(vocab, False), batch_size=10, shuffle=False, collate_fn=collate_fn, ) ucca_evaluator = UCCA_Evaluator( parser=ucca_parser, gold_dic=args.test_ood_path, ) ucca_evaluator.compute_accuracy(test_loader) ucca_evaluator.remove_temp()
def __call__(self, args): self.args = args if not os.path.exists(args.file): os.mkdir(args.file) if not os.path.exists(args.fields) or args.preprocess: logger.info("Preprocess the data") self.WORD = Field('words', pad=pad, unk=unk, bos=bos, lower=args.lower) if args.feat == 'char': self.FEAT = SubwordField('chars', pad=pad, unk=unk, bos=bos, fix_len=args.fix_len, tokenize=list) elif args.feat == 'bert': tokenizer = SubwordField.tokenizer(args.bert_model) self.FEAT = SubwordField('bert', tokenizer=tokenizer, fix_len=args.fix_len) self.bos = self.FEAT.bos or bos if hasattr(tokenizer, 'vocab'): self.FEAT.vocab = tokenizer.vocab else: self.FEAT.vocab = FieldVocab( tokenizer.unk_token_id, { tokenizer._convert_id_to_token(i): i for i in range(len(tokenizer)) }) else: self.FEAT = Field('tags', bos=self.bos) self.ARC = Field('arcs', bos=self.bos, use_vocab=False, fn=numericalize) self.REL = Field('rels', bos=self.bos) if args.feat == 'bert': if args.n_embed: self.fields = CoNLL(FORM=(self.WORD, self.FEAT), HEAD=self.ARC, DEPREL=self.REL) self.WORD.bos = self.bos # ensure representations of the same length else: self.fields = CoNLL(FORM=self.FEAT, HEAD=self.ARC, DEPREL=self.REL) self.WORD = None elif args.feat == 'char': self.fields = CoNLL(FORM=(self.WORD, self.FEAT), HEAD=self.ARC, DEPREL=self.REL) else: self.fields = CoNLL(FORM=self.WORD, CPOS=self.FEAT, HEAD=self.ARC, DEPREL=self.REL) train = Corpus.load(args.ftrain, self.fields, args.max_sent_length) if args.fembed: embed = Embedding.load(args.fembed, args.unk) else: embed = None if self.WORD: self.WORD.build(train, args.min_freq, embed) self.FEAT.build(train) self.REL.build(train) if args.feat == 'bert': # do not save the tokenize funztion, or else it might be incompatible with new releases tokenize = self.FEAT.tokenize # save it self.FEAT.tokenize = None torch.save(self.fields, args.fields) if args.feat == 'bert': self.FEAT.tokenize = tokenize # restore self.trainset = train # pass it on to subclasses else: self.trainset = None self.fields = torch.load(args.fields) if args.feat == 'bert': tokenizer = SubwordField.tokenizer(args.bert_model) if args.n_embed: self.fields.FORM[1].tokenize = tokenizer.tokenize else: self.fields.FORM.tokenize = tokenizer.tokenize if args.feat in ('char', 'bert'): if isinstance(self.fields.FORM, tuple): self.WORD, self.FEAT = self.fields.FORM else: self.WORD, self.FEAT = None, self.fields.FORM else: self.WORD, self.FEAT = self.fields.FORM, self.fields.CPOS self.ARC, self.REL = self.fields.HEAD, self.fields.DEPREL self.puncts = torch.tensor( [i for s, i in self.WORD.vocab.stoi.items() if ispunct(s)]).to(args.device) if self.WORD else [] # override parameters from embeddings: if self.WORD: args.update({ 'n_words': self.WORD.vocab.n_init, 'pad_index': self.WORD.pad_index, 'unk_index': self.WORD.unk_index, 'bos_index': self.WORD.bos_index, }) args.update({ 'n_feats': len(self.FEAT.vocab), 'n_rels': len(self.REL.vocab), 'feat_pad_index': self.FEAT.pad_index, }) logger.info("Features:") if self.WORD: logger.info(f" {self.WORD}") logger.info(f" {self.FEAT}\n {self.ARC}\n {self.REL}")
def __call__(self, args): self.args = args if not os.path.exists(args.file): os.mkdir(args.file) if not os.path.exists(args.fields) or args.preprocess: print("Preprocess the data") self.WORD = Field('words', pad=pad, unk=unk, bos=bos, lower=True) if args.feat == 'char': self.FEAT = CharField('chars', pad=pad, unk=unk, bos=bos, fix_len=args.fix_len, tokenize=list) elif args.feat == 'bert': tokenizer = BertTokenizer.from_pretrained(args.bert_model) self.FEAT = BertField('bert', pad='[PAD]', bos='[CLS]', tokenize=tokenizer.encode) else: self.FEAT = Field('tags', bos=bos) self.HEAD = Field('heads', bos=bos, use_vocab=False, fn=int) self.REL = Field('rels', bos=bos) if args.feat in ('char', 'bert'): self.fields = CoNLL(FORM=(self.WORD, self.FEAT), HEAD=self.HEAD, DEPREL=self.REL) else: self.fields = CoNLL(FORM=self.WORD, CPOS=self.FEAT, HEAD=self.HEAD, DEPREL=self.REL) train = Corpus.load(args.ftrain, self.fields) if args.fembed: embed = Embedding.load(args.fembed, args.unk) else: embed = None self.WORD.build(train, args.min_freq, embed) self.FEAT.build(train) self.REL.build(train) torch.save(self.fields, args.fields) else: self.fields = torch.load(args.fields) if args.feat in ('char', 'bert'): self.WORD, self.FEAT = self.fields.FORM else: self.WORD, self.FEAT = self.fields.FORM, self.fields.CPOS self.HEAD, self.REL = self.fields.HEAD, self.fields.DEPREL self.puncts = torch.tensor([ i for s, i in self.WORD.vocab.stoi.items() if ispunct(s) ]).to(args.device) self.criterion = nn.CrossEntropyLoss() print(f"{self.WORD}\n{self.FEAT}\n{self.HEAD}\n{self.REL}") args.update({ 'n_words': self.WORD.vocab.n_init, 'n_feats': len(self.FEAT.vocab), 'n_rels': len(self.REL.vocab), 'pad_index': self.WORD.pad_index, 'unk_index': self.WORD.unk_index, 'bos_index': self.WORD.bos_index })
def __call__(self, args): self.args = args logging.basicConfig(filename=args.output, filemode='w', format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') args.ud_dataset = { 'en': ( 'data/ud/UD_English-EWT/en_ewt-ud-train.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-dev.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-test.conllx', "data/fastText_data/wiki.en.ewt.vec.new", ), 'en20': ( 'data/ud/UD_English-EWT/en_ewt-ud-train20.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-dev.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-test.conllx', "data/fastText_data/wiki.en.ewt.vec.new", ), 'en40': ( 'data/ud/UD_English-EWT/en_ewt-ud-train40.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-dev.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-test.conllx', "data/fastText_data/wiki.en.ewt.vec.new", ), 'en60': ( 'data/ud/UD_English-EWT/en_ewt-ud-train60.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-dev.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-test.conllx', "data/fastText_data/wiki.en.ewt.vec.new", ), 'en80': ( 'data/ud/UD_English-EWT/en_ewt-ud-train80.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-dev.conllx', 'data/ud/UD_English-EWT/en_ewt-ud-test.conllx', "data/fastText_data/wiki.en.ewt.vec.new", ), 'ar': ( "data/ud/UD_Arabic-PADT/ar_padt-ud-train.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-dev.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-test.conllx", "data/fastText_data/wiki.ar.padt.vec.new", ), 'ar20': ( "data/ud/UD_Arabic-PADT/ar_padt-ud-train20.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-dev.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-test.conllx", "data/fastText_data/wiki.ar.padt.vec.new", ), 'ar40': ( "data/ud/UD_Arabic-PADT/ar_padt-ud-train40.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-dev.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-test.conllx", "data/fastText_data/wiki.ar.padt.vec.new", ), 'ar60': ( "data/ud/UD_Arabic-PADT/ar_padt-ud-train60.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-dev.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-test.conllx", "data/fastText_data/wiki.ar.padt.vec.new", ), 'ar80': ( "data/ud/UD_Arabic-PADT/ar_padt-ud-train80.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-dev.conllx", "data/ud/UD_Arabic-PADT/ar_padt-ud-test.conllx", "data/fastText_data/wiki.ar.padt.vec.new", ), 'bg': ( "data/ud/UD_Bulgarian-BTB/bg_btb-ud-train.conllx", "data/ud/UD_Bulgarian-BTB/bg_btb-ud-dev.conllx", "data/ud/UD_Bulgarian-BTB/bg_btb-ud-test.conllx", "data/fastText_data/wiki.bg.btb.vec.new", ), 'da': ( "data/ud/UD_Danish-DDT/da_ddt-ud-train.conllx", "data/ud/UD_Danish-DDT/da_ddt-ud-dev.conllx", "data/ud/UD_Danish-DDT/da_ddt-ud-test.conllx", "data/fastText_data/wiki.da.ddt.vec.new", ), 'de': ( "data/ud/UD_German-GSD/de_gsd-ud-train.conllx", "data/ud/UD_German-GSD/de_gsd-ud-dev.conllx", "data/ud/UD_German-GSD/de_gsd-ud-test.conllx", "data/fastText_data/wiki.de.gsd.vec.new", ), 'es': ( "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-train.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-dev.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-test.conllx", "data/fastText_data/wiki.es.gsdancora.vec.new", ), 'es20': ( "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-train20.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-dev.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-test.conllx", "data/fastText_data/wiki.es.gsdancora.vec.new", ), 'es40': ( "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-train40.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-dev.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-test.conllx", "data/fastText_data/wiki.es.gsdancora.vec.new", ), 'es60': ( "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-train60.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-dev.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-test.conllx", "data/fastText_data/wiki.es.gsdancora.vec.new", ), 'es80': ( "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-train80.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-dev.conllx", "data/ud/UD_Spanish-GSDAnCora/es_gsdancora-ud-test.conllx", "data/fastText_data/wiki.es.gsdancora.vec.new", ), 'fa': ( "data/ud/UD_Persian-Seraji/fa_seraji-ud-train.conllx", "data/ud/UD_Persian-Seraji/fa_seraji-ud-dev.conllx", "data/ud/UD_Persian-Seraji/fa_seraji-ud-test.conllx", "data/fastText_data/wiki.fa.seraji.vec.new", ), 'fr': ( "data/ud/UD_French-GSD/fr_gsd-ud-train.conllx", "data/ud/UD_French-GSD/fr_gsd-ud-dev.conllx", "data/ud/UD_French-GSD/fr_gsd-ud-test.conllx", "data/fastText_data/wiki.fr.gsd.vec.new", ), 'he': ( "data/ud/UD_Hebrew-HTB/he_htb-ud-train.conllx", "data/ud/UD_Hebrew-HTB/he_htb-ud-dev.conllx", "data/ud/UD_Hebrew-HTB/he_htb-ud-test.conllx", "data/fastText_data/wiki.he.htb.vec.new", ), 'hi': ( "data/ud/UD_Hindi-HDTB/hi_hdtb-ud-train.conllx", "data/ud/UD_Hindi-HDTB/hi_hdtb-ud-dev.conllx", "data/ud/UD_Hindi-HDTB/hi_hdtb-ud-test.conllx", "data/fastText_data/wiki.hi.hdtb.vec.new", ), 'hr': ( "data/ud/UD_Croatian-SET/hr_set-ud-train.conllx", "data/ud/UD_Croatian-SET/hr_set-ud-dev.conllx", "data/ud/UD_Croatian-SET/hr_set-ud-test.conllx", "data/fastText_data/wiki.hr.set.vec.new", ), 'id': ( "data/ud/UD_Indonesian-GSD/id_gsd-ud-train.conllx", "data/ud/UD_Indonesian-GSD/id_gsd-ud-dev.conllx", "data/ud/UD_Indonesian-GSD/id_gsd-ud-test.conllx", "data/fastText_data/wiki.id.gsd.vec.new", ), 'it': ( "data/ud/UD_Italian-ISDT/it_isdt-ud-train.conllx", "data/ud/UD_Italian-ISDT/it_isdt-ud-dev.conllx", "data/ud/UD_Italian-ISDT/it_isdt-ud-test.conllx", "data/fastText_data/wiki.it.isdt.vec.new", ), 'ja': ( "data/ud/UD_Japanese-GSD/ja_gsd-ud-train.conllx", "data/ud/UD_Japanese-GSD/ja_gsd-ud-dev.conllx", "data/ud/UD_Japanese-GSD/ja_gsd-ud-test.conllx", "data/fastText_data/wiki.ja.gsd.vec.new", ), 'ko': ( "data/ud/UD_Korean-GSDKaist/ko_gsdkaist-ud-train.conllx", "data/ud/UD_Korean-GSDKaist/ko_gsdkaist-ud-dev.conllx", "data/ud/UD_Korean-GSDKaist/ko_gsdkaist-ud-test.conllx", "data/fastText_data/wiki.ko.gsdkaist.vec.new", ), 'nl': ( "data/ud/UD_Dutch-AlpinoLassySmall/nl_alpinolassysmall-ud-train.conllx", "data/ud/UD_Dutch-AlpinoLassySmall/nl_alpinolassysmall-ud-dev.conllx", "data/ud/UD_Dutch-AlpinoLassySmall/nl_alpinolassysmall-ud-test.conllx", "data/fastText_data/wiki.nl.alpinolassysmall.vec.new", ), 'no': ( "data/ud/UD_Norwegian-BokmaalNynorsk/no_bokmaalnynorsk-ud-train.conllx", "data/ud/UD_Norwegian-BokmaalNynorsk/no_bokmaalnynorsk-ud-dev.conllx", "data/ud/UD_Norwegian-BokmaalNynorsk/no_bokmaalnynorsk-ud-test.conllx", "data/fastText_data/wiki.no.bokmaalnynorsk.vec.new", ), 'pt': ( "data/ud/UD_Portuguese-BosqueGSD/pt_bosquegsd-ud-train.conllx", "data/ud/UD_Portuguese-BosqueGSD/pt_bosquegsd-ud-dev.conllx", "data/ud/UD_Portuguese-BosqueGSD/pt_bosquegsd-ud-test.conllx", "data/fastText_data/wiki.pt.bosquegsd.vec.new", ), 'sv': ( "data/ud/UD_Swedish-Talbanken/sv_talbanken-ud-train.conllx", "data/ud/UD_Swedish-Talbanken/sv_talbanken-ud-dev.conllx", "data/ud/UD_Swedish-Talbanken/sv_talbanken-ud-test.conllx", "data/fastText_data/wiki.sv.talbanken.vec.new", ), 'tr': ( "data/ud/UD_Turkish-IMST/tr_imst-ud-train.conllx", "data/ud/UD_Turkish-IMST/tr_imst-ud-dev.conllx", "data/ud/UD_Turkish-IMST/tr_imst-ud-test.conllx", "data/fastText_data/wiki.tr.imst.vec.new", ), 'zh': ( "data/ud/UD_Chinese-GSD/zh_gsd-ud-train.conllx", "data/ud/UD_Chinese-GSD/zh_gsd-ud-dev.conllx", "data/ud/UD_Chinese-GSD/zh_gsd-ud-test.conllx", "data/fastText_data/wiki.zh.gsd.vec.new", )} self.args.ftrain = args.ud_dataset[args.lang][0] self.args.fdev = args.ud_dataset[args.lang][1] self.args.ftest = args.ud_dataset[args.lang][2] self.args.fembed = args.ud_dataset[args.lang][3] if not os.path.exists(args.file): os.mkdir(args.file) if not os.path.exists(args.fields) or args.preprocess: logging.info("Preprocess the data") self.WORD = Field('words', pad=pad, unk=unk, bos=bos, lower=True) tokenizer = BertTokenizer.from_pretrained(args.bert_model) self.BERT = BertField('bert', pad='[PAD]', bos='[CLS]', tokenize=tokenizer.encode) if args.feat == 'char': self.FEAT = CharField('chars', pad=pad, unk=unk, bos=bos, fix_len=args.fix_len, tokenize=list) elif args.feat == 'bert': tokenizer = BertTokenizer.from_pretrained(args.bert_model) self.FEAT = BertField('bert', pad='[PAD]', bos='[CLS]', tokenize=tokenizer.encode) else: self.FEAT = Field('tags', bos=bos) self.HEAD = Field('heads', bos=bos, use_vocab=False, fn=int) self.REL = Field('rels', bos=bos) if args.feat in ('char', 'bert'): self.fields = CoNLL(FORM=(self.WORD, self.BERT, self.FEAT), HEAD=self.HEAD, DEPREL=self.REL) else: self.fields = CoNLL(FORM=(self.WORD, self.BERT), CPOS=self.FEAT, HEAD=self.HEAD, DEPREL=self.REL) train = Corpus.load(args.ftrain, self.fields, args.max_len) if args.fembed: if args.bert is False: # fasttext embed = Embedding.load(args.fembed, args.lang, unk=args.unk) else: embed = None else: embed = None self.WORD.build(train, args.min_freq, embed) self.FEAT.build(train) self.BERT.build(train) self.REL.build(train) torch.save(self.fields, args.fields) else: self.fields = torch.load(args.fields) if args.feat in ('char', 'bert'): self.WORD, self.BERT, self.FEAT = self.fields.FORM else: self.WORD, self.BERT, self.FEAT = self.fields.FORM, self.fields.CPOS self.HEAD, self.REL = self.fields.HEAD, self.fields.DEPREL self.puncts = torch.tensor([i for s, i in self.WORD.vocab.stoi.items() if ispunct(s)]).to(args.device) self.criterion = nn.CrossEntropyLoss() logging.info(f"{self.WORD}\n{self.FEAT}\n{self.BERT}\n{self.HEAD}\n{self.REL}") args.update({ 'n_words': self.WORD.vocab.n_init, 'n_feats': len(self.FEAT.vocab), 'n_bert': len(self.BERT.vocab), 'n_rels': len(self.REL.vocab), 'pad_index': self.WORD.pad_index, 'unk_index': self.WORD.unk_index, 'bos_index': self.WORD.bos_index }) logging.info(f"n_words {args.n_words} n_feats {args.n_feats} n_bert {args.n_bert} pad_index {args.pad_index} bos_index {args.bos_index}")
def __call__(self, args): self.args = args if not os.path.exists(args.file): os.mkdir(args.file) if not os.path.exists(args.fields) or args.preprocess: print("Preprocess the data") self.CHAR = Field('chars', pad=pad, unk=unk, bos=bos, eos=eos, lower=True) # TODO span as label, modify chartfield to spanfield self.SEG = SegmentField('segs') if args.feat == 'bert': tokenizer = BertTokenizer.from_pretrained(args.bert_model) self.FEAT = BertField('bert', pad='[PAD]', bos='[CLS]', eos='[SEP]', tokenize=tokenizer.encode) self.fields = CoNLL(CHAR=(self.CHAR, self.FEAT), SEG=self.SEG) elif args.feat == 'bigram': self.BIGRAM = NGramField( 'bichar', n=2, pad=pad, unk=unk, bos=bos, eos=eos, lower=True) self.fields = CoNLL(CHAR=(self.CHAR, self.BIGRAM), SEG=self.SEG) elif args.feat == 'trigram': self.BIGRAM = NGramField( 'bichar', n=2, pad=pad, unk=unk, bos=bos, eos=eos, lower=True) self.TRIGRAM = NGramField( 'trichar', n=3, pad=pad, unk=unk, bos=bos, eos=eos, lower=True) self.fields = CoNLL(CHAR=(self.CHAR, self.BIGRAM, self.TRIGRAM), SEG=self.SEG) else: self.fields = CoNLL(CHAR=self.CHAR, SEG=self.SEG) train = Corpus.load(args.ftrain, self.fields) embed = Embedding.load( 'data/tencent.char.200.txt', args.unk) if args.embed else None self.CHAR.build(train, args.min_freq, embed) if hasattr(self, 'FEAT'): self.FEAT.build(train) if hasattr(self, 'BIGRAM'): embed = Embedding.load( 'data/tencent.bi.200.txt', args.unk) if args.embed else None self.BIGRAM.build(train, args.min_freq, embed=embed, dict_file=args.dict_file) if hasattr(self, 'TRIGRAM'): embed = Embedding.load( 'data/tencent.tri.200.txt', args.unk) if args.embed else None self.TRIGRAM.build(train, args.min_freq, embed=embed, dict_file=args.dict_file) # TODO self.SEG.build(train) torch.save(self.fields, args.fields) else: self.fields = torch.load(args.fields) if args.feat == 'bert': self.CHAR, self.FEAT = self.fields.CHAR elif args.feat == 'bigram': self.CHAR, self.BIGRAM = self.fields.CHAR elif args.feat == 'trigram': self.CHAR, self.BIGRAM, self.TRIGRAM = self.fields.CHAR else: self.CHAR = self.fields.CHAR # TODO self.SEG = self.fields.SEG # TODO loss funciton # self.criterion = nn.CrossEntropyLoss() # # [B, E, M, S] # self.trans = (torch.tensor([1., 0., 0., 1.]).log().to(args.device), # torch.tensor([0., 1., 0., 1.]).log().to(args.device), # torch.tensor([[0., 1., 1., 0.], # [1., 0., 0., 1.], # [0., 1., 1., 0.], # [1., 0., 0., 1.]]).log().to(args.device)) args.update({ 'n_chars': self.CHAR.vocab.n_init, 'pad_index': self.CHAR.pad_index, 'unk_index': self.CHAR.unk_index }) # TODO vocab = f"{self.CHAR}\n" if hasattr(self, 'FEAT'): args.update({ 'n_feats': self.FEAT.vocab.n_init, }) vocab += f"{self.FEAT}\n" if hasattr(self, 'BIGRAM'): args.update({ 'n_bigrams': self.BIGRAM.vocab.n_init, }) vocab += f"{self.BIGRAM}\n" if hasattr(self, 'TRIGRAM'): args.update({ 'n_trigrams': self.TRIGRAM.vocab.n_init, }) vocab += f"{self.TRIGRAM}\n" print(f"Override the default configs\n{args}") print(vocab[:-1])
def __call__(self, config): if not os.path.exists(config.file): os.mkdir(config.file) if config.preprocess or not os.path.exists(config.vocab): print("Preprocess the corpus") pos_train = Corpus.load(config.fptrain, [1, 4], config.pos) dep_train = Corpus.load(config.ftrain) pos_dev = Corpus.load(config.fpdev, [1, 4]) dep_dev = Corpus.load(config.fdev) pos_test = Corpus.load(config.fptest, [1, 4]) dep_test = Corpus.load(config.ftest) print("Create the vocab") vocab = Vocab.from_corpora(pos_train, dep_train, 2) vocab.read_embeddings(Embedding.load(config.fembed)) print("Load the dataset") pos_trainset = TextDataset(vocab.numericalize(pos_train, False), config.buckets) dep_trainset = TextDataset(vocab.numericalize(dep_train), config.buckets) pos_devset = TextDataset(vocab.numericalize(pos_dev, False), config.buckets) dep_devset = TextDataset(vocab.numericalize(dep_dev), config.buckets) pos_testset = TextDataset(vocab.numericalize(pos_test, False), config.buckets) dep_testset = TextDataset(vocab.numericalize(dep_test), config.buckets) torch.save(vocab, config.vocab) torch.save(pos_trainset, os.path.join(config.file, 'pos_trainset')) torch.save(dep_trainset, os.path.join(config.file, 'dep_trainset')) torch.save(pos_devset, os.path.join(config.file, 'pos_devset')) torch.save(dep_devset, os.path.join(config.file, 'dep_devset')) torch.save(pos_testset, os.path.join(config.file, 'pos_testset')) torch.save(dep_testset, os.path.join(config.file, 'dep_testset')) else: print("Load the vocab") vocab = torch.load(config.vocab) print("Load the datasets") pos_trainset = torch.load(os.path.join(config.file, 'pos_trainset')) dep_trainset = torch.load(os.path.join(config.file, 'dep_trainset')) pos_devset = torch.load(os.path.join(config.file, 'pos_devset')) dep_devset = torch.load(os.path.join(config.file, 'dep_devset')) pos_testset = torch.load(os.path.join(config.file, 'pos_testset')) dep_testset = torch.load(os.path.join(config.file, 'dep_testset')) config.update({ 'n_words': vocab.n_init, 'n_chars': vocab.n_chars, 'n_pos_tags': vocab.n_pos_tags, 'n_dep_tags': vocab.n_dep_tags, 'n_rels': vocab.n_rels, 'pad_index': vocab.pad_index, 'unk_index': vocab.unk_index }) # set the data loaders pos_train_loader = batchify( pos_trainset, config.pos_batch_size // config.update_steps, True) dep_train_loader = batchify(dep_trainset, config.batch_size // config.update_steps, True) pos_dev_loader = batchify(pos_devset, config.pos_batch_size) dep_dev_loader = batchify(dep_devset, config.batch_size) pos_test_loader = batchify(pos_testset, config.pos_batch_size) dep_test_loader = batchify(dep_testset, config.batch_size) print(vocab) print(f"{'pos_train:':10} {len(pos_trainset):7} sentences in total, " f"{len(pos_train_loader):4} batches provided") print(f"{'dep_train:':10} {len(dep_trainset):7} sentences in total, " f"{len(dep_train_loader):4} batches provided") print(f"{'pos_dev:':10} {len(pos_devset):7} sentences in total, " f"{len(pos_dev_loader):4} batches provided") print(f"{'dep_dev:':10} {len(dep_devset):7} sentences in total, " f"{len(dep_dev_loader):4} batches provided") print(f"{'pos_test:':10} {len(pos_testset):7} sentences in total, " f"{len(pos_test_loader):4} batches provided") print(f"{'dep_test:':10} {len(dep_testset):7} sentences in total, " f"{len(dep_test_loader):4} batches provided") print("Create the model") parser = BiaffineParser(config, vocab.embed).to(config.device) print(f"{parser}\n") model = Model(config, vocab, parser) total_time = timedelta() best_e, best_metric = 1, AttachmentMethod() model.optimizer = Adam(model.parser.parameters(), config.lr, (config.mu, config.nu), config.epsilon) model.scheduler = ExponentialLR(model.optimizer, config.decay**(1 / config.decay_steps)) for epoch in range(1, config.epochs + 1): start = datetime.now() # train one epoch and update the parameters model.train(pos_train_loader, dep_train_loader) print(f"Epoch {epoch} / {config.epochs}:") lp, ld, mp, mdt, mdp = model.evaluate(None, dep_train_loader) print(f"{'train:':6} LP: {lp:.4f} LD: {ld:.4f} {mp} {mdt} {mdp}") lp, ld, mp, mdt, dev_m = model.evaluate(pos_dev_loader, dep_dev_loader) print(f"{'dev:':6} LP: {lp:.4f} LD: {ld:.4f} {mp} {mdt} {dev_m}") lp, ld, mp, mdt, mdp = model.evaluate(pos_test_loader, dep_test_loader) print(f"{'test:':6} LP: {lp:.4f} LD: {ld:.4f} {mp} {mdt} {mdp}") t = datetime.now() - start # save the model if it is the best so far if dev_m > best_metric and epoch > config.patience: best_e, best_metric = epoch, dev_m model.parser.save(config.model) print(f"{t}s elapsed (saved)\n") else: print(f"{t}s elapsed\n") total_time += t if epoch - best_e >= config.patience: break model.parser = BiaffineParser.load(config.model) lp, ld, mp, mdt, mdp = model.evaluate(pos_test_loader, dep_test_loader) print(f"max score of dev is {best_metric.score:.2%} at epoch {best_e}") print(f"the score of test at epoch {best_e} is {mdp.score:.2%}") print(f"average time of each epoch is {total_time / epoch}s") print(f"{total_time}s elapsed")