def __call__(self, args): # override config from CLI parameters args = Config(args.conf).update(vars(args)) args.n_attentions = args.use_attentions # back compatibility # loads train corpus into self.trainset super().__call__(args) logger.info(f"Configuration parameters:\n{args}") #train = Corpus.load(args.ftrain, self.fields, args.max_sent_length) train = self.trainset dev = Corpus.load(args.fdev, self.fields, args.max_sent_length) if args.ftest: test = Corpus.load(args.ftest, self.fields, args.max_sent_length) train = TextDataset(train, self.fields, args.buckets) dev = TextDataset(dev, self.fields, args.buckets) if args.ftest: test = TextDataset(test, self.fields, args.buckets) # set the data loaders train.loader = batchify(train, args.batch_size, True) dev.loader = batchify(dev, args.batch_size) if args.ftest: test.loader = batchify(test, args.batch_size) logger.info(f"{'train:':6} {len(train):5} sentences, " f"{len(train.loader):3} batches, " f"{len(train.buckets)} buckets") logger.info(f"{'dev:':6} {len(dev):5} sentences, " f"{len(dev.loader):3} batches, " f"{len(train.buckets)} buckets") if args.ftest: logger.info(f"{'test:':6} {len(test):5} sentences, " f"{len(test.loader):3} batches, " f"{len(train.buckets)} buckets") logger.info("Create the model") self.model = Model(args, mask_token_id=self.FEAT.mask_token_id) if self.WORD: self.model.load_pretrained(self.WORD.embed) self.model = self.model.to(args.device) if torch.cuda.device_count() > 1: self.model = TransparentDataParallel(self.model) logger.info(f"{self.model}\n") if args.optimizer == 'adamw': self.optimizer = AdamW(self.model.parameters(), args.lr, (args.mu, args.nu), args.epsilon, args.decay) training_steps = len(train.loader) // self.args.accumulation_steps \ * self.args.epochs warmup_steps = math.ceil(training_steps * self.args.warmup_steps_ratio) self.scheduler = get_linear_schedule_with_warmup( self.optimizer, num_warmup_steps=warmup_steps, num_training_steps=training_steps) else: self.optimizer = Adam(self.model.parameters(), args.lr, (args.mu, args.nu), args.epsilon) self.scheduler = ExponentialLR(self.optimizer, args.decay**(1 / args.decay_steps)) total_time = timedelta() best_e, best_metric = 1, Metric() for epoch in range(1, args.epochs + 1): start = datetime.now() logger.info(f"Epoch {epoch} / {args.epochs}:") loss, train_metric = self.train(train.loader) logger.info(f"{'train:':6} Loss: {loss:.4f} {train_metric}") loss, dev_metric = self.evaluate(dev.loader) logger.info(f"{'dev:':6} Loss: {loss:.4f} {dev_metric}") if args.ftest: loss, test_metric = self.evaluate(test.loader) logger.info(f"{'test:':6} Loss: {loss:.4f} {test_metric}") t = datetime.now() - start # save the model if it is the best so far if dev_metric > best_metric and epoch > args.patience // 10: best_e, best_metric = epoch, dev_metric if hasattr(self.model, 'module'): self.model.module.save(args.model) else: self.model.save(args.model) logger.info(f"{t}s elapsed (saved)\n") else: logger.info(f"{t}s elapsed\n") total_time += t if epoch - best_e >= args.patience: break self.model = Model.load(args.model) if args.ftest: loss, metric = self.evaluate(test.loader) logger.info( f"max score of dev is {best_metric.score:.2%} at epoch {best_e}") if args.ftest: logger.info( f"the score of test at epoch {best_e} is {metric.score:.2%}") logger.info(f"average time of each epoch is {total_time / epoch}s") logger.info(f"{total_time}s elapsed")
def __call__(self, config): print("Preprocess the data") if config.input_type == "conllu": train = UniversalDependenciesDatasetReader() train.load(config.ftrain) dev = UniversalDependenciesDatasetReader() dev.load(config.fdev) test = UniversalDependenciesDatasetReader() test.load(config.ftest) else: train = Corpus.load(config.ftrain) dev = Corpus.load(config.fdev) test = Corpus.load(config.ftest) if config.use_predicted: if config.input_type == "conllu": train_predicted = UniversalDependenciesDatasetReader() train_predicted.load(config.fpredicted_train) dev_predicted = UniversalDependenciesDatasetReader() dev_predicted.load(config.fpredicted_dev) test_predicted = UniversalDependenciesDatasetReader() test_predicted.load(config.fpredicted_test) else: train_predicted = Corpus.load(config.fpredicted_train) dev_predicted = Corpus.load(config.fpredicted_dev) test_predicted = Corpus.load(config.fpredicted_test) if path.exists(config.main_path + "/exp") != True: os.mkdir(config.main_path + "/exp") if path.exists(config.main_path + "/model") != True: os.mkdir(config.main_path + "/model") if path.exists(config.main_path + config.model + config.modelname) != True: os.mkdir(config.main_path + config.model + config.modelname) vocab = Vocab.from_corpus(config=config, corpus=train, min_freq=2) torch.save(vocab, config.main_path + config.vocab + config.modelname + "/vocab.tag") config.update({ 'n_words': vocab.n_train_words, 'n_tags': vocab.n_tags, 'n_rels': vocab.n_rels, 'pad_index': vocab.pad_index, 'unk_index': vocab.unk_index }) print("Load the dataset") if config.use_predicted: trainset = TextDataset(vocab.numericalize(train, train_predicted)) devset = TextDataset(vocab.numericalize(dev, dev_predicted)) testset = TextDataset(vocab.numericalize(test, test_predicted)) else: trainset = TextDataset(vocab.numericalize(train)) devset = TextDataset(vocab.numericalize(dev)) testset = TextDataset(vocab.numericalize(test)) # set the data loaders train_loader, _ = batchify(dataset=trainset, batch_size=config.batch_size, n_buckets=config.buckets, shuffle=True) dev_loader, _ = batchify(dataset=devset, batch_size=config.batch_size, n_buckets=config.buckets) test_loader, _ = batchify(dataset=testset, batch_size=config.batch_size, n_buckets=config.buckets) print(f"{'train:':6} {len(trainset):5} sentences in total, " f"{len(train_loader):3} batches provided") print(f"{'dev:':6} {len(devset):5} sentences in total, " f"{len(dev_loader):3} batches provided") print(f"{'test:':6} {len(testset):5} sentences in total, " f"{len(test_loader):3} batches provided") print("Create the model") parser = BiaffineParser(config, vocab.n_rels, vocab.bertmodel) print("number of pars:{}".format(sum(p.numel() for p in parser.parameters() if p.requires_grad))) if torch.cuda.is_available(): print('device:cuda') device = torch.device('cuda') parser = parser.to(device) # print(f"{parser}\n") model = Model(vocab, parser, config, vocab.n_rels) total_time = timedelta() best_e, best_metric = 1, Metric() num_train_optimization_steps = int(config.num_iter_encoder * config.epochs * len(train_loader)) warmup_steps = int(config.warmupproportion * num_train_optimization_steps) if config.use_two_opts: model_nonbert = [] model_bert = [] layernorm_params = ['layernorm_key_layer', 'layernorm_value_layer', 'dp_relation_k', 'dp_relation_v'] for name, param in parser.named_parameters(): if 'bert' in name and not any(nd in name for nd in layernorm_params): model_bert.append((name, param)) else: model_nonbert.append((name, param)) # Prepare optimizer and schedule (linear warmup and decay) for Non-bert parameters no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters_nonbert = [ {'params': [p for n, p in model_nonbert if not any(nd in n for nd in no_decay)], 'weight_decay': config.weight_decay}, {'params': [p for n, p in model_nonbert if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] model.optimizer_nonbert = AdamW(optimizer_grouped_parameters_nonbert, lr=config.lr2) model.scheduler_nonbert = get_linear_schedule_with_warmup(model.optimizer_nonbert, num_warmup_steps=warmup_steps, num_training_steps=num_train_optimization_steps) # Prepare optimizer and schedule (linear warmup and decay) for Bert parameters optimizer_grouped_parameters_bert = [ {'params': [p for n, p in model_bert if not any(nd in n for nd in no_decay)], 'weight_decay': config.weight_decay}, {'params': [p for n, p in model_bert if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] model.optimizer_bert = AdamW(optimizer_grouped_parameters_bert, lr=config.lr1) model.scheduler_bert = get_linear_schedule_with_warmup( model.optimizer_bert, num_warmup_steps=warmup_steps, num_training_steps=num_train_optimization_steps ) else: # Prepare optimizer and schedule (linear warmup and decay) no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ {'params': [p for n, p in parser.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': config.weight_decay}, {'params': [p for n, p in parser.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] model.optimizer = AdamW(optimizer_grouped_parameters, lr=config.lr1) model.scheduler = get_linear_schedule_with_warmup( model.optimizer, num_warmup_steps=warmup_steps, num_training_steps=num_train_optimization_steps ) for epoch in range(1, config.epochs + 1): start = datetime.now() # train one epoch and update the parameters if config.use_predicted: model.train_predicted(train_loader) else: model.train(train_loader) print(f"Epoch {epoch} / {config.epochs}:") if config.use_predicted: loss, dev_metric = model.evaluate_predicted(dev_loader, config.punct) else: loss, dev_metric = model.evaluate(dev_loader, config.punct) print(f"{'dev:':6} Loss: {loss:.4f} {dev_metric}") if config.use_predicted: loss, test_metric = model.evaluate_predicted(test_loader, config.punct) else: loss, test_metric = model.evaluate(test_loader, config.punct) print(f"{'test:':6} Loss: {loss:.4f} {test_metric}") t = datetime.now() - start # save the model if it is the best so far if dev_metric > best_metric: best_e, best_metric = epoch, dev_metric print(config.model + config.modelname + "/model_weights") model.parser.save(config.main_path + config.model + config.modelname + "/model_weights") print(f"{t}s elapsed (saved)\n") else: print(f"{t}s elapsed\n") total_time += t if epoch - best_e >= config.patience: break model.parser = BiaffineParser.load(config.main_path + config.model + config.modelname + "/model_weights") if config.use_predicted: loss, metric = model.evaluate_predicted(test_loader, config.punct) else: loss, metric = model.evaluate(test_loader, config.punct) print(metric) print(f"max score of dev is {best_metric.score:.2%} at epoch {best_e}") print(f"the score of test at epoch {best_e} is {metric.score:.2%}") print(f"average time of each epoch is {total_time / epoch}s") print(f"{total_time}s elapsed")
def __call__(self, args): super(Train, self).__call__(args) train = Corpus.load(args.ftrain, self.fields) dev = Corpus.load(args.fdev, self.fields) test = Corpus.load(args.ftest, self.fields) train = TextDataset(train, self.fields, args.buckets) dev = TextDataset(dev, self.fields, args.buckets) test = TextDataset(test, self.fields, args.buckets) # set the data loaders train.loader = batchify(train, args.batch_size, True) dev.loader = batchify(dev, args.batch_size) test.loader = batchify(test, args.batch_size) print(f"{'train:':6} {len(train):5} sentences, " f"{len(train.loader):3} batches, " f"{len(train.buckets)} buckets") print(f"{'dev:':6} {len(dev):5} sentences, " f"{len(dev.loader):3} batches, " f"{len(train.buckets)} buckets") print(f"{'test:':6} {len(test):5} sentences, " f"{len(test.loader):3} batches, " f"{len(train.buckets)} buckets") print("Create the model") self.model = Model(args).load_pretrained(self.WORD.embed) print(f"{self.model}\n") self.model = self.model.to(args.device) if torch.cuda.device_count() > 1: self.model = nn.DataParallel(self.model) self.optimizer = Adam(self.model.parameters(), args.lr, (args.mu, args.nu), args.epsilon) self.scheduler = ExponentialLR(self.optimizer, args.decay**(1 / args.decay_steps)) total_time = timedelta() best_e, best_metric = 1, Metric() for epoch in range(1, args.epochs + 1): start = datetime.now() # train one epoch and update the parameters self.train(train.loader) print(f"Epoch {epoch} / {args.epochs}:") loss, train_metric = self.evaluate(train.loader) print(f"{'train:':6} Loss: {loss:.4f} {train_metric}") loss, dev_metric = self.evaluate(dev.loader) print(f"{'dev:':6} Loss: {loss:.4f} {dev_metric}") loss, test_metric = self.evaluate(test.loader) print(f"{'test:':6} Loss: {loss:.4f} {test_metric}") t = datetime.now() - start # save the model if it is the best so far if dev_metric > best_metric and epoch > args.patience: best_e, best_metric = epoch, dev_metric if hasattr(self.model, 'module'): self.model.module.save(args.model) else: self.model.save(args.model) print(f"{t}s elapsed (saved)\n") else: print(f"{t}s elapsed\n") total_time += t if epoch - best_e >= args.patience: break if hasattr(self.model, 'module'): self.model.module.save(args.model) else: self.model.save(args.model) print(f"{t}s elapsed (saved)\n") self.model = Model.load(args.model) loss, metric = self.evaluate(test.loader) print(f"max score of dev is {best_metric.score:.2%} at epoch {best_e}") print(f"the score of test at epoch {best_e} is {metric.score:.2%}") print(f"average time of each epoch is {total_time / epoch}s") print(f"{total_time}s elapsed")
def __call__(self, config): if not os.path.exists(config.file): os.mkdir(config.file) if config.preprocess or not os.path.exists(config.vocab): print("Preprocess the corpus") pos_train = Corpus.load(config.fptrain, [1, 4], config.pos) dep_train = Corpus.load(config.ftrain) pos_dev = Corpus.load(config.fpdev, [1, 4]) dep_dev = Corpus.load(config.fdev) pos_test = Corpus.load(config.fptest, [1, 4]) dep_test = Corpus.load(config.ftest) print("Create the vocab") vocab = Vocab.from_corpora(pos_train, dep_train, 2) vocab.read_embeddings(Embedding.load(config.fembed)) print("Load the dataset") pos_trainset = TextDataset(vocab.numericalize(pos_train, False), config.buckets) dep_trainset = TextDataset(vocab.numericalize(dep_train), config.buckets) pos_devset = TextDataset(vocab.numericalize(pos_dev, False), config.buckets) dep_devset = TextDataset(vocab.numericalize(dep_dev), config.buckets) pos_testset = TextDataset(vocab.numericalize(pos_test, False), config.buckets) dep_testset = TextDataset(vocab.numericalize(dep_test), config.buckets) torch.save(vocab, config.vocab) torch.save(pos_trainset, os.path.join(config.file, 'pos_trainset')) torch.save(dep_trainset, os.path.join(config.file, 'dep_trainset')) torch.save(pos_devset, os.path.join(config.file, 'pos_devset')) torch.save(dep_devset, os.path.join(config.file, 'dep_devset')) torch.save(pos_testset, os.path.join(config.file, 'pos_testset')) torch.save(dep_testset, os.path.join(config.file, 'dep_testset')) else: print("Load the vocab") vocab = torch.load(config.vocab) print("Load the datasets") pos_trainset = torch.load(os.path.join(config.file, 'pos_trainset')) dep_trainset = torch.load(os.path.join(config.file, 'dep_trainset')) pos_devset = torch.load(os.path.join(config.file, 'pos_devset')) dep_devset = torch.load(os.path.join(config.file, 'dep_devset')) pos_testset = torch.load(os.path.join(config.file, 'pos_testset')) dep_testset = torch.load(os.path.join(config.file, 'dep_testset')) config.update({ 'n_words': vocab.n_init, 'n_chars': vocab.n_chars, 'n_pos_tags': vocab.n_pos_tags, 'n_dep_tags': vocab.n_dep_tags, 'n_rels': vocab.n_rels, 'pad_index': vocab.pad_index, 'unk_index': vocab.unk_index }) # set the data loaders pos_train_loader = batchify( pos_trainset, config.pos_batch_size // config.update_steps, True) dep_train_loader = batchify(dep_trainset, config.batch_size // config.update_steps, True) pos_dev_loader = batchify(pos_devset, config.pos_batch_size) dep_dev_loader = batchify(dep_devset, config.batch_size) pos_test_loader = batchify(pos_testset, config.pos_batch_size) dep_test_loader = batchify(dep_testset, config.batch_size) print(vocab) print(f"{'pos_train:':10} {len(pos_trainset):7} sentences in total, " f"{len(pos_train_loader):4} batches provided") print(f"{'dep_train:':10} {len(dep_trainset):7} sentences in total, " f"{len(dep_train_loader):4} batches provided") print(f"{'pos_dev:':10} {len(pos_devset):7} sentences in total, " f"{len(pos_dev_loader):4} batches provided") print(f"{'dep_dev:':10} {len(dep_devset):7} sentences in total, " f"{len(dep_dev_loader):4} batches provided") print(f"{'pos_test:':10} {len(pos_testset):7} sentences in total, " f"{len(pos_test_loader):4} batches provided") print(f"{'dep_test:':10} {len(dep_testset):7} sentences in total, " f"{len(dep_test_loader):4} batches provided") print("Create the model") parser = BiaffineParser(config, vocab.embed).to(config.device) print(f"{parser}\n") model = Model(config, vocab, parser) total_time = timedelta() best_e, best_metric = 1, AttachmentMethod() model.optimizer = Adam(model.parser.parameters(), config.lr, (config.mu, config.nu), config.epsilon) model.scheduler = ExponentialLR(model.optimizer, config.decay**(1 / config.decay_steps)) for epoch in range(1, config.epochs + 1): start = datetime.now() # train one epoch and update the parameters model.train(pos_train_loader, dep_train_loader) print(f"Epoch {epoch} / {config.epochs}:") lp, ld, mp, mdt, mdp = model.evaluate(None, dep_train_loader) print(f"{'train:':6} LP: {lp:.4f} LD: {ld:.4f} {mp} {mdt} {mdp}") lp, ld, mp, mdt, dev_m = model.evaluate(pos_dev_loader, dep_dev_loader) print(f"{'dev:':6} LP: {lp:.4f} LD: {ld:.4f} {mp} {mdt} {dev_m}") lp, ld, mp, mdt, mdp = model.evaluate(pos_test_loader, dep_test_loader) print(f"{'test:':6} LP: {lp:.4f} LD: {ld:.4f} {mp} {mdt} {mdp}") t = datetime.now() - start # save the model if it is the best so far if dev_m > best_metric and epoch > config.patience: best_e, best_metric = epoch, dev_m model.parser.save(config.model) print(f"{t}s elapsed (saved)\n") else: print(f"{t}s elapsed\n") total_time += t if epoch - best_e >= config.patience: break model.parser = BiaffineParser.load(config.model) lp, ld, mp, mdt, mdp = model.evaluate(pos_test_loader, dep_test_loader) print(f"max score of dev is {best_metric.score:.2%} at epoch {best_e}") print(f"the score of test at epoch {best_e} is {mdp.score:.2%}") print(f"average time of each epoch is {total_time / epoch}s") print(f"{total_time}s elapsed")