def gen_features_vector(self, step=None): if not step: try: step = int(self.args.test_from.split('.')[-2].split('_')[-1]) except IndexError: step = 0 logger.info('Loading checkpoint from %s' % self.args.test_from) checkpoint = torch.load(self.args.test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in self.model_flags: setattr(self.args, k, opt[k]) config = BertConfig.from_json_file(self.args.bert_config_path) model = model_builder.Summarizer(self.args, self.device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() # logger.info(model) trainer = build_trainer(self.args, self.device_id, model, None) test_iter = data_loader.DataLoader(self.args, data_loader.load_dataset( self.args, 'test', shuffle=False), self.args.batch_size, self.device, shuffle=False, is_test=True) trainer.gen_features_vector(test_iter, step)
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" if pt != '': test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in model_flags: setattr(args, k, opt[k]) print(args) config = XLNetConfig.from_pretrained(args.config_path) model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() valid_iter = Dataloader(args, load_dataset(args, 'valid', shuffle=False), args.batch_size, device, shuffle=False, is_test=False) trainer = build_trainer(args, device_id, model, None) stats = trainer.validate(valid_iter, step) return stats.xent()
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" #if (pt != ''): if not (args.test_from): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) model = AbsSummarizer(args, device, checkpoint) model.eval() valid_iter = data_loader.Dataloader(args, load_dataset(args, 'valid', shuffle=False), args.batch_size, device, shuffle=False, is_test=False) tokenizer = BertTokenizer.from_pretrained( '../ETRI_koBERT/003_bert_eojeol_pytorch/vocab.txt', do_lower_case=False, cache_dir=args.temp_dir) if not args.share_emb: tokenizer = add_tokens(tokenizer) symbols = { 'BOS': tokenizer.vocab['<S>'], 'EOS': tokenizer.vocab['<T>'], 'PAD': tokenizer.vocab['[PAD]'] } # symbols = {'BOS': tokenizer.vocab['[BOS]'], 'EOS': tokenizer.vocab['[EOS]'], # 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[EOQ]']} # symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'], # 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']} # print(tokenizer.vocab_size) # print([(key, value) for key, value in tokenizer.vocab.items()][-10:]) # exit() valid_loss = abs_loss(model.generator, symbols, model.vocab_size, train=False, device=device) trainer = build_trainer(args, device_id, model, None, valid_loss) stats = trainer.validate(valid_iter, step) return stats.xent()
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpu == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) spm = sentencepiece.SentencePieceProcessor() spm.Load(args.vocab_path) word_padding_idx = spm.PieceToId('<PAD>') vocab_size = len(spm) model = Summarizer(args, word_padding_idx, vocab_size, device, checkpoint) model.eval() valid_iter = data_loader.Dataloader(args, load_dataset(args, 'valid', shuffle=False), {'PAD': word_padding_idx}, args.batch_size, device, shuffle=False, is_test=False) trainer = build_trainer(args, device_id, model, None) stats = trainer.validate(valid_iter) trainer._report_step(0, step, valid_stats=stats) return stats.xent()
def test(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) config = BertConfig.from_json_file(args.bert_config_path) model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False), args.batch_size, device, shuffle=False, is_test=True) trainer = build_trainer(args, device_id, model, None) trainer.test(test_iter, step)
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) model = AbsSummarizer(args, device, checkpoint) model.eval() valid_iter = data_loader.Dataloader(args, load_dataset(args, 'valid', shuffle=False), args.batch_size, device, shuffle=False, is_test=False) tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir) symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'], 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']} valid_loss = abs_loss(model.generator, symbols, model.vocab_size, train=False, device=device) trainer = build_trainer(args, device_id, model, None, valid_loss) stats = trainer.validate(valid_iter, step) return stats.xent()
def validate(self, step): logger.info('Loading checkpoint from %s' % self.args.validate_from) checkpoint = torch.load(self.args.validate_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in self.model_flags: setattr(self.args, k, opt[k]) print(self.args) config = BertConfig.from_json_file(self.args.bert_config_path) model = model_builder.Summarizer(self.args, self.device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() valid_iter = data_loader.DataLoader(self.args, data_loader.load_dataset( self.args, 'valid', shuffle=False), self.args.batch_size, self.device, shuffle=False, is_test=False) trainer = build_trainer(self.args, self.device_id, model, None) stats = trainer.validate(valid_iter, step) return stats.xent()
def validate(args, device_id, pt, step, tokenizer): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) model = AbsSummarizer(args, device, checkpoint) model.eval() valid_iter = data_loader.Dataloader(args, load_dataset(args, 'dev', shuffle=False), args.batch_size, device, shuffle=False, is_test=False) symbols = {'BOS': tokenizer.convert_tokens_to_ids('<s>'), 'EOS': tokenizer.convert_tokens_to_ids('</s>'), 'PAD': tokenizer.convert_tokens_to_ids('[PAD]')} valid_loss = abs_loss(model.generator, symbols, model.vocab_size, train=False, device=device) trainer = build_trainer(args, device_id, model, None, valid_loss) stats = trainer.validate(valid_iter, step) return stats.xent()
def train_abs_single(args, device_id): init_logger(args.log_file) logger.info(str(args)) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) else: checkpoint = None if (args.load_from_extractive != ''): logger.info('Loading bert from extractive model %s' % args.load_from_extractive) bert_from_extractive = torch.load(args.load_from_extractive, map_location=lambda storage, loc: storage) bert_from_extractive = bert_from_extractive['model'] else: bert_from_extractive = None torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) model = AbsSummarizer(args, device, checkpoint, bert_from_extractive) if (args.sep_optim): optim_bert = model_builder.build_optim_bert(args, model, checkpoint) optim_dec = model_builder.build_optim_dec(args, model, checkpoint) optim = [optim_bert, optim_dec] else: optim = [model_builder.build_optim(args, model, checkpoint)] logger.info(model) tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir) symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'], 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']} train_loss = abs_loss(model.generator, symbols, model.vocab_size, device, train=True, label_smoothing=args.label_smoothing) trainer = build_trainer(args, device_id, model, optim, train_loss) trainer.train(train_iter_fct, args.train_steps)
def train(args, device_id): init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) # temp change for reducing gpu memory model = Summarizer(args, device, load_pretrained_bert=True) #config = BertConfig.from_json_file(args.bert_config_path) #model = Summarizer(args, device, load_pretrained_bert=False, bert_config = config) if args.train_from != '': #train another part from beginning logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) model.load_cp(checkpoint, strict=False) # keys can not match #optim = model_builder.build_optim(args, model, checkpoint) optim = model_builder.build_optim(args, model, None) if args.model_name == "ctx" and args.fix_scorer: logger.info("fix the saliency scorer") #for param in self.bert.model.parameters(): for param in model.parameters(): param.requires_grad = False if hasattr(model.encoder, "selector") and model.encoder.selector is not None: for param in model.encoder.selector.parameters(): param.requires_grad = True #print([p for p in model.parameters() if p.requires_grad]) else: optim = model_builder.build_optim(args, model, None) logger.info(model) trainer = build_trainer(args, device_id, model, optim) _, neg_valid_loss = trainer.train(train_iter_fct, args.train_steps) while len(neg_valid_loss) > 0: #from 3rd to 2nd to 1st. neg_loss, saved_model = heapq.heappop(neg_valid_loss) print(-neg_loss, saved_model) step = int(saved_model.split('.')[-2].split('_')[-1]) test(args, device_id, saved_model, step) logger.info("Finish!")
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) model = AbsSummarizer(args, device, checkpoint) model.eval() valid_iter = data_loader.Dataloader(args, load_dataset(args, 'valid', shuffle=False), args.batch_size, device, shuffle=False, is_test=False) parser = argparse.ArgumentParser() parser.add_argument('--bpe-codes', default="/content/PhoBERT_base_transformers/bpe.codes", required=False, type=str, help='path to fastBPE BPE') args1, unknown = parser.parse_known_args() bpe = fastBPE(args1) # Load the dictionary vocab = Dictionary() vocab.add_from_file("/content/PhoBERT_base_transformers/dict.txt") tokenizer = bpe symbols = { 'BOS': vocab.indices['[unused0]'], 'EOS': vocab.indices['[unused1]'], 'PAD': vocab.indices['[PAD]'], 'EOQ': vocab.indices['[unused2]'] } valid_loss = abs_loss(model.generator, symbols, model.vocab_size, train=False, device=device) trainer = build_trainer(args, device_id, model, None, valid_loss) stats = trainer.validate(valid_iter, step) return stats.xent()
def train(args, device_id): # Start logger. init_logger(args.log_file) # Configure training device. device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) # Configure manual seed. torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True # Set CUDA device. if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) # Configure manual seed. torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True # Dataloader used for training. def train_iter_fct(): return Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) # Build the model. model = Summarizer(args, device, load_pretrained=True) # Configure the checkpoint. if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in model_flags: setattr(args, k, opt[k]) model.load_cp(checkpoint) optim = builder.build_optim(args, model, checkpoint) else: optim = builder.build_optim(args, model, None) logger.info(model) # Train the model trainer = build_trainer(args, device_id, model, optim) trainer.train(train_iter_fct, args.train_steps)
def baseline(args, cal_lead=False, cal_oracle=False): test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False), args.batch_size, 'cpu', shuffle=False, is_test=True) trainer = build_trainer(args, '-1', None, None, None) # if (cal_lead): trainer.test(test_iter, 0, cal_lead=True) elif (cal_oracle): trainer.test(test_iter, 0, cal_oracle=True)
def predict(self): test_iter = data_loader.DataLoader(self.args, data_loader.load_dataset( self.args, 'test', shuffle=False), self.args.batch_size, self.device, shuffle=False, is_test=True) trainer = build_trainer(self.args, self.device_id, self.model, None) trainer.predict(test_iter, self.step)
def train_abs_single(args, device_id): init_logger(args.log_file) logger.info(str(args)) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) else: checkpoint = None torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True def train_iter_fct(): return data_pretrain.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) model = PretrainModel(args, device, checkpoint) optim = [model_builder.build_optim(args, model, checkpoint)] logger.info(model) symbols = {'PAD': 0} train_loss = abs_loss(model.generator, symbols, model.dec_vocab_size, device, train=True, label_smoothing=args.label_smoothing) trainer = build_trainer(args, device_id, model, optim, train_loss) trainer.train(train_iter_fct, args.train_steps)
def train(args, device_id): # import ipdb; ipdb.set_trace() # import pdb; pdb.set_trace() init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: print("device_id = " + str(device_id)) torch.cuda.set_device(device_id) print("device set") torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) model = Summarizer(args, device, load_pretrained_bert=True) if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) model.load_cp(checkpoint) optim = model_builder.build_optim(args, model, checkpoint) else: optim = model_builder.build_optim(args, model, None) logger.info(model) trainer = build_trainer(args, device_id, model, optim) with comet_experiment.train(): trainer.train(train_iter_fct, args.train_steps)
def train(args, device_id): init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True logger.info('data begin load %s', time.strftime('%H:%M:%S', time.localtime(time.time()))) def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) logger.info('data end load %s', time.strftime('%H:%M:%S', time.localtime(time.time()))) model = Summarizer(args, device, load_pretrained_bert=True) if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) model.load_cp(checkpoint) optim = model_builder.build_optim(args, model, checkpoint) else: optim = model_builder.build_optim(args, model, None) logger.info(model) trainer = build_trainer(args, device_id, model, optim) trainer.train(train_iter_fct, args.train_steps)
def baseline(args, cal_lead=False, cal_oracle=False): test_iter = data_loader.Dataloader( args, load_dataset(args, "test", shuffle=False), args.batch_size, "cpu", shuffle=False, is_test=True, ) trainer = build_trainer(args, "-1", None, None, None) if cal_lead: trainer.test(test_iter, 0, cal_lead=True) elif cal_oracle: trainer.test(test_iter, 0, cal_oracle=True)
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpus == "-1" else "cuda" if pt != "": test_from = pt else: test_from = args.test_from logger.info("Loading checkpoint from %s" % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint["opt"]) for k in opt.keys(): if k in model_flags: setattr(args, k, opt[k]) print(args) model = AbsSummarizer(args, device, checkpoint) model.eval() valid_iter = data_loader.Dataloader( args, load_dataset(args, "valid", shuffle=False), args.batch_size, device, shuffle=False, is_test=False, ) tokenizer = BertTokenizer.from_pretrained( "chinese_roberta_wwm_ext_pytorch", do_lower_case=True, cache_dir=args.temp_dir) symbols = { "BOS": tokenizer.vocab["[unused1]"], "EOS": tokenizer.vocab["[unused2]"], "PAD": tokenizer.vocab["[PAD]"], "EOQ": tokenizer.vocab["[unused3]"], } valid_loss = abs_loss(model.generator, symbols, model.vocab_size, train=False, device=device) trainer = build_trainer(args, device_id, model, None, valid_loss) stats = trainer.validate(valid_iter, step) return stats.xent()
def train(args, device_id): init_logger(args.log_file) device = "cpu" if args.visible_gpus == "-1" else "cuda" logger.info("Device ID %d" % device_id) logger.info("Device %s" % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True def train_iter_fct(): return data_loader.Dataloader( args, load_dataset(args, "train", shuffle=True), args.batch_size, device, shuffle=True, is_test=False, ) model = Summarizer(args, device, load_pretrained_bert=True) if args.train_from != "": logger.info("Loading checkpoint from %s" % args.train_from) checkpoint = torch.load( args.train_from, map_location=lambda storage, loc: storage ) opt = vars(checkpoint["opt"]) for k in opt.keys(): if k in model_flags: setattr(args, k, opt[k]) model.load_cp(checkpoint) optim = model_builder.build_optim(args, model, checkpoint) else: optim = model_builder.build_optim(args, model, None) logger.info(model) trainer = build_trainer(args, device_id, model, optim) trainer.train(train_iter_fct, args.train_steps)
def baseline(self, cal_lead=False, cal_oracle=False): test_iter = data_loader.DataLoader(self.args, data_loader.load_dataset( self.args, 'test', shuffle=False), self.args.batch_size, self.device, shuffle=False, is_test=True) trainer = build_trainer(self.args, self.device_id, None, None) if cal_lead: trainer.test(test_iter, 0, cal_lead=True) elif cal_oracle: trainer.test(test_iter, 0, cal_oracle=True)
def oracle(args, device_id, pt): test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False), args.batch_size, device, shuffle=False, is_test=True) trainer = build_trainer(args, device_id, None, None) print( '___________________________________________________________________________' ) trainer.oracle(test_iter, cal_lead=False, cal_oracle=False, each=True)
def test_detector(self, loader, run_gen=False, portion='all'): """ Testing detector """ test = TreeDataset(self.args, loader, self.dataname, self.device, self.tokenizer) #test = CommentDataset(self.args, loader, self.dataname, self.device, self.tokenizer) data_iter = Iterator(test, train=False, device=self.device, batch_size=len(test) if len(test)<self.bs else self.bs, sort_key=lambda x: len(x.src), sort_within_batch=False) # Define trainer train_loss = abs_loss(self.args.label_num, self.maxepoch, self.device, train=False) trainer = build_trainer(self.args, self.model, self.optim, train_loss) logger.info('Test on best model (stage-1)') best_model = os.path.join(self.args.savepath, 'best_model.pt') if os.path.exists(best_model): try: self.model.load_state_dict(torch.load(best_model, map_location=lambda storage, loc: storage)['model']) except: self.model.load_state_dict(torch.load(best_model, map_location=lambda storage, loc: storage)['model'], strict=False) logger.info('[Warning] The keys in state dict do not strictly match') test_stat = trainer.testing(data_iter, tokenizer=self.tokenizer, gen_flag=False, info="Without Generated Response >>", write_type="w") test_stat.write_results(os.path.join(self.args.savepath, 'result_test.csv'), 'test-'+portion, self.args.label_num) if self.args.test_adv: logger.info('Test on adversarially-trained model (stage-2)') best_model = os.path.join(self.args.savepath, 'best_adv_model.pt') if os.path.exists(best_model): try: self.model.load_state_dict(torch.load(best_model, map_location=lambda storage, loc: storage)['model']) except: self.model.load_state_dict(torch.load(best_model, map_location=lambda storage, loc: storage)['model'], strict=False) logger.info('[Warning] The keys in state dict do not strictly match') test_stat = trainer.testing(data_iter, tokenizer=self.tokenizer, gen_flag=True, info="\nWith Generated Response from {} >>".format(best_model.split("/")[-1]), write_type=="a") predictor = build_predictor(self.args, self.model, self.tokenizer, self.symbols, logger) predictor.translate(data_iter, 'best', have_gold=False)
def exp(self, loader, exp_pos=False): """ Testing detector """ test = TreeDataset(self.args, loader, self.dataname, self.device, self.tokenizer) data_iter = Iterator(test, train=False, device=self.device, batch_size=self.bs, sort_key=lambda x: len(x.src), sort_within_batch=False) best_model = os.path.join(self.args.savepath, 'best_adv_model.pt') # Define trainer loss = abs_loss(self.args.label_num, self.maxepoch, self.device, train=False) trainer = build_trainer(self.args, self.model, self.optim, loss) test_stats = trainer.testing(data_iter, -1) if exp_pos: if os.path.exists(best_model): self.model.load_state_dict(torch.load(best_model, map_location=lambda storage, loc: storage)['model'], strict=True) test_stat = trainer.exp_pos(data_iter)
def train(args, device_id): init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) else: checkpoint = None torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True def train_iter_fct(): # return data_loader.AbstractiveDataloader(load_dataset('train', True), symbols, FLAGS.batch_size, device, True) return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) model = Summarizer(args, device, checkpoint) # optim = model_builder.build_optim(args, model.reg, checkpoint) optim = model_builder.build_optim(args, model, checkpoint) # optim = BertAdam() logger.info(model) trainer = build_trainer(args, device_id, model, optim) # trainer.train(train_iter_fct, args.train_steps)
def train(self): model = model_builder.Summarizer(self.args, self.device, load_pretrained_bert=True) if self.args.train_from: logger.info('Loading checkpoint from %s' % self.args.train_from) checkpoint = torch.load(self.args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in self.model_flags: setattr(self.args, k, opt[k]) model.load_cp(checkpoint) optimizer = model_builder.build_optim(self.args, model, checkpoint) else: optimizer = model_builder.build_optim(self.args, model, None) logger.info(model) trainer = build_trainer(self.args, self.device_id, model, optimizer) trainer.train(self.train_iter, self.args.train_steps)
def train(args, device_id): init_logger(args.log_file) if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) else: checkpoint = None torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True spm = sentencepiece.SentencePieceProcessor() spm.Load(args.vocab_path) word_padding_idx = spm.PieceToId('<PAD>') vocab_size = len(spm) def train_iter_fct(): # return data_loader.AbstractiveDataloader(load_dataset('train', True), symbols, FLAGS.batch_size, device, True) return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), {'PAD': word_padding_idx}, args.batch_size, device, shuffle=True, is_test=False) model = Summarizer(args, word_padding_idx, vocab_size, device, checkpoint) optim = model_builder.build_optim(args, model, checkpoint) logger.info(model) trainer = build_trainer(args, device_id, model, optim) # trainer.train(train_iter_fct, args.train_steps)
def single_train(args, device_id): init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: # 使用指定的gpu torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) else: checkpoint = None def train_iter_method(): return DataLoaderBert(load_dataset(args, 'train', shuffle=True), args.batch_size, shuffle=True, is_test=False) model = NextSentencePrediction(args, device, checkpoint) optim = build_optim(args, model, checkpoint) logger.info(model) trainer = build_trainer(args, device_id, model, optim) trainer.train(train_iter_method, args.train_steps)
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) model = HMMModel(args, device, checkpoint) model.eval() valid_iter = data_hmm.Dataloader(args, load_dataset(args, 'dev', shuffle=False), args.batch_size, device, shuffle=False, is_test=False) valid_loss = hmm_loss(model.generator, args.pad_id, args.relation_path, args.fake_global, model.dec_vocab_size, device, train=False) trainer = build_trainer(args, device_id, model, None, valid_loss) stats = trainer.validate(valid_iter, step) return stats.xent()
def train(args, device_id): """ Starts training pipeline given CLI args """ device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) seed(args.seed, device_id) def train_iter_fct(): return data.load(args, 'train', device) if args.train_from != '': logger.info("Training from checkpoint %s", args.train_from) model, optim = load_model(args, device, load_bert=True, checkpoint=args.train_from) else: logger.info("Training without checkpoint") model, optim = load_model(args, device, load_bert=True) logger.info(model) trainer = build_trainer(args, device_id, model, optim) trainer.train(train_iter_fct, args.train_steps)