'how to generate oracle summaries, greedy or combination, combination will generate more accurate oracles but take much longer time.' ) parser.add_argument("-map_path", default='../urls/') parser.add_argument("-raw_path", default='../merged_stories_tokenized') parser.add_argument("-save_path", default='../json_data/wx') parser.add_argument("-shard_size", default=2000, type=int) parser.add_argument('-min_nsents', default=3, type=int) parser.add_argument('-max_nsents', default=100, type=int) parser.add_argument('-min_src_ntokens', default=5, type=int) parser.add_argument('-max_src_ntokens', default=200, type=int) parser.add_argument("-lower", type=str2bool, nargs='?', const=True, default=True) parser.add_argument('-log_file', default='../logs/cnndm.log') parser.add_argument( '-dataset', default='', help='train, valid or test, defaul will process all datasets') parser.add_argument('-n_cpus', default=2, type=int) args = parser.parse_args() init_logger(args.log_file) eval('data_builder.' + args.mode + '(args)')
def train_abs_single(args, device_id): init_logger(args.log_file) logger.info(str(args)) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) else: checkpoint = None if (args.load_from_extractive != ''): logger.info('Loading bert from extractive model %s' % args.load_from_extractive) bert_from_extractive = torch.load( args.load_from_extractive, map_location=lambda storage, loc: storage) bert_from_extractive = bert_from_extractive['model'] else: bert_from_extractive = None torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) model = Z_AbsSummarizer(args, device, checkpoint, bert_from_extractive) if (args.sep_optim): optim_bert = model_builder.build_optim_bert(args, model, checkpoint) optim_dec = model_builder.build_optim_dec(args, model, checkpoint) optim = [optim_bert, optim_dec] else: optim = [model_builder.build_optim(args, model, checkpoint)] logger.info(model) tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir) symbols = { 'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'], 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]'] } if COPY: train_loss = abs_loss(model.generator, symbols, model.vocab_size, device, train=True, label_smoothing=args.label_smoothing, copy_generator=model.copy_generator) else: train_loss = abs_loss(model.generator, symbols, model.vocab_size, device, train=True, label_smoothing=args.label_smoothing) trainer = build_trainer(args, device_id, model, optim, train_loss) trainer.train(train_iter_fct, args.train_steps)
def abs_train(args, device_id, pt, recover_all=False): init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True # load extractive model if pt != None: test_from = pt logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) config = BertConfig.from_json_file(args.bert_config_path) # build extractive model model = Summarizer(args, device_id, load_pretrained_bert=False, bert_config=config) # decoder decoder = Decoder(model.bert.model.config.hidden_size // 2, model.bert.model.config.vocab_size, model.bert.model.config.hidden_size, model.bert.model.embeddings, device_id) # 2*hidden_dim = embedding_size # get initial s_t s_t_1 = get_initial_s(model.bert.model.config.hidden_size, device_id) if recover_all: model.load_cp(checkpoint) s_t_1.load_cp(checkpoint) decoder.load_cp(checkpoint) optim = model_builder.build_optim(args, [model, decoder, s_t_1], checkpoint) elif pt != None: model.load_cp(checkpoint) optim = model_builder.build_optim(args, [model, decoder, s_t_1], checkpoint) else: optim = model_builder.build_optim(args, [model, decoder, s_t_1], None) # tokenizer,nlp tokenizer = BertTokenizer.from_pretrained( 'bert-base-uncased', do_lower_case=True, never_split=('[SEP]', '[CLS]', '[PAD]', '[unused0]', '[unused1]', '[unused2]', '[UNK]'), no_word_piece=True) nlp = StanfordCoreNLP(r'/home1/bqw/stanford-corenlp-full-2018-10-05') # build optim # load train dataset def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device_id, shuffle=True, is_test=False) # build trainer trainer = build_trainer(args, device_id, model, optim, decoder=decoder, get_s_t=s_t_1, device=device_id, tokenizer=tokenizer, nlp=nlp) trainer.abs_train(train_iter_fct, args.train_steps)
def train_abs(args, device_id): init_logger(args.log_file) logger.info(str(args)) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in model_flags: setattr(args, k, opt[k]) else: checkpoint = None if args.load_from_extractive != '': logger.info('Loading bert from extractive model %s' % args.load_from_extractive) bert_from_extractive = torch.load( args.load_from_extractive, map_location=lambda storage, loc: storage) bert_from_extractive = bert_from_extractive['model'] else: bert_from_extractive = None torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True symbols, tokenizer = get_symbol_and_tokenizer(args.encoder, args.temp_dir) model = AbsSummarizer(args, device, checkpoint, bert_from_extractive, symbols=symbols) if args.sep_optim: optim_enc = model_builder.build_optim_enc(args, model, checkpoint) optim_dec = model_builder.build_optim_dec(args, model, checkpoint) optim = [optim_enc, optim_dec] else: optim = [model_builder.build_optim(args, model, checkpoint)] logger.info(model) def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False, tokenizer=tokenizer) train_loss = abs_loss(model.generator, symbols, model.vocab_size, device, train=True, label_smoothing=args.label_smoothing) trainer = build_trainer(args, device_id, model, optim, train_loss) trainer.train(train_iter_fct, args.train_steps)
def load_model(): parser = argparse.ArgumentParser() parser.add_argument("-task", default='abs', type=str, choices=['ext', 'abs']) parser.add_argument("-encoder", default='bert', type=str, choices=['bert', 'baseline']) parser.add_argument("-mode", default='test', type=str, choices=['train', 'validate', 'test']) parser.add_argument("-bert_data_path", default='../bert_data/cnndm') parser.add_argument("-model_path", default='../models/') parser.add_argument("-result_path", default='../results/cnndm') parser.add_argument("-temp_dir", default='../../temp') parser.add_argument("-batch_size", default=140, type=int) parser.add_argument("-test_batch_size", default=200, type=int) parser.add_argument("-max_pos", default=512, type=int) parser.add_argument("-use_interval", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-large", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-load_from_extractive", default='', type=str) parser.add_argument("-sep_optim", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-lr_bert", default=2e-3, type=float) parser.add_argument("-lr_dec", default=2e-3, type=float) parser.add_argument("-use_bert_emb", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-share_emb", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-finetune_bert", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-dec_dropout", default=0.2, type=float) parser.add_argument("-dec_layers", default=6, type=int) parser.add_argument("-dec_hidden_size", default=768, type=int) parser.add_argument("-dec_heads", default=8, type=int) parser.add_argument("-dec_ff_size", default=2048, type=int) parser.add_argument("-enc_hidden_size", default=512, type=int) parser.add_argument("-enc_ff_size", default=512, type=int) parser.add_argument("-enc_dropout", default=0.2, type=float) parser.add_argument("-enc_layers", default=6, type=int) # params for EXT parser.add_argument("-ext_dropout", default=0.2, type=float) parser.add_argument("-ext_layers", default=2, type=int) parser.add_argument("-ext_hidden_size", default=768, type=int) parser.add_argument("-ext_heads", default=8, type=int) parser.add_argument("-ext_ff_size", default=2048, type=int) parser.add_argument("-label_smoothing", default=0.1, type=float) parser.add_argument("-generator_shard_size", default=32, type=int) parser.add_argument("-alpha", default=0.6, type=float) parser.add_argument("-beam_size", default=5, type=int) parser.add_argument("-min_length", default=15, type=int) parser.add_argument("-max_length", default=150, type=int) parser.add_argument("-max_tgt_len", default=140, type=int) # params for preprocessing parser.add_argument("-shard_size", default=2000, type=int) parser.add_argument('-min_src_nsents', default=3, type=int) parser.add_argument('-max_src_nsents', default=100, type=int) parser.add_argument('-min_src_ntokens_per_sent', default=5, type=int) parser.add_argument('-max_src_ntokens_per_sent', default=200, type=int) parser.add_argument('-min_tgt_ntokens', default=5, type=int) parser.add_argument('-max_tgt_ntokens', default=500, type=int) parser.add_argument("-lower", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-use_bert_basic_tokenizer", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-param_init", default=0, type=float) parser.add_argument("-param_init_glorot", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-optim", default='adam', type=str) parser.add_argument("-lr", default=1, type=float) parser.add_argument("-beta1", default=0.9, type=float) parser.add_argument("-beta2", default=0.999, type=float) parser.add_argument("-warmup_steps", default=8000, type=int) parser.add_argument("-warmup_steps_bert", default=8000, type=int) parser.add_argument("-warmup_steps_dec", default=8000, type=int) parser.add_argument("-max_grad_norm", default=0, type=float) parser.add_argument("-save_checkpoint_steps", default=5, type=int) parser.add_argument("-accum_count", default=1, type=int) parser.add_argument("-report_every", default=1, type=int) parser.add_argument("-train_steps", default=1000, type=int) parser.add_argument("-recall_eval", type=str2bool, nargs='?', const=True, default=False) parser.add_argument('-visible_gpus', default='-1', type=str) parser.add_argument('-gpu_ranks', default='0', type=str) parser.add_argument('-log_file', default='../logs/cnndm.log') parser.add_argument('-seed', default=666, type=int) parser.add_argument("-test_all", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-test_start_from", default=-1, type=int) parser.add_argument("-train_from", default='') parser.add_argument("-report_rouge", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-block_trigram", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-test_from", default='../models/model_step_148000.pt') args = parser.parse_args() args.gpu_ranks = [int(i) for i in range(len(args.visible_gpus.split(',')))] args.world_size = len(args.gpu_ranks) os.environ["CUDA_VISIBLE_DEVICES"] = args.visible_gpus init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" device_id = 0 if device == "cuda" else -1 print(args.task, args.mode) cp = '../models/model_step_148000.pt' try: step = int(cp.split('.')[-2].split('_')[-1]) except: step = 0 predictor = load_models_abs(args, device_id, cp, step) return args, device_id, cp, step, predictor