'how to generate oracle summaries, greedy or combination, combination will generate more accurate oracles but take much longer time.'
)
parser.add_argument("-map_path", default='../urls/')
parser.add_argument("-raw_path", default='../merged_stories_tokenized')
parser.add_argument("-save_path", default='../json_data/wx')
parser.add_argument("-shard_size", default=2000, type=int)
parser.add_argument('-min_nsents', default=3, type=int)
parser.add_argument('-max_nsents', default=100, type=int)
parser.add_argument('-min_src_ntokens', default=5, type=int)
parser.add_argument('-max_src_ntokens', default=200, type=int)
parser.add_argument("-lower",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument('-log_file', default='../logs/cnndm.log')
parser.add_argument(
'-dataset',
default='',
help='train, valid or test, defaul will process all datasets')
parser.add_argument('-n_cpus', default=2, type=int)
args = parser.parse_args()
init_logger(args.log_file)
eval('data_builder.' + args.mode + '(args)')
def train_abs_single(args, device_id):
init_logger(args.log_file)
logger.info(str(args))
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
else:
checkpoint = None
if (args.load_from_extractive != ''):
logger.info('Loading bert from extractive model %s' %
args.load_from_extractive)
bert_from_extractive = torch.load(
args.load_from_extractive,
map_location=lambda storage, loc: storage)
bert_from_extractive = bert_from_extractive['model']
else:
bert_from_extractive = None
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
def train_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args, 'train',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
model = Z_AbsSummarizer(args, device, checkpoint, bert_from_extractive)
if (args.sep_optim):
optim_bert = model_builder.build_optim_bert(args, model, checkpoint)
optim_dec = model_builder.build_optim_dec(args, model, checkpoint)
optim = [optim_bert, optim_dec]
else:
optim = [model_builder.build_optim(args, model, checkpoint)]
logger.info(model)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True,
cache_dir=args.temp_dir)
symbols = {
'BOS': tokenizer.vocab['[unused0]'],
'EOS': tokenizer.vocab['[unused1]'],
'PAD': tokenizer.vocab['[PAD]'],
'EOQ': tokenizer.vocab['[unused2]']
}
if COPY:
train_loss = abs_loss(model.generator,
symbols,
model.vocab_size,
device,
train=True,
label_smoothing=args.label_smoothing,
copy_generator=model.copy_generator)
else:
train_loss = abs_loss(model.generator,
symbols,
model.vocab_size,
device,
train=True,
label_smoothing=args.label_smoothing)
trainer = build_trainer(args, device_id, model, optim, train_loss)
trainer.train(train_iter_fct, args.train_steps)
def abs_train(args, device_id, pt, recover_all=False):
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
# load extractive model
if pt != None:
test_from = pt
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
config = BertConfig.from_json_file(args.bert_config_path)
# build extractive model
model = Summarizer(args,
device_id,
load_pretrained_bert=False,
bert_config=config)
# decoder
decoder = Decoder(model.bert.model.config.hidden_size // 2,
model.bert.model.config.vocab_size,
model.bert.model.config.hidden_size,
model.bert.model.embeddings,
device_id) # 2*hidden_dim = embedding_size
# get initial s_t
s_t_1 = get_initial_s(model.bert.model.config.hidden_size, device_id)
if recover_all:
model.load_cp(checkpoint)
s_t_1.load_cp(checkpoint)
decoder.load_cp(checkpoint)
optim = model_builder.build_optim(args, [model, decoder, s_t_1],
checkpoint)
elif pt != None:
model.load_cp(checkpoint)
optim = model_builder.build_optim(args, [model, decoder, s_t_1],
checkpoint)
else:
optim = model_builder.build_optim(args, [model, decoder, s_t_1], None)
# tokenizer,nlp
tokenizer = BertTokenizer.from_pretrained(
'bert-base-uncased',
do_lower_case=True,
never_split=('[SEP]', '[CLS]', '[PAD]', '[unused0]', '[unused1]',
'[unused2]', '[UNK]'),
no_word_piece=True)
nlp = StanfordCoreNLP(r'/home1/bqw/stanford-corenlp-full-2018-10-05')
# build optim
# load train dataset
def train_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args, 'train',
shuffle=True),
args.batch_size,
device_id,
shuffle=True,
is_test=False)
# build trainer
trainer = build_trainer(args,
device_id,
model,
optim,
decoder=decoder,
get_s_t=s_t_1,
device=device_id,
tokenizer=tokenizer,
nlp=nlp)
trainer.abs_train(train_iter_fct, args.train_steps)
def train_abs(args, device_id):
init_logger(args.log_file)
logger.info(str(args))
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in model_flags:
setattr(args, k, opt[k])
else:
checkpoint = None
if args.load_from_extractive != '':
logger.info('Loading bert from extractive model %s' %
args.load_from_extractive)
bert_from_extractive = torch.load(
args.load_from_extractive,
map_location=lambda storage, loc: storage)
bert_from_extractive = bert_from_extractive['model']
else:
bert_from_extractive = None
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
symbols, tokenizer = get_symbol_and_tokenizer(args.encoder, args.temp_dir)
model = AbsSummarizer(args,
device,
checkpoint,
bert_from_extractive,
symbols=symbols)
if args.sep_optim:
optim_enc = model_builder.build_optim_enc(args, model, checkpoint)
optim_dec = model_builder.build_optim_dec(args, model, checkpoint)
optim = [optim_enc, optim_dec]
else:
optim = [model_builder.build_optim(args, model, checkpoint)]
logger.info(model)
def train_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args, 'train',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False,
tokenizer=tokenizer)
train_loss = abs_loss(model.generator,
symbols,
model.vocab_size,
device,
train=True,
label_smoothing=args.label_smoothing)
trainer = build_trainer(args, device_id, model, optim, train_loss)
trainer.train(train_iter_fct, args.train_steps)
def load_model():
parser = argparse.ArgumentParser()
parser.add_argument("-task",
default='abs',
type=str,
choices=['ext', 'abs'])
parser.add_argument("-encoder",
default='bert',
type=str,
choices=['bert', 'baseline'])
parser.add_argument("-mode",
default='test',
type=str,
choices=['train', 'validate', 'test'])
parser.add_argument("-bert_data_path", default='../bert_data/cnndm')
parser.add_argument("-model_path", default='../models/')
parser.add_argument("-result_path", default='../results/cnndm')
parser.add_argument("-temp_dir", default='../../temp')
parser.add_argument("-batch_size", default=140, type=int)
parser.add_argument("-test_batch_size", default=200, type=int)
parser.add_argument("-max_pos", default=512, type=int)
parser.add_argument("-use_interval",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-large",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-load_from_extractive", default='', type=str)
parser.add_argument("-sep_optim",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-lr_bert", default=2e-3, type=float)
parser.add_argument("-lr_dec", default=2e-3, type=float)
parser.add_argument("-use_bert_emb",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-share_emb",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-finetune_bert",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-dec_dropout", default=0.2, type=float)
parser.add_argument("-dec_layers", default=6, type=int)
parser.add_argument("-dec_hidden_size", default=768, type=int)
parser.add_argument("-dec_heads", default=8, type=int)
parser.add_argument("-dec_ff_size", default=2048, type=int)
parser.add_argument("-enc_hidden_size", default=512, type=int)
parser.add_argument("-enc_ff_size", default=512, type=int)
parser.add_argument("-enc_dropout", default=0.2, type=float)
parser.add_argument("-enc_layers", default=6, type=int)
# params for EXT
parser.add_argument("-ext_dropout", default=0.2, type=float)
parser.add_argument("-ext_layers", default=2, type=int)
parser.add_argument("-ext_hidden_size", default=768, type=int)
parser.add_argument("-ext_heads", default=8, type=int)
parser.add_argument("-ext_ff_size", default=2048, type=int)
parser.add_argument("-label_smoothing", default=0.1, type=float)
parser.add_argument("-generator_shard_size", default=32, type=int)
parser.add_argument("-alpha", default=0.6, type=float)
parser.add_argument("-beam_size", default=5, type=int)
parser.add_argument("-min_length", default=15, type=int)
parser.add_argument("-max_length", default=150, type=int)
parser.add_argument("-max_tgt_len", default=140, type=int)
# params for preprocessing
parser.add_argument("-shard_size", default=2000, type=int)
parser.add_argument('-min_src_nsents', default=3, type=int)
parser.add_argument('-max_src_nsents', default=100, type=int)
parser.add_argument('-min_src_ntokens_per_sent', default=5, type=int)
parser.add_argument('-max_src_ntokens_per_sent', default=200, type=int)
parser.add_argument('-min_tgt_ntokens', default=5, type=int)
parser.add_argument('-max_tgt_ntokens', default=500, type=int)
parser.add_argument("-lower",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-use_bert_basic_tokenizer",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-param_init", default=0, type=float)
parser.add_argument("-param_init_glorot",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-optim", default='adam', type=str)
parser.add_argument("-lr", default=1, type=float)
parser.add_argument("-beta1", default=0.9, type=float)
parser.add_argument("-beta2", default=0.999, type=float)
parser.add_argument("-warmup_steps", default=8000, type=int)
parser.add_argument("-warmup_steps_bert", default=8000, type=int)
parser.add_argument("-warmup_steps_dec", default=8000, type=int)
parser.add_argument("-max_grad_norm", default=0, type=float)
parser.add_argument("-save_checkpoint_steps", default=5, type=int)
parser.add_argument("-accum_count", default=1, type=int)
parser.add_argument("-report_every", default=1, type=int)
parser.add_argument("-train_steps", default=1000, type=int)
parser.add_argument("-recall_eval",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument('-visible_gpus', default='-1', type=str)
parser.add_argument('-gpu_ranks', default='0', type=str)
parser.add_argument('-log_file', default='../logs/cnndm.log')
parser.add_argument('-seed', default=666, type=int)
parser.add_argument("-test_all",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-test_start_from", default=-1, type=int)
parser.add_argument("-train_from", default='')
parser.add_argument("-report_rouge",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-block_trigram",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-test_from", default='../models/model_step_148000.pt')
args = parser.parse_args()
args.gpu_ranks = [int(i) for i in range(len(args.visible_gpus.split(',')))]
args.world_size = len(args.gpu_ranks)
os.environ["CUDA_VISIBLE_DEVICES"] = args.visible_gpus
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
device_id = 0 if device == "cuda" else -1
print(args.task, args.mode)
cp = '../models/model_step_148000.pt'
try:
step = int(cp.split('.')[-2].split('_')[-1])
except:
step = 0
predictor = load_models_abs(args, device_id, cp, step)
return args, device_id, cp, step, predictor