def gen_features_vector(self, step=None):
if not step:
try:
step = int(self.args.test_from.split('.')[-2].split('_')[-1])
except IndexError:
step = 0
logger.info('Loading checkpoint from %s' % self.args.test_from)
checkpoint = torch.load(self.args.test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in self.model_flags:
setattr(self.args, k, opt[k])
config = BertConfig.from_json_file(self.args.bert_config_path)
model = model_builder.Summarizer(self.args,
self.device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
# logger.info(model)
trainer = build_trainer(self.args, self.device_id, model, None)
test_iter = data_loader.DataLoader(self.args,
data_loader.load_dataset(
self.args,
'test',
shuffle=False),
self.args.batch_size,
self.device,
shuffle=False,
is_test=True)
trainer.gen_features_vector(test_iter, step)
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if pt != '':
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in model_flags:
setattr(args, k, opt[k])
print(args)
config = XLNetConfig.from_pretrained(args.config_path)
model = Summarizer(args,
device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
valid_iter = Dataloader(args,
load_dataset(args, 'valid', shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=False)
trainer = build_trainer(args, device_id, model, None)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
#if (pt != ''):
if not (args.test_from):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
model = AbsSummarizer(args, device, checkpoint)
model.eval()
valid_iter = data_loader.Dataloader(args,
load_dataset(args,
'valid',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=False)
tokenizer = BertTokenizer.from_pretrained(
'../ETRI_koBERT/003_bert_eojeol_pytorch/vocab.txt',
do_lower_case=False,
cache_dir=args.temp_dir)
if not args.share_emb:
tokenizer = add_tokens(tokenizer)
symbols = {
'BOS': tokenizer.vocab['<S>'],
'EOS': tokenizer.vocab['<T>'],
'PAD': tokenizer.vocab['[PAD]']
}
# symbols = {'BOS': tokenizer.vocab['[BOS]'], 'EOS': tokenizer.vocab['[EOS]'],
# 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[EOQ]']}
# symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'],
# 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']}
# print(tokenizer.vocab_size)
# print([(key, value) for key, value in tokenizer.vocab.items()][-10:])
# exit()
valid_loss = abs_loss(model.generator,
symbols,
model.vocab_size,
train=False,
device=device)
trainer = build_trainer(args, device_id, model, None, valid_loss)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpu == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
spm = sentencepiece.SentencePieceProcessor()
spm.Load(args.vocab_path)
word_padding_idx = spm.PieceToId('<PAD>')
vocab_size = len(spm)
model = Summarizer(args, word_padding_idx, vocab_size, device, checkpoint)
model.eval()
valid_iter = data_loader.Dataloader(args,
load_dataset(args,
'valid',
shuffle=False),
{'PAD': word_padding_idx},
args.batch_size,
device,
shuffle=False,
is_test=False)
trainer = build_trainer(args, device_id, model, None)
stats = trainer.validate(valid_iter)
trainer._report_step(0, step, valid_stats=stats)
return stats.xent()
def test(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
config = BertConfig.from_json_file(args.bert_config_path)
model = Summarizer(args,
device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
test_iter = data_loader.Dataloader(args,
load_dataset(args,
'test',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=True)
trainer = build_trainer(args, device_id, model, None)
trainer.test(test_iter, step)
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
model = AbsSummarizer(args, device, checkpoint)
model.eval()
valid_iter = data_loader.Dataloader(args, load_dataset(args, 'valid', shuffle=False),
args.batch_size, device,
shuffle=False, is_test=False)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir)
symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'],
'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']}
valid_loss = abs_loss(model.generator, symbols, model.vocab_size, train=False, device=device)
trainer = build_trainer(args, device_id, model, None, valid_loss)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def validate(self, step):
logger.info('Loading checkpoint from %s' % self.args.validate_from)
checkpoint = torch.load(self.args.validate_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in self.model_flags:
setattr(self.args, k, opt[k])
print(self.args)
config = BertConfig.from_json_file(self.args.bert_config_path)
model = model_builder.Summarizer(self.args,
self.device,
load_pretrained_bert=False,
bert_config=config)
model.load_cp(checkpoint)
model.eval()
valid_iter = data_loader.DataLoader(self.args,
data_loader.load_dataset(
self.args,
'valid',
shuffle=False),
self.args.batch_size,
self.device,
shuffle=False,
is_test=False)
trainer = build_trainer(self.args, self.device_id, model, None)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def validate(args, device_id, pt, step, tokenizer):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
model = AbsSummarizer(args, device, checkpoint)
model.eval()
valid_iter = data_loader.Dataloader(args, load_dataset(args, 'dev', shuffle=False),
args.batch_size, device,
shuffle=False, is_test=False)
symbols = {'BOS': tokenizer.convert_tokens_to_ids('<s>'), 'EOS': tokenizer.convert_tokens_to_ids('</s>'),
'PAD': tokenizer.convert_tokens_to_ids('[PAD]')}
valid_loss = abs_loss(model.generator, symbols, model.vocab_size, train=False, device=device)
trainer = build_trainer(args, device_id, model, None, valid_loss)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def train_abs_single(args, device_id):
init_logger(args.log_file)
logger.info(str(args))
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
else:
checkpoint = None
if (args.load_from_extractive != ''):
logger.info('Loading bert from extractive model %s' % args.load_from_extractive)
bert_from_extractive = torch.load(args.load_from_extractive, map_location=lambda storage, loc: storage)
bert_from_extractive = bert_from_extractive['model']
else:
bert_from_extractive = None
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
def train_iter_fct():
return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device,
shuffle=True, is_test=False)
model = AbsSummarizer(args, device, checkpoint, bert_from_extractive)
if (args.sep_optim):
optim_bert = model_builder.build_optim_bert(args, model, checkpoint)
optim_dec = model_builder.build_optim_dec(args, model, checkpoint)
optim = [optim_bert, optim_dec]
else:
optim = [model_builder.build_optim(args, model, checkpoint)]
logger.info(model)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir)
symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'],
'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']}
train_loss = abs_loss(model.generator, symbols, model.vocab_size, device, train=True,
label_smoothing=args.label_smoothing)
trainer = build_trainer(args, device_id, model, optim, train_loss)
trainer.train(train_iter_fct, args.train_steps)
def train(args, device_id):
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
def train_iter_fct():
return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device,
shuffle=True, is_test=False)
# temp change for reducing gpu memory
model = Summarizer(args, device, load_pretrained_bert=True)
#config = BertConfig.from_json_file(args.bert_config_path)
#model = Summarizer(args, device, load_pretrained_bert=False, bert_config = config)
if args.train_from != '': #train another part from beginning
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
model.load_cp(checkpoint, strict=False)
# keys can not match
#optim = model_builder.build_optim(args, model, checkpoint)
optim = model_builder.build_optim(args, model, None)
if args.model_name == "ctx" and args.fix_scorer:
logger.info("fix the saliency scorer")
#for param in self.bert.model.parameters():
for param in model.parameters():
param.requires_grad = False
if hasattr(model.encoder, "selector") and model.encoder.selector is not None:
for param in model.encoder.selector.parameters():
param.requires_grad = True
#print([p for p in model.parameters() if p.requires_grad])
else:
optim = model_builder.build_optim(args, model, None)
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
_, neg_valid_loss = trainer.train(train_iter_fct, args.train_steps)
while len(neg_valid_loss) > 0:
#from 3rd to 2nd to 1st.
neg_loss, saved_model = heapq.heappop(neg_valid_loss)
print(-neg_loss, saved_model)
step = int(saved_model.split('.')[-2].split('_')[-1])
test(args, device_id, saved_model, step)
logger.info("Finish!")
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
model = AbsSummarizer(args, device, checkpoint)
model.eval()
valid_iter = data_loader.Dataloader(args,
load_dataset(args,
'valid',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=False)
parser = argparse.ArgumentParser()
parser.add_argument('--bpe-codes',
default="/content/PhoBERT_base_transformers/bpe.codes",
required=False,
type=str,
help='path to fastBPE BPE')
args1, unknown = parser.parse_known_args()
bpe = fastBPE(args1)
# Load the dictionary
vocab = Dictionary()
vocab.add_from_file("/content/PhoBERT_base_transformers/dict.txt")
tokenizer = bpe
symbols = {
'BOS': vocab.indices['[unused0]'],
'EOS': vocab.indices['[unused1]'],
'PAD': vocab.indices['[PAD]'],
'EOQ': vocab.indices['[unused2]']
}
valid_loss = abs_loss(model.generator,
symbols,
model.vocab_size,
train=False,
device=device)
trainer = build_trainer(args, device_id, model, None, valid_loss)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def train(args, device_id):
# Start logger.
init_logger(args.log_file)
# Configure training device.
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
# Configure manual seed.
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
# Set CUDA device.
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
# Configure manual seed.
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
# Dataloader used for training.
def train_iter_fct():
return Dataloader(args,
load_dataset(args, 'train', shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
# Build the model.
model = Summarizer(args, device, load_pretrained=True)
# Configure the checkpoint.
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in model_flags:
setattr(args, k, opt[k])
model.load_cp(checkpoint)
optim = builder.build_optim(args, model, checkpoint)
else:
optim = builder.build_optim(args, model, None)
logger.info(model)
# Train the model
trainer = build_trainer(args, device_id, model, optim)
trainer.train(train_iter_fct, args.train_steps)
def baseline(args, cal_lead=False, cal_oracle=False):
test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False),
args.batch_size, 'cpu',
shuffle=False, is_test=True)
trainer = build_trainer(args, '-1', None, None, None)
#
if (cal_lead):
trainer.test(test_iter, 0, cal_lead=True)
elif (cal_oracle):
trainer.test(test_iter, 0, cal_oracle=True)
def predict(self):
test_iter = data_loader.DataLoader(self.args,
data_loader.load_dataset(
self.args,
'test',
shuffle=False),
self.args.batch_size,
self.device,
shuffle=False,
is_test=True)
trainer = build_trainer(self.args, self.device_id, self.model, None)
trainer.predict(test_iter, self.step)
def train_abs_single(args, device_id):
init_logger(args.log_file)
logger.info(str(args))
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
else:
checkpoint = None
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
def train_iter_fct():
return data_pretrain.Dataloader(args,
load_dataset(args,
'train',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
model = PretrainModel(args, device, checkpoint)
optim = [model_builder.build_optim(args, model, checkpoint)]
logger.info(model)
symbols = {'PAD': 0}
train_loss = abs_loss(model.generator,
symbols,
model.dec_vocab_size,
device,
train=True,
label_smoothing=args.label_smoothing)
trainer = build_trainer(args, device_id, model, optim, train_loss)
trainer.train(train_iter_fct, args.train_steps)
def train(args, device_id):
# import ipdb; ipdb.set_trace()
# import pdb; pdb.set_trace()
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
print("device_id = " + str(device_id))
torch.cuda.set_device(device_id)
print("device set")
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
def train_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args, 'train',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
model = Summarizer(args, device, load_pretrained_bert=True)
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
model.load_cp(checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
else:
optim = model_builder.build_optim(args, model, None)
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
with comet_experiment.train():
trainer.train(train_iter_fct, args.train_steps)
def train(args, device_id):
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
logger.info('data begin load %s',
time.strftime('%H:%M:%S', time.localtime(time.time())))
def train_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args, 'train',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
logger.info('data end load %s',
time.strftime('%H:%M:%S', time.localtime(time.time())))
model = Summarizer(args, device, load_pretrained_bert=True)
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
model.load_cp(checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
else:
optim = model_builder.build_optim(args, model, None)
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
trainer.train(train_iter_fct, args.train_steps)
def baseline(args, cal_lead=False, cal_oracle=False):
test_iter = data_loader.Dataloader(
args,
load_dataset(args, "test", shuffle=False),
args.batch_size,
"cpu",
shuffle=False,
is_test=True,
)
trainer = build_trainer(args, "-1", None, None, None)
if cal_lead:
trainer.test(test_iter, 0, cal_lead=True)
elif cal_oracle:
trainer.test(test_iter, 0, cal_oracle=True)
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == "-1" else "cuda"
if pt != "":
test_from = pt
else:
test_from = args.test_from
logger.info("Loading checkpoint from %s" % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint["opt"])
for k in opt.keys():
if k in model_flags:
setattr(args, k, opt[k])
print(args)
model = AbsSummarizer(args, device, checkpoint)
model.eval()
valid_iter = data_loader.Dataloader(
args,
load_dataset(args, "valid", shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=False,
)
tokenizer = BertTokenizer.from_pretrained(
"chinese_roberta_wwm_ext_pytorch",
do_lower_case=True,
cache_dir=args.temp_dir)
symbols = {
"BOS": tokenizer.vocab["[unused1]"],
"EOS": tokenizer.vocab["[unused2]"],
"PAD": tokenizer.vocab["[PAD]"],
"EOQ": tokenizer.vocab["[unused3]"],
}
valid_loss = abs_loss(model.generator,
symbols,
model.vocab_size,
train=False,
device=device)
trainer = build_trainer(args, device_id, model, None, valid_loss)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def train(args, device_id):
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == "-1" else "cuda"
logger.info("Device ID %d" % device_id)
logger.info("Device %s" % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
def train_iter_fct():
return data_loader.Dataloader(
args,
load_dataset(args, "train", shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False,
)
model = Summarizer(args, device, load_pretrained_bert=True)
if args.train_from != "":
logger.info("Loading checkpoint from %s" % args.train_from)
checkpoint = torch.load(
args.train_from, map_location=lambda storage, loc: storage
)
opt = vars(checkpoint["opt"])
for k in opt.keys():
if k in model_flags:
setattr(args, k, opt[k])
model.load_cp(checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
else:
optim = model_builder.build_optim(args, model, None)
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
trainer.train(train_iter_fct, args.train_steps)
def baseline(self, cal_lead=False, cal_oracle=False):
test_iter = data_loader.DataLoader(self.args,
data_loader.load_dataset(
self.args,
'test',
shuffle=False),
self.args.batch_size,
self.device,
shuffle=False,
is_test=True)
trainer = build_trainer(self.args, self.device_id, None, None)
if cal_lead:
trainer.test(test_iter, 0, cal_lead=True)
elif cal_oracle:
trainer.test(test_iter, 0, cal_oracle=True)
def oracle(args, device_id, pt):
test_iter = data_loader.Dataloader(args,
load_dataset(args,
'test',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=True)
trainer = build_trainer(args, device_id, None, None)
print(
'___________________________________________________________________________'
)
trainer.oracle(test_iter, cal_lead=False, cal_oracle=False, each=True)
def test_detector(self, loader, run_gen=False, portion='all'):
""" Testing detector """
test = TreeDataset(self.args, loader, self.dataname, self.device, self.tokenizer)
#test = CommentDataset(self.args, loader, self.dataname, self.device, self.tokenizer)
data_iter = Iterator(test, train=False, device=self.device,
batch_size=len(test) if len(test)<self.bs else self.bs,
sort_key=lambda x: len(x.src),
sort_within_batch=False)
# Define trainer
train_loss = abs_loss(self.args.label_num, self.maxepoch, self.device, train=False)
trainer = build_trainer(self.args, self.model, self.optim, train_loss)
logger.info('Test on best model (stage-1)')
best_model = os.path.join(self.args.savepath, 'best_model.pt')
if os.path.exists(best_model):
try:
self.model.load_state_dict(torch.load(best_model, map_location=lambda storage,
loc: storage)['model'])
except:
self.model.load_state_dict(torch.load(best_model, map_location=lambda storage,
loc: storage)['model'], strict=False)
logger.info('[Warning] The keys in state dict do not strictly match')
test_stat = trainer.testing(data_iter, tokenizer=self.tokenizer, gen_flag=False, info="Without Generated Response >>", write_type="w")
test_stat.write_results(os.path.join(self.args.savepath, 'result_test.csv'), 'test-'+portion, self.args.label_num)
if self.args.test_adv:
logger.info('Test on adversarially-trained model (stage-2)')
best_model = os.path.join(self.args.savepath, 'best_adv_model.pt')
if os.path.exists(best_model):
try:
self.model.load_state_dict(torch.load(best_model, map_location=lambda storage,
loc: storage)['model'])
except:
self.model.load_state_dict(torch.load(best_model, map_location=lambda storage,
loc: storage)['model'], strict=False)
logger.info('[Warning] The keys in state dict do not strictly match')
test_stat = trainer.testing(data_iter, tokenizer=self.tokenizer, gen_flag=True, info="\nWith Generated Response from {} >>".format(best_model.split("/")[-1]), write_type=="a")
predictor = build_predictor(self.args, self.model, self.tokenizer, self.symbols, logger)
predictor.translate(data_iter, 'best', have_gold=False)
def exp(self, loader, exp_pos=False):
""" Testing detector """
test = TreeDataset(self.args, loader, self.dataname, self.device, self.tokenizer)
data_iter = Iterator(test, train=False, device=self.device, batch_size=self.bs,
sort_key=lambda x: len(x.src),
sort_within_batch=False)
best_model = os.path.join(self.args.savepath, 'best_adv_model.pt')
# Define trainer
loss = abs_loss(self.args.label_num, self.maxepoch, self.device, train=False)
trainer = build_trainer(self.args, self.model, self.optim, loss)
test_stats = trainer.testing(data_iter, -1)
if exp_pos:
if os.path.exists(best_model):
self.model.load_state_dict(torch.load(best_model, map_location=lambda storage,
loc: storage)['model'], strict=True)
test_stat = trainer.exp_pos(data_iter)
def train(args, device_id):
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
else:
checkpoint = None
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
def train_iter_fct():
# return data_loader.AbstractiveDataloader(load_dataset('train', True), symbols, FLAGS.batch_size, device, True)
return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device,
shuffle=True, is_test=False)
model = Summarizer(args, device, checkpoint)
# optim = model_builder.build_optim(args, model.reg, checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
# optim = BertAdam()
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
#
trainer.train(train_iter_fct, args.train_steps)
def train(self):
model = model_builder.Summarizer(self.args,
self.device,
load_pretrained_bert=True)
if self.args.train_from:
logger.info('Loading checkpoint from %s' % self.args.train_from)
checkpoint = torch.load(self.args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in self.model_flags:
setattr(self.args, k, opt[k])
model.load_cp(checkpoint)
optimizer = model_builder.build_optim(self.args, model, checkpoint)
else:
optimizer = model_builder.build_optim(self.args, model, None)
logger.info(model)
trainer = build_trainer(self.args, self.device_id, model, optimizer)
trainer.train(self.train_iter, self.args.train_steps)
def train(args, device_id):
init_logger(args.log_file)
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
else:
checkpoint = None
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
spm = sentencepiece.SentencePieceProcessor()
spm.Load(args.vocab_path)
word_padding_idx = spm.PieceToId('<PAD>')
vocab_size = len(spm)
def train_iter_fct():
# return data_loader.AbstractiveDataloader(load_dataset('train', True), symbols, FLAGS.batch_size, device, True)
return data_loader.Dataloader(args,
load_dataset(args, 'train',
shuffle=True),
{'PAD': word_padding_idx},
args.batch_size,
device,
shuffle=True,
is_test=False)
model = Summarizer(args, word_padding_idx, vocab_size, device, checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
#
trainer.train(train_iter_fct, args.train_steps)
def single_train(args, device_id):
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
# 使用指定的gpu
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
else:
checkpoint = None
def train_iter_method():
return DataLoaderBert(load_dataset(args, 'train', shuffle=True),
args.batch_size,
shuffle=True,
is_test=False)
model = NextSentencePrediction(args, device, checkpoint)
optim = build_optim(args, model, checkpoint)
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
trainer.train(train_iter_method, args.train_steps)
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
model = HMMModel(args, device, checkpoint)
model.eval()
valid_iter = data_hmm.Dataloader(args, load_dataset(args, 'dev', shuffle=False),
args.batch_size, device,
shuffle=False, is_test=False)
valid_loss = hmm_loss(model.generator, args.pad_id, args.relation_path, args.fake_global, model.dec_vocab_size, device, train=False)
trainer = build_trainer(args, device_id, model, None, valid_loss)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def train(args, device_id):
""" Starts training pipeline given CLI args """
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
seed(args.seed, device_id)
def train_iter_fct():
return data.load(args, 'train', device)
if args.train_from != '':
logger.info("Training from checkpoint %s", args.train_from)
model, optim = load_model(args,
device,
load_bert=True,
checkpoint=args.train_from)
else:
logger.info("Training without checkpoint")
model, optim = load_model(args, device, load_bert=True)
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
trainer.train(train_iter_fct, args.train_steps)
