def test_ext(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
model = ExtSummarizer(args, device, checkpoint)
model.eval()
test_iter = data_loader.Dataloader(args,
load_dataset(args,
'test',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=True)
trainer = build_trainer(args, device_id, model, None)
trainer.test(test_iter, step)
def load_model(model_type):
checkpoint = torch.load(f'checkpoints/{model_type}_ext.pt',
map_location='cpu')
model = ExtSummarizer(device="cpu",
checkpoint=checkpoint,
bert_type=model_type)
return model
def validate(args, device_id, pt, step):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
if args.ext_sum_dec:
model = SentenceTransformer(args, device, checkpoint, sum_or_jigsaw=0)
else:
model = ExtSummarizer(args, device, checkpoint)
model.eval()
valid_iter = data_loader.Dataloader(args,
load_dataset(args,
'valid',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=False)
trainer = build_trainer(args, device_id, model, None)
stats = trainer.validate(valid_iter, step)
return stats.xent()
def train_single_ext(args, device, device_id):
init_logger(args.log_file)
logger.info("Device ID {}".format(device_id))
logger.info("Device {}".format(device))
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if args.train_from_ckpt != '':
logger.info('Load ckpt from'.format(args.train_from_ckpt))
ckpt = torch.load(args.train_from_ckpt, map_location = lambdastorage, loc : storage)
ckpt_args = vars(ckpt['ckpt_args'])
for k in ckpt_args.keys():
if k in model_flags:
setattr(args, k, ckpt_args[k])
else:
ckpt = None
model = ExtSummarizer(args, device, ckpt)
def train_single_ext(args, device_id):
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
#TODO -> add ability to load model from chkpt
if args.train_from != '':
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in model_flags:
setattr(args, k, opt[k])
else:
checkpoint = None
def train_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args, 'train',
shuffle=True),
args.batch_size,
args.device,
shuffle=True,
is_test=False)
model = ExtSummarizer(args, checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
trainer = build_trainer(args, device_id, model, optim)
trainer.train(train_iter_fct, args.train_steps)
def train_ext(args):
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
#TODO -> add ability to load model from chkpt
if args.train_from != '':
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if k in model_flags:
setattr(args, k, opt[k])
else:
checkpoint = None
model = ExtSummarizer(args, checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
train_iter = data_loader.load_text(args, args.text_src, args.text_tgt,
device)
for i in range(train_iter):
print(i)
def train_single_ext(args, device_id):
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
else:
checkpoint = None
def train_iter_fct():
if args.is_debugging:
print("YES it is debugging")
return data_loader.Dataloader(args,
load_dataset(args,
'test',
shuffle=False),
args.batch_size,
device,
shuffle=False,
is_test=False)
# exit()
else:
return data_loader.Dataloader(args,
load_dataset(args,
'train',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
model = ExtSummarizer(args, device, checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
logger.info(model)
trainer = build_trainer(args, device_id, model, optim)
trainer.train(train_iter_fct, args.train_steps)
def test_text_ext(args):
logger.info('Loading checkpoint from %s' % args.test_from)
checkpoint = torch.load(args.test_from, map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
device_id = 0 if device == "cuda" else -1
model = ExtSummarizer(args, device, checkpoint)
model.eval()
test_iter = data_loader.load_text(args, args.text_src, args.text_tgt, device)
trainer = build_trainer(args, device_id, model, None)
trainer.test(test_iter, -1)
def train_single_ext(args, device_id):
init_logger(args.log_file)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
logger.info('Device ID %d' % device_id)
logger.info('Device %s' % device)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if device_id >= 0:
torch.cuda.set_device(device_id)
torch.cuda.manual_seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
torch.backends.cudnn.deterministic = True
if args.train_from != '':
logger.info('Loading checkpoint from %s' % args.train_from)
checkpoint = torch.load(args.train_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
else:
checkpoint = None
def train_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args, 'train',
shuffle=True),
args.batch_size,
device,
shuffle=True,
is_test=False)
if args.ext_sum_dec:
model = SentenceTransformer(args, device, checkpoint, sum_or_jigsaw=0)
else:
model = ExtSummarizer(args, device, checkpoint)
optim = model_builder.build_optim(args, model, checkpoint)
logger.info(model)
# if args.fp16:
# opt_level = 'O1' # typical fp16 training, can also try O2 to compare performance
# else:
# opt_level = 'O0' # pure fp32 traning
# model, optim.optimizer = amp.initialize(model, optim.optimizer, opt_level=opt_level)
trainer = build_trainer(args, device_id, model, optim)
trainer.train(train_iter_fct, args.train_steps)
def test_ext(args, device_id, pt, step, is_joint=False):
device = "cpu" if args.visible_gpus == '-1' else "cuda"
if (pt != ''):
test_from = pt
else:
test_from = args.test_from
logger.info('Loading checkpoint from %s' % test_from)
checkpoint = torch.load(test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
def test_iter_fct():
return data_loader.Dataloader(args,
load_dataset(args,
args.exp_set,
shuffle=False),
args.test_batch_size,
device,
shuffle=False,
is_test=True)
model = ExtSummarizer(args, device, checkpoint, is_joint=is_joint)
model.eval()
# test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False),
# args.test_batch_size, device,
# shuffle=False, is_test=True)
trainer = build_trainer(args, device_id, model, None)
# trainer.test(test_iter, step)
# trainer.test(test_iter_fct, step)
# trainer.validate_rouge_mmr(test_iter_fct, step)
# trainer.validate_rouge(test_iter_fct, step)
trainer.validate_rouge_baseline(test_iter_fct,
step,
write_scores_to_pickle=True)
def summarize(text, model = 'distilbert'):
with open('raw_data/input.txt', 'w') as f:
f.write(text)
# Load model
model_type = model
checkpoint = torch.load(f'checkpoints/{model_type}_ext.pt', map_location='cpu')
model = ExtSummarizer(checkpoint=checkpoint, bert_type=model_type, device="cpu")
%%time
# Run summarization
input_fp = 'raw_data/input.txt'
result_fp = 'results/summary.txt'
summary = summarize(input_fp, result_fp, model, max_length=3)
# Print summary
wrapper = textwrap.TextWrapper(width=80)
print(wrapper.fill(summary))
def only_model(args, device_id):
logger.info('Loading checkpoint from %s' % args.test_from)
checkpoint = torch.load(args.test_from,
map_location=lambda storage, loc: storage)
### We load our ExtSummarizer model
model = ExtSummarizer(args, device, checkpoint)
model.eval()
### We create an encoder and a decoder like those of ExtSummarizer and load the latter parameters into the former
### This is for the test sake
encoder = Bert(False, '/tmp', True)
load_my_state_dict(encoder, checkpoint['model'])
decoder = ExtTransformerEncoder(encoder.model.config.hidden_size,
args.ext_ff_size, args.ext_heads,
args.ext_dropout, args.ext_layers)
load_my_state_dict_decoder(decoder, checkpoint['model'])
encoder.eval()
decoder.eval()
seq_len = 250
### We test if the parameters have been well loaded
input_ids = torch.tensor([np.random.randint(100, 15000, seq_len)],
dtype=torch.long)
mask = torch.ones(1, seq_len, dtype=torch.float)
clss = torch.tensor([[20, 36, 55, 100, 122, 130, 200, 222]],
dtype=torch.long)
mask_cls = torch.tensor([[1] * len(clss[0])], dtype=torch.long)
"""## test encoder
top_vec = model.bert(input_ids, mask)
top_vec1 = encoder(input_ids, mask)
logger.info((top_vec-top_vec1).sum())
## test decoder
sents_vec = top_vec[torch.arange(top_vec.size(0)).unsqueeze(1), clss]
sents_vec = sents_vec * mask_cls[:, :, None].float()
sents_vec1 = top_vec1[torch.arange(top_vec1.size(0)).unsqueeze(1), clss]
sents_vec1 = sents_vec1 * mask_cls[:, :, None].float()
scores = model.ext_layer(sents_vec, mask_cls)
scores1 = decoder(sents_vec1, mask_cls)
logger.info((scores-scores1).sum())"""
################# ONNX ########################"
## Now we are exporting the encoder and the decoder into onnx
"""input_names = ["input_ids", "mask"]
output_names = ["hidden_outputs"]
torch.onnx.export(model.bert.to('cpu'), (input_ids, mask), "/tmp/encoder5.onnx", verbose=True,
input_names=input_names, output_names=output_names, export_params=True, keep_initializers_as_inputs=True)"""
k_model = pytorch_to_keras(model.bert.to('cpu'), [input_ids, mask], [(
1,
250,
), (
1,
250,
)],
verbose=True)
print("okkk")
"""logger.info("Load onnx and test")
from models.trainer_ext import build_trainer
from others.logging import logger, init_logger
model_flags = ['hidden_size', 'ff_size', 'heads', 'inter_layers', 'encoder', 'ff_actv', 'use_interval', 'rnn_size']
logger.info('Loading checkpoint from %s' % args.test_from)
checkpoint = torch.load(args.test_from, map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
print(args)
device = "cpu" if args.visible_gpus == '-1' else "cuda"
device_id = 0 if device == "cuda" else -1
model = ExtSummarizer(args, device, checkpoint)
model.eval()
# load data_files
# args.text_src and args.result_path change for every paper
file_dir_papers = "N:/Organisatorisches/Bereiche_Teams/ID/03_Studenten/Korte/Newsletter/Automatic Text Summarization/PreSumm_dev/cnndm/papers/"
file_dir_results = "N:/Organisatorisches/Bereiche_Teams/ID/03_Studenten/Korte/Newsletter/Automatic Text Summarization/PreSumm_dev/cnndm/results/"
for filename in os.listdir(file_dir_papers):
print(filename)
print("Inference for ", filename)
#change parameter for every trial
def call_train():
parser = argparse.ArgumentParser()
parser.add_argument("-task", default='ext', type=str, choices=['ext'])
parser.add_argument("-encoder",
default='bert',
type=str,
choices=['bert', 'baseline'])
parser.add_argument("-mode",
default='test_text',
type=str,
choices=['test_text'])
parser.add_argument("-bert_data_path",
default='bert_data/bert_data_cnndm_final/cnndm')
parser.add_argument("-model_path", default='models/')
parser.add_argument("-result_path", default='logs/ext_bert_cnndm')
parser.add_argument("-temp_dir", default='./temp')
parser.add_argument("-text_src", default='raw_data/temp.raw_src')
parser.add_argument("-text_tgt", default='')
parser.add_argument("-batch_size", default=140, type=int)
parser.add_argument("-test_batch_size", default=200, type=int)
parser.add_argument("-max_ndocs_in_batch", default=6, type=int)
parser.add_argument("-max_pos", default=512, type=int)
parser.add_argument("-use_interval",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-large",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-load_from_extractive", default='', type=str)
parser.add_argument("-sep_optim",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-lr_bert", default=2e-3, type=float)
parser.add_argument("-lr_dec", default=2e-3, type=float)
parser.add_argument("-use_bert_emb",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-share_emb",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-finetune_bert",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-dec_dropout", default=0.2, type=float)
parser.add_argument("-dec_layers", default=6, type=int)
parser.add_argument("-dec_hidden_size", default=768, type=int)
parser.add_argument("-dec_heads", default=8, type=int)
parser.add_argument("-dec_ff_size", default=2048, type=int)
parser.add_argument("-enc_hidden_size", default=512, type=int)
parser.add_argument("-enc_ff_size", default=512, type=int)
parser.add_argument("-enc_dropout", default=0.2, type=float)
parser.add_argument("-enc_layers", default=6, type=int)
# params for EXT
parser.add_argument("-ext_dropout", default=0.2, type=float)
parser.add_argument("-ext_layers", default=2, type=int)
parser.add_argument("-ext_hidden_size", default=768, type=int)
parser.add_argument("-ext_heads", default=8, type=int)
parser.add_argument("-ext_ff_size", default=2048, type=int)
parser.add_argument("-label_smoothing", default=0.1, type=float)
parser.add_argument("-generator_shard_size", default=32, type=int)
parser.add_argument("-alpha", default=0.95, type=float)
parser.add_argument("-beam_size", default=5, type=int)
parser.add_argument("-min_length", default=50, type=int)
parser.add_argument("-max_length", default=200, type=int)
parser.add_argument("-max_tgt_len", default=140, type=int)
parser.add_argument("-param_init", default=0, type=float)
parser.add_argument("-param_init_glorot",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-optim", default='adam', type=str)
parser.add_argument("-lr", default=1, type=float)
parser.add_argument("-beta1", default=0.9, type=float)
parser.add_argument("-beta2", default=0.999, type=float)
parser.add_argument("-warmup_steps", default=8000, type=int)
parser.add_argument("-warmup_steps_bert", default=8000, type=int)
parser.add_argument("-warmup_steps_dec", default=8000, type=int)
parser.add_argument("-max_grad_norm", default=0, type=float)
parser.add_argument("-save_checkpoint_steps", default=5, type=int)
parser.add_argument("-accum_count", default=1, type=int)
parser.add_argument("-report_every", default=1, type=int)
parser.add_argument("-train_steps", default=1000, type=int)
parser.add_argument("-recall_eval",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument('-visible_gpus', default='0', type=str)
parser.add_argument('-gpu_ranks', default='0', type=str)
parser.add_argument('-log_file', default='logs/cnndm.log')
parser.add_argument('-seed', default=666, type=int)
parser.add_argument("-test_all",
type=str2bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-test_from",
default='models/bertext_cnndm_transformer.pt')
parser.add_argument("-test_start_from", default=-1, type=int)
parser.add_argument("-train_from", default='')
parser.add_argument("-report_rouge",
type=str2bool,
nargs='?',
const=True,
default=True)
parser.add_argument("-block_trigram",
type=str2bool,
nargs='?',
const=True,
default=True)
args = parser.parse_args()
args.gpu_ranks = [int(i) for i in range(len(args.visible_gpus.split(',')))]
args.world_size = len(args.gpu_ranks)
os.environ["CUDA_VISIBLE_DEVICES"] = args.visible_gpus
init_logger(args.log_file)
logger.info('Loading checkpoint from %s' % args.test_from)
checkpoint = torch.load(args.test_from,
map_location=lambda storage, loc: storage)
opt = vars(checkpoint['opt'])
for k in opt.keys():
if (k in model_flags):
setattr(args, k, opt[k])
device = "cpu" if args.visible_gpus == '-1' else "cuda"
device_id = 0 if device == "cuda" else -1
logger.info('Coming here:1')
model = ExtSummarizer(args, device, checkpoint)
model.eval()
trainer = build_trainer(args, device_id, model, None)
logger.info('Coming here:2')
logger.info('args: %s' % args)
return model, args, trainer
from ext_sum import summarize
from newspaper import Article
url = "https://www.cnn.com/2020/05/29/tech/facebook-violence-trump/index.html" #@param {type: 'string'}
article = Article(url)
article.download()
article.parse()
print(wrapper.fill(article.text))
# Save input text into `raw_data/input.txt`
with open(chemin_d + 'article_exemple.txt', 'w') as f:
f.write(article.text)
# Load model
model_type = 'mobilebert' #@param ['bertbase', 'distilbert', 'mobilebert']
checkpoint = torch.load(f'checkpoints/{model_type}_ext.pt', map_location='cpu')
model = ExtSummarizer(checkpoint=checkpoint,
bert_type=model_type,
device='cpu')
# Run summarization
input_fp = chemin_d + 'article_exemple.txt'
result_fp = chemin_d + 'article_exemple_summary.txt'
summary = summarize(input_fp, result_fp, model, max_length=3)
print(summary)
#%%
####################
########## BARThez
####################
text_sentence = "Citant les préoccupations de ses clients dénonçant des cas de censure après la suppression du compte de Trump, un fournisseur d'accès Internet de l'État de l'Idaho a décidé de bloquer Facebook et Twitter. La mesure ne concernera cependant que les clients mécontents de la politique de ces réseaux sociaux."
