def test_ext(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) model = ExtSummarizer(args, device, checkpoint) model.eval() test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False), args.batch_size, device, shuffle=False, is_test=True) trainer = build_trainer(args, device_id, model, None) trainer.test(test_iter, step)
def load_model(model_type): checkpoint = torch.load(f'checkpoints/{model_type}_ext.pt', map_location='cpu') model = ExtSummarizer(device="cpu", checkpoint=checkpoint, bert_type=model_type) return model
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) if args.ext_sum_dec: model = SentenceTransformer(args, device, checkpoint, sum_or_jigsaw=0) else: model = ExtSummarizer(args, device, checkpoint) model.eval() valid_iter = data_loader.Dataloader(args, load_dataset(args, 'valid', shuffle=False), args.batch_size, device, shuffle=False, is_test=False) trainer = build_trainer(args, device_id, model, None) stats = trainer.validate(valid_iter, step) return stats.xent()
def train_single_ext(args, device, device_id): init_logger(args.log_file) logger.info("Device ID {}".format(device_id)) logger.info("Device {}".format(device)) if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if args.train_from_ckpt != '': logger.info('Load ckpt from'.format(args.train_from_ckpt)) ckpt = torch.load(args.train_from_ckpt, map_location = lambdastorage, loc : storage) ckpt_args = vars(ckpt['ckpt_args']) for k in ckpt_args.keys(): if k in model_flags: setattr(args, k, ckpt_args[k]) else: ckpt = None model = ExtSummarizer(args, device, ckpt)
def train_single_ext(args, device_id): torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True #TODO -> add ability to load model from chkpt if args.train_from != '': checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in model_flags: setattr(args, k, opt[k]) else: checkpoint = None def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, args.device, shuffle=True, is_test=False) model = ExtSummarizer(args, checkpoint) optim = model_builder.build_optim(args, model, checkpoint) trainer = build_trainer(args, device_id, model, optim) trainer.train(train_iter_fct, args.train_steps)
def train_ext(args): torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True #TODO -> add ability to load model from chkpt if args.train_from != '': checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in model_flags: setattr(args, k, opt[k]) else: checkpoint = None model = ExtSummarizer(args, checkpoint) optim = model_builder.build_optim(args, model, checkpoint) train_iter = data_loader.load_text(args, args.text_src, args.text_tgt, device) for i in range(train_iter): print(i)
def train_single_ext(args, device_id): init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) else: checkpoint = None def train_iter_fct(): if args.is_debugging: print("YES it is debugging") return data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False), args.batch_size, device, shuffle=False, is_test=False) # exit() else: return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) model = ExtSummarizer(args, device, checkpoint) optim = model_builder.build_optim(args, model, checkpoint) logger.info(model) trainer = build_trainer(args, device_id, model, optim) trainer.train(train_iter_fct, args.train_steps)
def test_text_ext(args): logger.info('Loading checkpoint from %s' % args.test_from) checkpoint = torch.load(args.test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) device = "cpu" if args.visible_gpus == '-1' else "cuda" device_id = 0 if device == "cuda" else -1 model = ExtSummarizer(args, device, checkpoint) model.eval() test_iter = data_loader.load_text(args, args.text_src, args.text_tgt, device) trainer = build_trainer(args, device_id, model, None) trainer.test(test_iter, -1)
def train_single_ext(args, device_id): init_logger(args.log_file) device = "cpu" if args.visible_gpus == '-1' else "cuda" logger.info('Device ID %d' % device_id) logger.info('Device %s' % device) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if device_id >= 0: torch.cuda.set_device(device_id) torch.cuda.manual_seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) torch.backends.cudnn.deterministic = True if args.train_from != '': logger.info('Loading checkpoint from %s' % args.train_from) checkpoint = torch.load(args.train_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) else: checkpoint = None def train_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, 'train', shuffle=True), args.batch_size, device, shuffle=True, is_test=False) if args.ext_sum_dec: model = SentenceTransformer(args, device, checkpoint, sum_or_jigsaw=0) else: model = ExtSummarizer(args, device, checkpoint) optim = model_builder.build_optim(args, model, checkpoint) logger.info(model) # if args.fp16: # opt_level = 'O1' # typical fp16 training, can also try O2 to compare performance # else: # opt_level = 'O0' # pure fp32 traning # model, optim.optimizer = amp.initialize(model, optim.optimizer, opt_level=opt_level) trainer = build_trainer(args, device_id, model, optim) trainer.train(train_iter_fct, args.train_steps)
def test_ext(args, device_id, pt, step, is_joint=False): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) def test_iter_fct(): return data_loader.Dataloader(args, load_dataset(args, args.exp_set, shuffle=False), args.test_batch_size, device, shuffle=False, is_test=True) model = ExtSummarizer(args, device, checkpoint, is_joint=is_joint) model.eval() # test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False), # args.test_batch_size, device, # shuffle=False, is_test=True) trainer = build_trainer(args, device_id, model, None) # trainer.test(test_iter, step) # trainer.test(test_iter_fct, step) # trainer.validate_rouge_mmr(test_iter_fct, step) # trainer.validate_rouge(test_iter_fct, step) trainer.validate_rouge_baseline(test_iter_fct, step, write_scores_to_pickle=True)
def summarize(text, model = 'distilbert'): with open('raw_data/input.txt', 'w') as f: f.write(text) # Load model model_type = model checkpoint = torch.load(f'checkpoints/{model_type}_ext.pt', map_location='cpu') model = ExtSummarizer(checkpoint=checkpoint, bert_type=model_type, device="cpu") %%time # Run summarization input_fp = 'raw_data/input.txt' result_fp = 'results/summary.txt' summary = summarize(input_fp, result_fp, model, max_length=3) # Print summary wrapper = textwrap.TextWrapper(width=80) print(wrapper.fill(summary))
def only_model(args, device_id): logger.info('Loading checkpoint from %s' % args.test_from) checkpoint = torch.load(args.test_from, map_location=lambda storage, loc: storage) ### We load our ExtSummarizer model model = ExtSummarizer(args, device, checkpoint) model.eval() ### We create an encoder and a decoder like those of ExtSummarizer and load the latter parameters into the former ### This is for the test sake encoder = Bert(False, '/tmp', True) load_my_state_dict(encoder, checkpoint['model']) decoder = ExtTransformerEncoder(encoder.model.config.hidden_size, args.ext_ff_size, args.ext_heads, args.ext_dropout, args.ext_layers) load_my_state_dict_decoder(decoder, checkpoint['model']) encoder.eval() decoder.eval() seq_len = 250 ### We test if the parameters have been well loaded input_ids = torch.tensor([np.random.randint(100, 15000, seq_len)], dtype=torch.long) mask = torch.ones(1, seq_len, dtype=torch.float) clss = torch.tensor([[20, 36, 55, 100, 122, 130, 200, 222]], dtype=torch.long) mask_cls = torch.tensor([[1] * len(clss[0])], dtype=torch.long) """## test encoder top_vec = model.bert(input_ids, mask) top_vec1 = encoder(input_ids, mask) logger.info((top_vec-top_vec1).sum()) ## test decoder sents_vec = top_vec[torch.arange(top_vec.size(0)).unsqueeze(1), clss] sents_vec = sents_vec * mask_cls[:, :, None].float() sents_vec1 = top_vec1[torch.arange(top_vec1.size(0)).unsqueeze(1), clss] sents_vec1 = sents_vec1 * mask_cls[:, :, None].float() scores = model.ext_layer(sents_vec, mask_cls) scores1 = decoder(sents_vec1, mask_cls) logger.info((scores-scores1).sum())""" ################# ONNX ########################" ## Now we are exporting the encoder and the decoder into onnx """input_names = ["input_ids", "mask"] output_names = ["hidden_outputs"] torch.onnx.export(model.bert.to('cpu'), (input_ids, mask), "/tmp/encoder5.onnx", verbose=True, input_names=input_names, output_names=output_names, export_params=True, keep_initializers_as_inputs=True)""" k_model = pytorch_to_keras(model.bert.to('cpu'), [input_ids, mask], [( 1, 250, ), ( 1, 250, )], verbose=True) print("okkk") """logger.info("Load onnx and test")
from models.trainer_ext import build_trainer from others.logging import logger, init_logger model_flags = ['hidden_size', 'ff_size', 'heads', 'inter_layers', 'encoder', 'ff_actv', 'use_interval', 'rnn_size'] logger.info('Loading checkpoint from %s' % args.test_from) checkpoint = torch.load(args.test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) device = "cpu" if args.visible_gpus == '-1' else "cuda" device_id = 0 if device == "cuda" else -1 model = ExtSummarizer(args, device, checkpoint) model.eval() # load data_files # args.text_src and args.result_path change for every paper file_dir_papers = "N:/Organisatorisches/Bereiche_Teams/ID/03_Studenten/Korte/Newsletter/Automatic Text Summarization/PreSumm_dev/cnndm/papers/" file_dir_results = "N:/Organisatorisches/Bereiche_Teams/ID/03_Studenten/Korte/Newsletter/Automatic Text Summarization/PreSumm_dev/cnndm/results/" for filename in os.listdir(file_dir_papers): print(filename) print("Inference for ", filename) #change parameter for every trial
def call_train(): parser = argparse.ArgumentParser() parser.add_argument("-task", default='ext', type=str, choices=['ext']) parser.add_argument("-encoder", default='bert', type=str, choices=['bert', 'baseline']) parser.add_argument("-mode", default='test_text', type=str, choices=['test_text']) parser.add_argument("-bert_data_path", default='bert_data/bert_data_cnndm_final/cnndm') parser.add_argument("-model_path", default='models/') parser.add_argument("-result_path", default='logs/ext_bert_cnndm') parser.add_argument("-temp_dir", default='./temp') parser.add_argument("-text_src", default='raw_data/temp.raw_src') parser.add_argument("-text_tgt", default='') parser.add_argument("-batch_size", default=140, type=int) parser.add_argument("-test_batch_size", default=200, type=int) parser.add_argument("-max_ndocs_in_batch", default=6, type=int) parser.add_argument("-max_pos", default=512, type=int) parser.add_argument("-use_interval", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-large", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-load_from_extractive", default='', type=str) parser.add_argument("-sep_optim", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-lr_bert", default=2e-3, type=float) parser.add_argument("-lr_dec", default=2e-3, type=float) parser.add_argument("-use_bert_emb", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-share_emb", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-finetune_bert", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-dec_dropout", default=0.2, type=float) parser.add_argument("-dec_layers", default=6, type=int) parser.add_argument("-dec_hidden_size", default=768, type=int) parser.add_argument("-dec_heads", default=8, type=int) parser.add_argument("-dec_ff_size", default=2048, type=int) parser.add_argument("-enc_hidden_size", default=512, type=int) parser.add_argument("-enc_ff_size", default=512, type=int) parser.add_argument("-enc_dropout", default=0.2, type=float) parser.add_argument("-enc_layers", default=6, type=int) # params for EXT parser.add_argument("-ext_dropout", default=0.2, type=float) parser.add_argument("-ext_layers", default=2, type=int) parser.add_argument("-ext_hidden_size", default=768, type=int) parser.add_argument("-ext_heads", default=8, type=int) parser.add_argument("-ext_ff_size", default=2048, type=int) parser.add_argument("-label_smoothing", default=0.1, type=float) parser.add_argument("-generator_shard_size", default=32, type=int) parser.add_argument("-alpha", default=0.95, type=float) parser.add_argument("-beam_size", default=5, type=int) parser.add_argument("-min_length", default=50, type=int) parser.add_argument("-max_length", default=200, type=int) parser.add_argument("-max_tgt_len", default=140, type=int) parser.add_argument("-param_init", default=0, type=float) parser.add_argument("-param_init_glorot", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-optim", default='adam', type=str) parser.add_argument("-lr", default=1, type=float) parser.add_argument("-beta1", default=0.9, type=float) parser.add_argument("-beta2", default=0.999, type=float) parser.add_argument("-warmup_steps", default=8000, type=int) parser.add_argument("-warmup_steps_bert", default=8000, type=int) parser.add_argument("-warmup_steps_dec", default=8000, type=int) parser.add_argument("-max_grad_norm", default=0, type=float) parser.add_argument("-save_checkpoint_steps", default=5, type=int) parser.add_argument("-accum_count", default=1, type=int) parser.add_argument("-report_every", default=1, type=int) parser.add_argument("-train_steps", default=1000, type=int) parser.add_argument("-recall_eval", type=str2bool, nargs='?', const=True, default=False) parser.add_argument('-visible_gpus', default='0', type=str) parser.add_argument('-gpu_ranks', default='0', type=str) parser.add_argument('-log_file', default='logs/cnndm.log') parser.add_argument('-seed', default=666, type=int) parser.add_argument("-test_all", type=str2bool, nargs='?', const=True, default=False) parser.add_argument("-test_from", default='models/bertext_cnndm_transformer.pt') parser.add_argument("-test_start_from", default=-1, type=int) parser.add_argument("-train_from", default='') parser.add_argument("-report_rouge", type=str2bool, nargs='?', const=True, default=True) parser.add_argument("-block_trigram", type=str2bool, nargs='?', const=True, default=True) args = parser.parse_args() args.gpu_ranks = [int(i) for i in range(len(args.visible_gpus.split(',')))] args.world_size = len(args.gpu_ranks) os.environ["CUDA_VISIBLE_DEVICES"] = args.visible_gpus init_logger(args.log_file) logger.info('Loading checkpoint from %s' % args.test_from) checkpoint = torch.load(args.test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) device = "cpu" if args.visible_gpus == '-1' else "cuda" device_id = 0 if device == "cuda" else -1 logger.info('Coming here:1') model = ExtSummarizer(args, device, checkpoint) model.eval() trainer = build_trainer(args, device_id, model, None) logger.info('Coming here:2') logger.info('args: %s' % args) return model, args, trainer
from ext_sum import summarize from newspaper import Article url = "https://www.cnn.com/2020/05/29/tech/facebook-violence-trump/index.html" #@param {type: 'string'} article = Article(url) article.download() article.parse() print(wrapper.fill(article.text)) # Save input text into `raw_data/input.txt` with open(chemin_d + 'article_exemple.txt', 'w') as f: f.write(article.text) # Load model model_type = 'mobilebert' #@param ['bertbase', 'distilbert', 'mobilebert'] checkpoint = torch.load(f'checkpoints/{model_type}_ext.pt', map_location='cpu') model = ExtSummarizer(checkpoint=checkpoint, bert_type=model_type, device='cpu') # Run summarization input_fp = chemin_d + 'article_exemple.txt' result_fp = chemin_d + 'article_exemple_summary.txt' summary = summarize(input_fp, result_fp, model, max_length=3) print(summary) #%% #################### ########## BARThez #################### text_sentence = "Citant les préoccupations de ses clients dénonçant des cas de censure après la suppression du compte de Trump, un fournisseur d'accès Internet de l'État de l'Idaho a décidé de bloquer Facebook et Twitter. La mesure ne concernera cependant que les clients mécontents de la politique de ces réseaux sociaux."