from transformers import BartForConditionalGeneration, BartTokenizer model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-xsum') tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-xsum') model = BartForConditionalGeneration.from_pretrained('sshleifer/distilbart-xsum-12-3') tokenizer = BartTokenizer.from_pretrained('sshleifer/distilbart-xsum-12-3') ARTICLE_TO_SUMMARIZE = " \"The accident meant the motorway was closed, making travel to Mourneview Park impossible for the team and fans travelling from Belfast, \" said the Irish Football Association . A new date for the match has yet to be confirmed by Uefa . Northern Ireland have three points from their first two Group Six qualifiers." inputs = tokenizer.batch_encode_plus([ARTICLE_TO_SUMMARIZE], max_length=512, return_tensors='pt') summary_ids = model.generate(inputs['input_ids'], num_beams=5, max_length=62, min_length=10, early_stopping=True) print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in summary_ids])
def test_dummy_inputs(self): config, *_ = self._get_config_and_data() model = BartForConditionalGeneration(config).eval().to(torch_device) model(**model.dummy_inputs)
def Seq2Seq(df): model_type = 'bart-large' tokenizer = BartTokenizer.from_pretrained('facebook/bart-large') model = BartModel.from_pretrained('facebook/bart-large') mask_model = BartForConditionalGeneration.from_pretrained( 'facebook/bart-large') sep_token = '</s>' mask_token = '<mask>' mask_id = tokenizer(mask_token, return_tensors='pt')['input_ids'][0][1] sep_id = tokenizer(sep_token, return_tensors='pt')['input_ids'][0][1] optimizer = AdamW(model.parameters()) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) mask_model.to(device) auxiliary_tokens = ['the', 'aspect', 'term', 'is'] df['mask_tokens'] = 0 df['auxiliary_tokens'] = 0 df = df.astype('object') for i in range(len(df)): #for j in range(len(df['aspect_terms'].iloc[i])): auxiliary_sents = [] for j in range(len(df['aspect_terms'].iloc[i])): aspect_terms = df['aspect_terms'].iloc[i][j] auxiliary_sent = auxiliary_tokens + [aspect_terms] + [ sep_token ] + df['tokens'].iloc[i] auxiliary_sents.append(auxiliary_sent) mask_sent = auxiliary_tokens + [mask_token] + [sep_token ] + df['tokens'].iloc[i] df['mask_tokens'].iloc[i] = mask_sent df['auxiliary_tokens'].iloc[i] = auxiliary_sents df['distance'] = 0 df = df.astype('object') for i in range(len(df)): tokenized = tokenizer.encode(df['mask_tokens'].iloc[i]) sep_index = tokenized.index(sep_id) mask_index = tokenized.index(mask_id) tokenized = pd.Series([tokenized]) padded = pad_sequences(tokenized, maxlen=MAX_LEN, dtype="long", value=0, truncating="post", padding="post") attention_mask = np.where(padded != 0, 1, 0) input_ids = torch.tensor(padded).to(device) attention_mask = torch.tensor(attention_mask).to(device) with torch.no_grad(): last_hidden_states = model(input_ids, attention_mask=attention_mask) original_mask_embedding = last_hidden_states[0][:, mask_index, :].cpu( ).numpy() distance = [] for pertubed_index in range(sep_index + 1, MAX_LEN): padded = pad_sequences(tokenized, maxlen=MAX_LEN, dtype="long", value=0, truncating="post", padding="post") if padded[0][pertubed_index] != 0 and padded[0][ pertubed_index] != sep_id: #print(padded.shape) cur_id = padded[0][pertubed_index] padded[0][pertubed_index] = mask_id cur_embedding = mask_embedding(model, padded, mask_index) d = dist(original_mask_embedding, cur_embedding) distance.append((cur_id, d)) df['distance'].iloc[i] = distance df['perturbed_mask_index'] = 0 df = df.astype('object') for i in range(len(df)): perturbed_mask_index = [] mask_threshold = calculate_threshold( np.array(df['distance'].iloc[i])[:, 1], std_strength) for dis_index in range(len(df['distance'].iloc[i])): if df['distance'].iloc[i][dis_index][1] < mask_threshold and df[ 'labels'].iloc[i][dis_index] != 'B' and df['labels'].iloc[ i][dis_index] != 'I': perturbed_mask_index.append(dis_index) df['perturbed_mask_index'].iloc[i] = perturbed_mask_index df['augment_token_id'] = 0 df = df.astype('object') for i in range(len(df)): augment_tokenizeds = [] for j in range(len(df['aspect_terms'].iloc[i])): tokenized = tokenizer.encode(df['auxiliary_tokens'].iloc[i][j]) tokenized = torch.Tensor(tokenized).unsqueeze(0).to( torch.int64).to(device) augment_tokenized = tokenizer.encode( df['auxiliary_tokens'].iloc[i][j]) for k in range(len(df['perturbed_mask_index'].iloc[i])): mask_tokenized = tokenizer.encode( df['auxiliary_tokens'].iloc[i][j]) sep_index = mask_tokenized.index(sep_id) perturbed_mask_index = df['perturbed_mask_index'].iloc[i][ k] + sep_index + 1 mask_tokenized[perturbed_mask_index] = mask_id mask_tokenized = torch.Tensor(mask_tokenized).unsqueeze(0).to( torch.int64).to(device) logits = mask_model(mask_tokenized).logits probs = logits[0, perturbed_mask_index].softmax(dim=0) values, predictions = probs.topk(1) augment_tokenized[perturbed_mask_index] = int( predictions.cpu().numpy()) augment_tokenizeds.append(augment_tokenized) df['augment_token_id'].iloc[i] = augment_tokenizeds df['augment_tokens'] = 0 df = df.astype('object') for i in range(len(df)): tokens_lists = [] for j in range(len(df['aspect_terms'].iloc[i])): tokens_list = [] for k in range(1, len(df['augment_token_id'].iloc[i][j]) - 1): tokens_list.append( tokenizer.decode([df['augment_token_id'].iloc[i][j][k]])) sep_index = tokens_list.index(sep_token) tokens_list = tokens_list[sep_index + 1:] tokens_lists.append(tokens_list) df['augment_tokens'].iloc[i] = tokens_lists return df
def test_model_download(): """This warms up the cache so that we can time the next test without including download time, which varies between machines.""" BartForConditionalGeneration.from_pretrained(MODEL_NAME) MarianMTModel.from_pretrained(MARIAN_MODEL)
def test_base_model_fp16(self): config, input_ids, batch_size = self._get_config_and_data() attention_mask = input_ids.ne(1).to(torch_device) lm_model = BartForConditionalGeneration(config).eval().to(torch_device).half() lm_model(input_ids, attention_mask=attention_mask)
def main(args): # load pretrained or finetuned transformer model print(f'loading pre-trained model: {args["model_id"]}') # we have to load fine-tuned models in a different way because of pytorch-lightning # fine-tuned if args['model_id'].endswith('.ckpt'): from transformer_decoding.finetune import SummarizationTrainer lightning_model = SummarizationTrainer.load_from_checkpoint(args['model_id']) args['model'] = lightning_model.model args['tokenizer'] = lightning_model.tokenizer else: # transformers pretrained args['model'] = BartForConditionalGeneration.from_pretrained(args['model_id']) args['tokenizer'] = BartTokenizer.from_pretrained(args['model_id']) # Set the model in evaluation mode to deactivate the DropOut modules # This is IMPORTANT to have reproducible results during evaluation! args['model'].eval() if torch.cuda.is_available(): args['model'].to('cuda') # summarize MDS / summarization dataset with model # print and write out evaluation results if args['evaluation_dataset'].endswith('.jsonl'): dataset = [json.loads(l) for l in open(args['evaluation_dataset'])][:args['rows_to_eval']] else: raise AssertionError('Right now we only know how to handle .jsonl evaluation datasets') # WORKING: also write out summaries as they're generated eval_prefix = args['eval_prefix'] preds_output = open(f'{eval_prefix}eval_predicted_summaries.out', 'w', buffering=1) gold_output = open(f'{eval_prefix}eval_gold_summaries.out', 'w', buffering=1) metadata_output = open(f'{eval_prefix}decoding_metadata.jsonl', 'w', buffering=1) summaries = [] # get summary for each cluster # note here we have a macro-batch size of one cluster by definition for cluster in tqdm.tqdm(dataset): # shuffle articles before selecting topk to use in ensemble articles = [article_to_text(a) for a in cluster['articles']] np.random.shuffle(articles) articles = articles[:args['max_articles_in_cluster']] if args['min_input_char_length'] is not None: articles_ = [a for a in articles if len(a) >= args['min_input_char_length']] if len(articles_) == 0: articles_ = [articles[0]] articles = articles_ predictions, sorted_hyps = summarize_articles(articles, args) # sorted_hyps -- (token_idxs, score, metadata) # they're in sorted order according to ensemble score, so first one is the best # we will have one list of timestamp metadata for each input length_penalty = args['length_penalty'] component_scores = [] for input_idx, state_metadata in enumerate(sorted_hyps[0][2]): timestep_scores = np.array([o['score'] for o in state_metadata]) global_score = np.sum(timestep_scores) / len(timestep_scores) ** length_penalty component_scores.append(global_score) component_scores = np.array(component_scores) for idx in np.argsort(component_scores)[::-1]: print(f'ARTICLE: {articles[idx][:200]}') print(f'Input {idx} score: {component_scores[idx]}') print() print(f'Ensemble score: {sorted_hyps[0][1]}') print(f'Gold: {cluster["summary"]}') print(f'Predicted: {predictions[0]}') print() #import ipdb; ipdb.set_trace() #tok_ids = [o['token'] for o in sorted_hyps[0][2][0]] #print(args['tokenizer'].decode(tok_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) # NOTE: hack to just take the first one right now, disregarding scores of different beam items predicted_summary = predictions[0] gold_summary = cluster['summary'].strip() summaries.append((predicted_summary, gold_summary)) preds_output.write(f'{predicted_summary}\n') gold_output.write(f'{gold_summary}\n') # TODO: need to map some stuff to put in json? sorted_hyps_ = [] for tok_idxs, score, tok_scores in sorted_hyps: tok_idxs = [int(idx) for idx in tok_idxs.cpu().numpy()] sorted_hyps_.append((tok_idxs, score, tok_scores)) sorted_hyps = sorted_hyps_ metadata_output.write( json.dumps( { 'cluster': cluster, 'predictions': predictions, 'inputs_used': articles, 'component_scores': list(component_scores), 'decoding_metadata': sorted_hyps }) + '\n') preds_output.close() gold_output.close() # Evaluation hyps, refs = zip(*summaries) results, rouge_types = evaluate_rouge(hyps, refs) print_mean(results, rouge_types)
# BART tests (run from unifiedqa-tjh/bart directory): import torch from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig from bart import MyBart base_model = "facebook/bart-large" #unifiedqa_path = "unifiedQA-uncased/best-model.pt" # path to the downloaded checkpoint unifiedqa_path = "/data/thar011/ckpts/unifiedqa-bart-large-allenai/unifiedQA-uncased/best-model.pt" # path to the downloaded checkpoint tokenizer = BartTokenizer.from_pretrained(base_model) model = MyBart.from_pretrained(base_model, state_dict=torch.load(unifiedqa_path)) model.eval() # ERROR: TypeError: forward() got an unexpected keyword argument 'past_key_values' x = model.generate_from_string( "Which is best conductor? \\n (A) iron (B) feather", tokenizer=tokenizer) print(x) x = model.generate_from_string( "What is the sum of 3 and 5? \\n (A) 8 (B) 3 (C) 5 (D) 10", tokenizer=tokenizer) print(x) #try basic bart model (no error): model = BartForConditionalGeneration.from_pretrained(base_model) model.eval() run_model("which is best conductor? \\n (a) iron (b) feather" ) #['whichwhich is best conductor?']
"attention_mask": input_encodings["attention_mask"].numpy().copy(), "decoder_input_ids": decoder_input_ids.numpy().copy(), "labels": labels.numpy().copy(), } return encodings # 2. store the preprocessed data on disk in a .pckl file # 3. create a dataloader that loads the data from disk to memory on-demand if __name__ == "__main__": dataset = datasets.load_dataset( "json", data_files=str(disk.VERSIONED_DATA_DIR / "generated/examples.json"), )["train"].train_test_split(test_size=100, shuffle=True) dataset["validation"] = dataset["test"] del dataset["test"] tokenizer = BartTokenizer.from_pretrained("facebook/bart-base") model = BartForConditionalGeneration.from_pretrained("facebook/bart-base") dataset = dataset.map(convert_to_features, batched=True) columns = ["input_ids", "labels", "decoder_input_ids", "attention_mask"] dataset.set_format(type="torch", columns=columns) dataset.save_to_disk(disk.UNVERSIONED_DATA_DIR / "features")
def __init__(self, cfg: DictConfig, trainer: Trainer = None): # must assign tokenizers before init if cfg.language_model.pretrained_model_name: if cfg.language_model.pretrained_encoder_model_name or cfg.language_model.pretrained_decoder_model_name: raise ValueError( "Must have either pretrained_model_name or both pretrained_encoder_model name and " "pretrained_decoder_model_name.") # setup tokenizer self.encoder_tokenizer = self.setup_tokenizer( cfg.encoder_tokenizer) self.encoder_add_special_tokens = cfg.encoder_tokenizer.add_special_tokens # set decoder to encoder self.decoder_tokenizer = self.encoder_tokenizer self.decoder_add_special_tokens = self.encoder_add_special_tokens else: if not (cfg.language_model.pretrained_encoder_model_name and cfg.language_model.pretrained_decoder_model_name): raise ValueError("Both encoder and decoder must be specified") # setup tokenizers self.encoder_tokenizer = self.setup_tokenizer( cfg.encoder_tokenizer) self.encoder_add_special_tokens = cfg.encoder_tokenizer.add_special_tokens self.decoder_tokenizer = self.setup_tokenizer( cfg.decoder_tokenizer) self.decoder_add_special_tokens = cfg.decoder_tokenizer.add_special_tokens if not self.encoder_tokenizer: raise TypeError("encoder_tokenizer failed to initialize") if not self.decoder_tokenizer: raise TypeError("decoder_tokenizer failed to initialize") # init superclass super().__init__(cfg=cfg, trainer=trainer) # must assign modules after init if cfg.language_model.pretrained_model_name: # Setup end-to-end model if "bart" in cfg.language_model.pretrained_model_name: self.model = BartForConditionalGeneration.from_pretrained( cfg.language_model.pretrained_model_name) else: self.model = AutoModel.from_pretrained( cfg.language_model.pretrained_model_name) else: if not (cfg.language_model.pretrained_encoder_model_name and cfg.language_model.pretrained_decoder_model_name): raise ValueError("Both encoder and decoder must be specified") # Setup encoder/decoder model self.model = EncoderDecoderModel.from_encoder_decoder_pretrained( encoder=cfg.language_model.pretrained_encoder_model_name, decoder=cfg.language_model.pretrained_decoder_model_name, ) self.validation_perplexity = Perplexity(compute_on_step=False) self.setup_optimization(cfg.optim)
def __init__( self, pretrained_model=None, additional_special_tokens_encoder=None, additional_special_tokens_decoder=None, model_config=None, vocab_file=None, args=None, use_cuda=True, cuda_device=-1, **kwargs, ): self.args = self._load_model_args() if isinstance(args, dict): self.args.update_from_dict(args) elif isinstance(args, Seq2SeqArgs): self.args = args if "sweep_config" in kwargs: self.is_sweeping = True sweep_config = kwargs.pop("sweep_config") sweep_values = sweep_config_to_sweep_values(sweep_config) self.args.update_from_dict(sweep_values) else: self.is_sweeping = False if self.args.manual_seed: random.seed(self.args.manual_seed) np.random.seed(self.args.manual_seed) torch.manual_seed(self.args.manual_seed) if self.args.n_gpu > 0: torch.cuda.manual_seed_all(self.args.manual_seed) if use_cuda: if torch.cuda.is_available(): if cuda_device == -1: self.device = torch.device("cuda") else: self.device = torch.device(f"cuda:{cuda_device}") else: raise ValueError( "'use_cuda' set to True when cuda is unavailable." "Make sure CUDA is available or set `use_cuda=False`.") else: self.device = "cpu" self.results = {} if not use_cuda: self.args.fp16 = False # BartConfig, BartForConditionalGeneration, BartTokenizer # config = EncoderDecoderConfig.from_encoder_decoder_configs(config, config) model_config = BartConfig.from_json_file(model_config) if pretrained_model is None: self.model = BartForConditionalGeneration(config=model_config) self.encoder_tokenizer = BartTokenizer.from_pretrained(vocab_file) else: self.model = BartForConditionalGeneration.from_pretrained( pretrained_model) self.encoder_tokenizer = BartTokenizer.from_pretrained(vocab_file) self.decoder_tokenizer = self.encoder_tokenizer # special AST token # additional_special_tokens_encoder = {'additional_special_tokens': ['Assertion', 'RegExp', 'Repetition', 'Quantifier', 'ClassRange', 'CharacterClass']} # additional_special_tokens_decoder = {'additional_special_tokens': ['Assertion', 'RegExp', 'Repetition', 'Quantifier', 'ClassRange', 'CharacterClass']} self.config = self.model.config if additional_special_tokens_encoder is not None: self.encoder_tokenizer.add_special_tokens( additional_special_tokens_encoder) if additional_special_tokens_decoder is not None: self.decoder_tokenizer.add_special_tokens( additional_special_tokens_decoder) if self.args.wandb_project and not wandb_available: warnings.warn( "wandb_project specified but wandb is not available. Wandb disabled." ) self.args.wandb_project = None self.args.model_type = 'bart' self.args.model_name = 'ExplainREGEX'
import numpy as np from transformers import BartForConditionalGeneration as BCD, BartTokenizerFast as BTF import dataset import sys batch_size = int(sys.argv[1]) vit = timm.create_model('vit_base_patch32_384', pretrained=True, num_classes=0) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print(device) bart = BCD.from_pretrained('facebook/bart-base') tokenizer = BTF.from_pretrained('facebook/bart-base') transform = transforms.Compose([ transforms.ToPILImage(), transforms.Resize((384, 384)), transforms.ToTensor(), transforms.Lambda(dataset.make_img_rgb), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) trainset = dataset.NarrativesDataset(root='./data/images/', file='./data/dataset.jsonl', transform=transform) trainloader = torch.utils.data.DataLoader(trainset,
class BartModel: def __init__( self, pretrained_model=None, additional_special_tokens_encoder=None, additional_special_tokens_decoder=None, model_config=None, vocab_file=None, args=None, use_cuda=True, cuda_device=-1, **kwargs, ): self.args = self._load_model_args() if isinstance(args, dict): self.args.update_from_dict(args) elif isinstance(args, Seq2SeqArgs): self.args = args if "sweep_config" in kwargs: self.is_sweeping = True sweep_config = kwargs.pop("sweep_config") sweep_values = sweep_config_to_sweep_values(sweep_config) self.args.update_from_dict(sweep_values) else: self.is_sweeping = False if self.args.manual_seed: random.seed(self.args.manual_seed) np.random.seed(self.args.manual_seed) torch.manual_seed(self.args.manual_seed) if self.args.n_gpu > 0: torch.cuda.manual_seed_all(self.args.manual_seed) if use_cuda: if torch.cuda.is_available(): if cuda_device == -1: self.device = torch.device("cuda") else: self.device = torch.device(f"cuda:{cuda_device}") else: raise ValueError( "'use_cuda' set to True when cuda is unavailable." "Make sure CUDA is available or set `use_cuda=False`.") else: self.device = "cpu" self.results = {} if not use_cuda: self.args.fp16 = False # BartConfig, BartForConditionalGeneration, BartTokenizer # config = EncoderDecoderConfig.from_encoder_decoder_configs(config, config) model_config = BartConfig.from_json_file(model_config) if pretrained_model is None: self.model = BartForConditionalGeneration(config=model_config) self.encoder_tokenizer = BartTokenizer.from_pretrained(vocab_file) else: self.model = BartForConditionalGeneration.from_pretrained( pretrained_model) self.encoder_tokenizer = BartTokenizer.from_pretrained(vocab_file) self.decoder_tokenizer = self.encoder_tokenizer # special AST token # additional_special_tokens_encoder = {'additional_special_tokens': ['Assertion', 'RegExp', 'Repetition', 'Quantifier', 'ClassRange', 'CharacterClass']} # additional_special_tokens_decoder = {'additional_special_tokens': ['Assertion', 'RegExp', 'Repetition', 'Quantifier', 'ClassRange', 'CharacterClass']} self.config = self.model.config if additional_special_tokens_encoder is not None: self.encoder_tokenizer.add_special_tokens( additional_special_tokens_encoder) if additional_special_tokens_decoder is not None: self.decoder_tokenizer.add_special_tokens( additional_special_tokens_decoder) if self.args.wandb_project and not wandb_available: warnings.warn( "wandb_project specified but wandb is not available. Wandb disabled." ) self.args.wandb_project = None self.args.model_type = 'bart' self.args.model_name = 'ExplainREGEX' def train_model( self, train_data, output_dir=None, show_running_loss=True, args=None, eval_data=None, verbose=True, **kwargs, ): """ Trains the model using 'train_data' Args: train_data: Pandas DataFrame containing the 2 columns - `input_text`, `target_text`. - `input_text`: The input text sequence. - `target_text`: The target text sequence output_dir: The directory where model files will be saved. If not given, self.args.output_dir will be used. show_running_loss (optional): Set to False to prevent running loss from being printed to console. Defaults to True. args (optional): Optional changes to the args dict of the model. Any changes made will persist for the model. eval_data (optional): A DataFrame against which evaluation will be performed when evaluate_during_training is enabled. Is required if evaluate_during_training is enabled. **kwargs: Additional metrics that should be used. Pass in the metrics as keyword arguments (name of metric: function to use). A metric function should take in two parameters. The first parameter will be the true labels, and the second parameter will be the predictions. Both inputs will be lists of strings. Note that this will slow down training significantly as the predicted sequences need to be generated. Returns: global_step: Number of global steps trained training_details: Average training loss if evaluate_during_training is False or full training progress scores if evaluate_during_training is True """ # noqa: ignore flake8" if args: self.args.update_from_dict(args) # if self.args.silent: # show_running_loss = False if self.args.evaluate_during_training and eval_data is None: raise ValueError( "evaluate_during_training is enabled but eval_data is not specified." " Pass eval_data to model.train_model() if using evaluate_during_training." ) if not output_dir: output_dir = self.args.output_dir if os.path.exists(output_dir) and os.listdir( output_dir) and not self.args.overwrite_output_dir: raise ValueError( "Output directory ({}) already exists and is not empty." " Set args.overwrite_output_dir = True to overcome.".format( output_dir)) self._move_model_to_device() train_dataset = self.load_and_cache_examples(train_data, verbose=verbose) os.makedirs(output_dir, exist_ok=True) global_step, training_details = self.train( train_dataset, output_dir, show_running_loss=show_running_loss, eval_data=eval_data, verbose=verbose, **kwargs, ) self.save_model(self.args.output_dir, model=self.model) # model_to_save = self.model.module if hasattr(self.model, "module") else self.model # model_to_save.save_pretrained(output_dir) # self.encoder_tokenizer.save_pretrained(output_dir) # self.decoder_tokenizer.save_pretrained(output_dir) # torch.save(self.args, os.path.join(output_dir, "training_args.bin")) if verbose: logger.info(" Training of {} model complete. Saved to {}.".format( self.args.model_name, output_dir)) return global_step, training_details def train( self, train_dataset, output_dir, show_running_loss=True, eval_data=None, verbose=True, **kwargs, ): """ Trains the model on train_dataset. Utility function to be used by the train_model() method. Not intended to be used directly. """ model = self.model args = self.args tb_writer = SummaryWriter(logdir=args.tensorboard_dir) train_sampler = RandomSampler(train_dataset) train_dataloader = DataLoader( train_dataset, sampler=train_sampler, batch_size=args.train_batch_size, num_workers=self.args.dataloader_num_workers, ) if args.max_steps > 0: t_total = args.max_steps args.num_train_epochs = args.max_steps // ( len(train_dataloader) // args.gradient_accumulation_steps) + 1 else: t_total = len( train_dataloader ) // args.gradient_accumulation_steps * args.num_train_epochs no_decay = ["bias", "LayerNorm.weight"] optimizer_grouped_parameters = [] custom_parameter_names = set() for group in self.args.custom_parameter_groups: params = group.pop("params") custom_parameter_names.update(params) param_group = {**group} param_group["params"] = [ p for n, p in model.named_parameters() if n in params ] optimizer_grouped_parameters.append(param_group) for group in self.args.custom_layer_parameters: layer_number = group.pop("layer") layer = f"layer.{layer_number}." group_d = {**group} group_nd = {**group} group_nd["weight_decay"] = 0.0 params_d = [] params_nd = [] for n, p in model.named_parameters(): if n not in custom_parameter_names and layer in n: if any(nd in n for nd in no_decay): params_nd.append(p) else: params_d.append(p) custom_parameter_names.add(n) group_d["params"] = params_d group_nd["params"] = params_nd optimizer_grouped_parameters.append(group_d) optimizer_grouped_parameters.append(group_nd) if not self.args.train_custom_parameters_only: optimizer_grouped_parameters.extend([ { "params": [ p for n, p in model.named_parameters() if n not in custom_parameter_names and not any( nd in n for nd in no_decay) ], "weight_decay": args.weight_decay, }, { "params": [ p for n, p in model.named_parameters() if n not in custom_parameter_names and any( nd in n for nd in no_decay) ], "weight_decay": 0.0, }, ]) warmup_steps = math.ceil(t_total * args.warmup_ratio) args.warmup_steps = warmup_steps if args.warmup_steps == 0 else args.warmup_steps # TODO: Use custom optimizer like with BertSum? if args.optimizer == "AdamW": optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) elif args.optimizer == "Adafactor": optimizer = Adafactor( optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adafactor_eps, clip_threshold=args.adafactor_clip_threshold, decay_rate=args.adafactor_decay_rate, beta1=args.adafactor_beta1, weight_decay=args.weight_decay, scale_parameter=args.adafactor_scale_parameter, relative_step=args.adafactor_relative_step, warmup_init=args.adafactor_warmup_init, ) print("Using Adafactor for T5") else: raise ValueError( "{} is not a valid optimizer class. Please use one of ('AdamW', 'Adafactor') instead." .format(args.optimizer)) if args.scheduler == "constant_schedule": scheduler = get_constant_schedule(optimizer) elif args.scheduler == "constant_schedule_with_warmup": scheduler = get_constant_schedule_with_warmup( optimizer, num_warmup_steps=args.warmup_steps) elif args.scheduler == "linear_schedule_with_warmup": scheduler = get_linear_schedule_with_warmup( optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) elif args.scheduler == "cosine_schedule_with_warmup": scheduler = get_cosine_schedule_with_warmup( optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total, num_cycles=args.cosine_schedule_num_cycles, ) elif args.scheduler == "cosine_with_hard_restarts_schedule_with_warmup": scheduler = get_cosine_with_hard_restarts_schedule_with_warmup( optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total, num_cycles=args.cosine_schedule_num_cycles, ) elif args.scheduler == "polynomial_decay_schedule_with_warmup": scheduler = get_polynomial_decay_schedule_with_warmup( optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total, lr_end=args.polynomial_decay_schedule_lr_end, power=args.polynomial_decay_schedule_power, ) else: raise ValueError("{} is not a valid scheduler.".format( args.scheduler)) if (args.model_name and os.path.isfile( os.path.join(args.model_name, "optimizer.pt")) and os.path.isfile( os.path.join(args.model_name, "scheduler.pt"))): # Load in optimizer and scheduler states optimizer.load_state_dict( torch.load(os.path.join(args.model_name, "optimizer.pt"))) scheduler.load_state_dict( torch.load(os.path.join(args.model_name, "scheduler.pt"))) if args.n_gpu > 1: model = torch.nn.DataParallel(model) logger.info(" Training started") global_step = 0 training_progress_scores = None tr_loss, logging_loss = 0.0, 0.0 model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.silent, mininterval=0) epoch_number = 0 best_eval_metric = None early_stopping_counter = 0 steps_trained_in_current_epoch = 0 epochs_trained = 0 if args.model_name and os.path.exists(args.model_name): try: # set global_step to gobal_step of last saved checkpoint from model path checkpoint_suffix = args.model_name.split("/")[-1].split("-") if len(checkpoint_suffix) > 2: checkpoint_suffix = checkpoint_suffix[1] else: checkpoint_suffix = checkpoint_suffix[-1] global_step = int(checkpoint_suffix) epochs_trained = global_step // ( len(train_dataloader) // args.gradient_accumulation_steps) steps_trained_in_current_epoch = global_step % ( len(train_dataloader) // args.gradient_accumulation_steps) logger.info( " Continuing training from checkpoint, will skip to saved global_step" ) logger.info(" Continuing training from epoch %d", epochs_trained) logger.info(" Continuing training from global step %d", global_step) logger.info( " Will skip the first %d steps in the current epoch", steps_trained_in_current_epoch) except ValueError: logger.info(" Starting fine-tuning.") if args.evaluate_during_training: training_progress_scores = self._create_training_progress_scores( **kwargs) if args.wandb_project: wandb.init(project=args.wandb_project, config={**asdict(args)}, **args.wandb_kwargs) wandb.watch(self.model) if args.fp16: from torch.cuda import amp scaler = amp.GradScaler() for current_epoch in train_iterator: model.train() if epochs_trained > 0: epochs_trained -= 1 continue train_iterator.set_description( f"Epoch {epoch_number + 1} of {args.num_train_epochs}") batch_iterator = tqdm( train_dataloader, desc=f"Running Epoch {epoch_number} of {args.num_train_epochs}", disable=args.silent, mininterval=0, ) for step, batch in enumerate(batch_iterator): if steps_trained_in_current_epoch > 0: steps_trained_in_current_epoch -= 1 continue # batch = tuple(t.to(device) for t in batch) inputs = self._get_inputs_dict(batch) if args.fp16: with amp.autocast(): outputs = model(**inputs) # model outputs are always tuple in pytorch-transformers (see doc) loss = outputs[0] else: outputs = model(**inputs) # model outputs are always tuple in pytorch-transformers (see doc) loss = outputs[0] if args.n_gpu > 1: loss = loss.mean( ) # mean() to average on multi-gpu parallel training current_loss = loss.item() if show_running_loss: batch_iterator.set_description( f"Epochs {epoch_number}/{args.num_train_epochs}. Running Loss: {current_loss:9.4f}" ) if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps if args.fp16: scaler.scale(loss).backward() else: loss.backward() tr_loss += loss.item() if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: scaler.unscale_(optimizer) if args.optimizer == "AdamW": torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) if args.fp16: scaler.step(optimizer) scaler.update() else: optimizer.step() scheduler.step() # Update learning rate schedule model.zero_grad() global_step += 1 if args.logging_steps > 0 and global_step % args.logging_steps == 0: # Log metrics tb_writer.add_scalar("lr", scheduler.get_last_lr()[0], global_step) tb_writer.add_scalar("loss", (tr_loss - logging_loss) / args.logging_steps, global_step) logging_loss = tr_loss if args.wandb_project or self.is_sweeping: wandb.log({ "Training loss": current_loss, "lr": scheduler.get_last_lr()[0], "global_step": global_step, }) if args.save_steps > 0 and global_step % args.save_steps == 0: # Save model checkpoint output_dir_current = os.path.join( output_dir, "checkpoint-{}".format(global_step)) self.save_model(output_dir_current, optimizer, scheduler, model=model) if args.evaluate_during_training and ( args.evaluate_during_training_steps > 0 and global_step % args.evaluate_during_training_steps == 0): # Only evaluate when single GPU otherwise metrics may not average well results = self.eval_model( eval_data, verbose=verbose and args.evaluate_during_training_verbose, silent=args.evaluate_during_training_silent, **kwargs, ) for key, value in results.items(): tb_writer.add_scalar("eval_{}".format(key), value, global_step) output_dir_current = os.path.join( output_dir, "checkpoint-{}".format(global_step)) if args.save_eval_checkpoints: self.save_model(output_dir_current, optimizer, scheduler, model=model, results=results) training_progress_scores["global_step"].append( global_step) training_progress_scores["train_loss"].append( current_loss) for key in results: training_progress_scores[key].append(results[key]) report = pd.DataFrame(training_progress_scores) report.to_csv( os.path.join(args.output_dir, "training_progress_scores.csv"), index=False, ) if args.wandb_project or self.is_sweeping: wandb.log( self._get_last_metrics( training_progress_scores)) if not best_eval_metric: best_eval_metric = results[ args.early_stopping_metric] if args.save_best_model: self.save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results) if best_eval_metric and args.early_stopping_metric_minimize: if results[ args. early_stopping_metric] - best_eval_metric < args.early_stopping_delta: best_eval_metric = results[ args.early_stopping_metric] if args.save_best_model: self.save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results) early_stopping_counter = 0 else: if args.use_early_stopping: if early_stopping_counter < args.early_stopping_patience: early_stopping_counter += 1 if verbose: logger.info( f" No improvement in {args.early_stopping_metric}" ) logger.info( f" Current step: {early_stopping_counter}" ) logger.info( f" Early stopping patience: {args.early_stopping_patience}" ) else: if verbose: logger.info( f" Patience of {args.early_stopping_patience} steps reached" ) logger.info( " Training terminated.") train_iterator.close() return ( global_step, tr_loss / global_step if not self. args.evaluate_during_training else training_progress_scores, ) else: if results[ args. early_stopping_metric] - best_eval_metric > args.early_stopping_delta: best_eval_metric = results[ args.early_stopping_metric] if args.save_best_model: self.save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results) early_stopping_counter = 0 else: if args.use_early_stopping: if early_stopping_counter < args.early_stopping_patience: early_stopping_counter += 1 if verbose: logger.info( f" No improvement in {args.early_stopping_metric}" ) logger.info( f" Current step: {early_stopping_counter}" ) logger.info( f" Early stopping patience: {args.early_stopping_patience}" ) else: if verbose: logger.info( f" Patience of {args.early_stopping_patience} steps reached" ) logger.info( " Training terminated.") train_iterator.close() return ( global_step, tr_loss / global_step if not self. args.evaluate_during_training else training_progress_scores, ) model.train() epoch_number += 1 output_dir_current = os.path.join( output_dir, "checkpoint-{}-epoch-{}".format(global_step, epoch_number)) if args.save_model_every_epoch or args.evaluate_during_training: os.makedirs(output_dir_current, exist_ok=True) if args.save_model_every_epoch: self.save_model(output_dir_current, optimizer, scheduler, model=model) if args.evaluate_during_training and args.evaluate_each_epoch: results = self.eval_model( eval_data, verbose=verbose and args.evaluate_during_training_verbose, silent=args.evaluate_during_training_silent, **kwargs, ) if args.save_eval_checkpoints: self.save_model(output_dir_current, optimizer, scheduler, results=results) training_progress_scores["global_step"].append(global_step) training_progress_scores["train_loss"].append(current_loss) for key in results: training_progress_scores[key].append(results[key]) report = pd.DataFrame(training_progress_scores) report.to_csv(os.path.join(args.output_dir, "training_progress_scores.csv"), index=False) if args.wandb_project or self.is_sweeping: wandb.log(self._get_last_metrics(training_progress_scores)) if not best_eval_metric: best_eval_metric = results[args.early_stopping_metric] if args.save_best_model: self.save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results) if best_eval_metric and args.early_stopping_metric_minimize: if results[ args. early_stopping_metric] - best_eval_metric < args.early_stopping_delta: best_eval_metric = results[args.early_stopping_metric] if args.save_best_model: self.save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results) early_stopping_counter = 0 else: if args.use_early_stopping and args.early_stopping_consider_epochs: if early_stopping_counter < args.early_stopping_patience: early_stopping_counter += 1 if verbose: logger.info( f" No improvement in {args.early_stopping_metric}" ) logger.info( f" Current step: {early_stopping_counter}" ) logger.info( f" Early stopping patience: {args.early_stopping_patience}" ) else: if verbose: logger.info( f" Patience of {args.early_stopping_patience} steps reached" ) logger.info(" Training terminated.") train_iterator.close() return ( global_step, tr_loss / global_step if not self.args.evaluate_during_training else training_progress_scores, ) else: if results[ args. early_stopping_metric] - best_eval_metric > args.early_stopping_delta: best_eval_metric = results[args.early_stopping_metric] if args.save_best_model: self.save_model(args.best_model_dir, optimizer, scheduler, model=model, results=results) early_stopping_counter = 0 else: if args.use_early_stopping and args.early_stopping_consider_epochs: if early_stopping_counter < args.early_stopping_patience: early_stopping_counter += 1 if verbose: logger.info( f" No improvement in {args.early_stopping_metric}" ) logger.info( f" Current step: {early_stopping_counter}" ) logger.info( f" Early stopping patience: {args.early_stopping_patience}" ) else: if verbose: logger.info( f" Patience of {args.early_stopping_patience} steps reached" ) logger.info(" Training terminated.") train_iterator.close() return ( global_step, tr_loss / global_step if not self.args.evaluate_during_training else training_progress_scores, ) return ( global_step, tr_loss / global_step if not self.args.evaluate_during_training else training_progress_scores, ) def eval_model(self, eval_data, output_dir=None, verbose=True, silent=False, **kwargs): """ Evaluates the model on eval_data. Saves results to output_dir. Args: eval_data: Pandas DataFrame containing the 2 columns - `input_text`, `target_text`. - `input_text`: The input text sequence. - `target_text`: The target text sequence. output_dir: The directory where model files will be saved. If not given, self.args.output_dir will be used. verbose: If verbose, results will be printed to the console on completion of evaluation. silent: If silent, tqdm progress bars will be hidden. **kwargs: Additional metrics that should be used. Pass in the metrics as keyword arguments (name of metric: function to use). A metric function should take in two parameters. The first parameter will be the true labels, and the second parameter will be the predictions. Both inputs will be lists of strings. Note that this will slow down evaluation significantly as the predicted sequences need to be generated. Returns: results: Dictionary containing evaluation results. """ # noqa: ignore flake8" if not output_dir: output_dir = self.args.output_dir self._move_model_to_device() eval_dataset = self.load_and_cache_examples(eval_data, evaluate=True, verbose=verbose, silent=silent) os.makedirs(output_dir, exist_ok=True) result = self.evaluate(eval_dataset, output_dir, verbose=verbose, silent=silent, **kwargs) self.results.update(result) if self.args.evaluate_generated_text: to_predict = eval_data["input_text"].tolist() preds = self.predict(to_predict) result = self.compute_metrics(eval_data["target_text"].tolist(), preds, **kwargs) self.results.update(result) if verbose: logger.info(self.results) return self.results def evaluate(self, eval_dataset, output_dir, verbose=True, silent=False, **kwargs): """ Evaluates the model on eval_dataset. Utility function to be used by the eval_model() method. Not intended to be used directly. """ model = self.model args = self.args eval_output_dir = output_dir results = {} eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) if args.n_gpu > 1: model = torch.nn.DataParallel(model) eval_loss = 0.0 nb_eval_steps = 0 model.eval() if args.n_gpu > 1: model = torch.nn.DataParallel(model) if self.args.fp16: from torch.cuda import amp for batch in tqdm(eval_dataloader, disable=args.silent or silent, desc="Running Evaluation"): # batch = tuple(t.to(device) for t in batch) inputs = self._get_inputs_dict(batch) with torch.no_grad(): if self.args.fp16: with amp.autocast(): outputs = model(**inputs) tmp_eval_loss = outputs[0] else: outputs = model(**inputs) tmp_eval_loss = outputs[0] if self.args.n_gpu > 1: tmp_eval_loss = tmp_eval_loss.mean() eval_loss += tmp_eval_loss.item() nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps results["eval_loss"] = eval_loss output_eval_file = os.path.join(eval_output_dir, "eval_results.txt") with open(output_eval_file, "w") as writer: for key in sorted(results.keys()): writer.write("{} = {}\n".format(key, str(results[key]))) return results def predict(self, to_predict): """ Performs predictions on a list of text. Args: to_predict: A python list of text (str) to be sent to the model for prediction. Note that the prefix should be prepended to the text. Returns: preds: A python list of the generated sequences. """ # noqa: ignore flake8" self._move_model_to_device() all_outputs = [] # Batching for batch in tqdm( [ to_predict[i:i + self.args.eval_batch_size] for i in range(0, len(to_predict), self.args.eval_batch_size) ], desc="Generating outputs", disable=self.args.silent, ): input_ids = self.encoder_tokenizer.batch_encode_plus( batch, max_length=self.args.max_seq_length, padding="max_length", return_tensors="pt", truncation=True, )["input_ids"] input_ids = input_ids.to(self.device) outputs = self.model.generate( input_ids=input_ids, num_beams=self.args.num_beams, max_length=self.args.max_length, length_penalty=self.args.length_penalty, early_stopping=self.args.early_stopping, repetition_penalty=self.args.repetition_penalty, do_sample=self.args.do_sample, top_k=self.args.top_k, top_p=self.args.top_p, num_return_sequences=self.args.num_return_sequences, ) all_outputs.extend(outputs.cpu().numpy()) if self.args.use_multiprocessed_decoding: if self.args.multiprocessing_chunksize == -1: chunksize = max( len(all_outputs) // (self.args.process_count * 2), 500) else: chunksize = self.args.multiprocessing_chunksize self.model.to("cpu") with Pool(self.args.process_count) as p: outputs = list( tqdm( p.imap(self._decode, all_outputs, chunksize=chunksize), total=len(all_outputs), desc="Decoding outputs", disable=self.args.silent, )) self._move_model_to_device() else: outputs = [ self.decoder_tokenizer.decode( output_id, skip_special_tokens=self.args.skip_special_tokens, clean_up_tokenization_spaces=True) for output_id in all_outputs ] data_list = [] for data in outputs: if isinstance(data, str): data = data.replace('. ', '.') data = data.replace(' .', '.') if data.endswith('.'): data = data.replace('.', ' .') if data.endswith('?'): data = data.replace('?', ' ?') data_list.append(data) if self.args.num_return_sequences > 1: return [ data_list[i:i + self.args.num_return_sequences] for i in range( 0, len(data_list), self.args.num_return_sequences) ] else: return data_list def _decode(self, output_id): return self.decoder_tokenizer.decode( output_id, skip_special_tokens=self.args.skip_special_tokens, clean_up_tokenization_spaces=True) def compute_metrics(self, labels, preds, **kwargs): """ Computes the evaluation metrics for the model predictions. Args: labels: List of target sequences preds: List of model generated outputs **kwargs: Custom metrics that should be used. Pass in the metrics as keyword arguments (name of metric: function to use). A metric function should take in two parameters. The first parameter will be the true labels, and the second parameter will be the predictions. Both inputs will be lists of strings. Note that this will slow down evaluation significantly as the predicted sequences need to be generated. Returns: result: Dictionary containing evaluation results. """ # noqa: ignore flake8" # assert len(labels) == len(preds) results = {} for metric, func in kwargs.items(): results[metric] = func(labels, preds) return results def load_and_cache_examples(self, data, evaluate=False, no_cache=False, verbose=True, silent=False): """ Creates a T5Dataset from data. Utility function for train() and eval() methods. Not intended to be used directly. """ encoder_tokenizer = self.encoder_tokenizer decoder_tokenizer = self.decoder_tokenizer args = self.args if not no_cache: no_cache = args.no_cache if not no_cache: os.makedirs(self.args.cache_dir, exist_ok=True) mode = "dev" if evaluate else "train" if args.dataset_class: CustomDataset = args.dataset_class return CustomDataset(encoder_tokenizer, decoder_tokenizer, args, data, mode) else: return SimpleSummarizationDataset(encoder_tokenizer, self.args, data, mode) def _create_training_progress_scores(self, **kwargs): extra_metrics = {key: [] for key in kwargs} training_progress_scores = { "global_step": [], "eval_loss": [], "train_loss": [], **extra_metrics, } return training_progress_scores def _get_last_metrics(self, metric_values): return {metric: values[-1] for metric, values in metric_values.items()} def save_model(self, output_dir=None, optimizer=None, scheduler=None, model=None, results=None): if not output_dir: output_dir = self.args.output_dir os.makedirs(output_dir, exist_ok=True) logger.info(f"Saving model into {output_dir}") if model and not self.args.no_save: # Take care of distributed/parallel training model_to_save = model.module if hasattr(model, "module") else model self.save_model_args(output_dir) os.makedirs(os.path.join(output_dir), exist_ok=True) model_to_save.save_pretrained(output_dir) self.config.save_pretrained(output_dir) self.encoder_tokenizer.save_pretrained(output_dir) torch.save(self.args, os.path.join(output_dir, "training_args.bin")) if optimizer and scheduler and self.args.save_optimizer_and_scheduler: torch.save(optimizer.state_dict(), os.path.join(output_dir, "optimizer.pt")) torch.save(scheduler.state_dict(), os.path.join(output_dir, "scheduler.pt")) if results: output_eval_file = os.path.join(output_dir, "eval_results.txt") with open(output_eval_file, "w") as writer: for key in sorted(results.keys()): writer.write("{} = {}\n".format(key, str(results[key]))) def _move_model_to_device(self): self.model.to(self.device) def _get_inputs_dict(self, batch): device = self.device pad_token_id = self.encoder_tokenizer.pad_token_id source_ids, source_mask, y = batch["source_ids"], batch[ "source_mask"], batch["target_ids"] y_ids = y[:, :-1].contiguous() labels = y[:, 1:].clone() labels[y[:, 1:] == pad_token_id] = -100 inputs = { "input_ids": source_ids.to(device), "attention_mask": source_mask.to(device), "decoder_input_ids": y_ids.to(device), "labels": labels.to(device), } return inputs def save_model_args(self, output_dir): os.makedirs(output_dir, exist_ok=True) self.args.save(output_dir) def _load_model_args(self, input_dir=None): args = Seq2SeqArgs() if input_dir is not None: args.load(input_dir) return args def get_named_parameters(self): return [n for n, p in self.model.named_parameters()]
print(args) if args.t5_type == "e2e" and args.is_fquad == True: print( "WARNING: e2e is meant to generate questions by context. The ouput of the script will be a csv instead of a json." ) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print("Device:", device) model_created = False print("Loading model and tokenizer...", end="", flush=True) if args.checkpoint != None: model_created = True if args.bart: config = BartConfig.from_json_file(args.checkpoint + "/config.json") model = BartForConditionalGeneration.from_pretrained( args.checkpoint + "/pytorch_model.bin", config=config) if args.t5: config = T5Config.from_json_file(args.checkpoint + "/config.json") model = T5ForConditionalGeneration.from_pretrained( args.checkpoint + "/pytorch_model.bin", config=config) elif not args.bart and not args.t5: config = EncoderDecoderConfig.from_json_file(args.checkpoint + "/config.json") model = EncoderDecoderModel.from_pretrained(args.checkpoint + "/pytorch_model.bin", config=config) model_name = args.checkpoint if args.bart: if args.checkpoint == None: model_name = "WikinewsSum/bart-large-multi-fr-wiki-news" if args.model_name == "" else args.model_name
def run(args): save_args(args, with_tensorboard=True) tokenizer = BartTokenizer.from_pretrained("facebook/bart-large") tokenizer.bos_token = BOS_OUTPUT_TOKEN # For decoding specifically train_dataset, eval_dataset, test_dataset = [ SequentialJSONIterableDataset( os.path.join(args.datadir, f"{split}_*.clf.jsonl"), args=args, process_lines=False, reservoir_shuffle=shuffle, repeat=repeat, reservoir_size=args.reservoir_shuffle_size, ) for (split, shuffle, repeat) in [ ("train", True, True), ("valid", False, True), ("test", False, False), ] ] # Multiple inputs. Use Informativeness if args.input_type == "all": # ControlCode or generic Bart model = MultiInputBartForConditionalGeneration.from_pretrained( args.model_name_or_path ) # MultiHead if args.use_multi_head: model = MultiInputMultiHeadBartForConditionalGeneration.from_pretrained_multi( args.model_name_or_path ) elif args.use_multi_head: # MultiHead model = MultiHeadBartForConditionalGeneration.from_pretrained_multi( args.model_name_or_path ) else: # ControlCode or generic Bart model = BartForConditionalGeneration.from_pretrained(args.model_name_or_path) # Set special token IDs for eval function model.config.decoder_start_token_id = tokenizer.bos_token_id model.config.pad_token_id = ( tokenizer.pad_token_id ) # Might not be necessary, but idk if args.cuda: model = model.to("cuda") if args.distributed: model = utils_dist.wrap(model, args) optimizer = get_optimizer(args, model) if args.use_apex and HAS_APEX: model, optimizer = amp.initialize(model, optimizer, opt_level="O2") collate_fn_filled = functools.partial( collate_fn, input_type=args.input_type, modify_prefix=(not args.no_modify_prefix), target_type=args.target_type, ) if args.test_only: # run on test set print("=== TEST/EVAL ONLY, no training") named_splits = { "train": train_dataset, "valid": eval_dataset, "test": test_dataset, } selected_split = named_splits[args.test_split] eval_iter = DataLoader( selected_split, batch_size=args.eval_batch_size, collate_fn=collate_fn_filled, num_workers=1, worker_init_fn=worker_init_fn, ) results = evaluation(args, model, 0, tokenizer, eval_iter, write_summary=False) print(results["rouge_scores"]) # Save results in JSON file results_filename = Path(args.logdir) / f"{args.test_split}_results.json" with results_filename.open("w") as f: json.dump(results, f, indent=2, sort_keys=True) return model.train() global_step = 0 grad_acc_step = 0 loss_tensor_log = [] train_iter = DataLoader( train_dataset, batch_size=args.per_unit_batch_size, collate_fn=collate_fn_filled, num_workers=args.num_data_workers, worker_init_fn=worker_init_fn, ) eval_iter = DataLoader( eval_dataset, batch_size=args.eval_batch_size, collate_fn=collate_fn_filled, num_workers=args.num_data_workers, worker_init_fn=worker_init_fn, ) for _, (_, input_texts, output_texts) in enumerate(train_iter): if len(input_texts) == 0: continue # Prohibit batches with no contribution summaries at all if sum(len(out) for out in output_texts) == 0: continue # MultiHead + Auxiliary loss (Informativeness) if args.target_type == "both" and args.use_multi_head: if args.input_type == "paper": ips = [i for ip in input_texts for i in ip] elif args.input_type == "all": ips = [list(ip[0]) for ip in input_texts] ops = [o for op in output_texts for o in op] tok_input, tok_output, labels = tokenize_batch( ips, ops, model, tokenizer, args ) # Prepare inputs if args.input_type == "paper": tok_input["input_ids"] = tok_input["input_ids"].view( args.per_unit_batch_size, 2, -1 )[:, 0, :] tok_input["attention_mask"] = tok_input["attention_mask"].view( args.per_unit_batch_size, 2, -1 )[:, 0, :] additional_kwargs = {} elif args.input_type == "all": new_tok_input = {} new_tok_input["input_ids"] = [t["input_ids"] for t in tok_input] new_tok_input["attention_mask"] = [ t["attention_mask"] for t in tok_input ] tok_input = new_tok_input additional_kwargs = { "final_layer": [None, None, None], "input_modes": ["LogL", "MI_inbound", "MI_outbound"], "informativeness": args.use_informativeness, } # b x [cont, ctx] x seq_len tok_output["input_ids"] = tok_output["input_ids"].view( args.per_unit_batch_size, 2, -1 ) tok_output["attention_mask"] = tok_output["attention_mask"].view( args.per_unit_batch_size, 2, -1 ) labels = labels.view(args.per_unit_batch_size, 2, -1) # Fixing the strange behavior of torch.distributed where some values # are overwritten when the sequence length is just one. for b in range(args.per_unit_batch_size): tok_output["input_ids"][b][tok_output["input_ids"][b][:, 0] == 1, 0] = 2 tok_output["attention_mask"][b][ tok_output["attention_mask"][b][:, 0] == 0, 0 ] = 1 labels[b][labels[b][:, 0] == -100, 0] = 2 all_labels = [] all_dec_inputs = [] for idx in range(2): all_dec_inputs.append( dict( input_ids=tok_output["input_ids"][:, idx, :], attention_mask=tok_output["attention_mask"][:, idx, :], ) ) all_labels.append(labels[:, idx, :]) # Disable sync except at the beginning and the end of gradient accumulation if args.distributed: if (grad_acc_step == 0) or ( (grad_acc_step + 1) % args.gradient_accumulation_steps == 0 ): model.require_forward_param_sync = True model.require_backward_grad_sync = True else: model.require_forward_param_sync = False model.require_backward_grad_sync = False outs = model( input_ids=tok_input["input_ids"], attention_mask=tok_input["attention_mask"], decoder_input_ids=[ shift_left(tok_output["input_ids"], tokenizer.bos_token_id) for tok_output in all_dec_inputs ], decoder_attention_mask=[ tok_output["attention_mask"] for tok_output in all_dec_inputs ], lm_labels=all_labels, **additional_kwargs, ) # MultiHead + Informativeness if args.input_type == "all": # losses for generating both contrib & context if args.use_informativeness: # MI_outbound: informativeness contrib_loss = ( outs["LogL"][0][0] + args.aux_scale * outs["MI_outbound"][0][0] ) context_loss = ( outs["LogL"][1][0] + args.aux_scale * outs["MI_inbound"][1][0] ) else: contrib_loss = ( outs["LogL"][0][0] - args.aux_scale * outs["MI_inbound"][0][0] + ( args.aux_scale * outs["MI_outbound"][0][0] if not args.use_adaptive_scale else 0 ) ) context_loss = ( outs["LogL"][1][0] + ( args.aux_scale * outs["MI_inbound"][1][0] if not args.use_adaptive_scale else 0 ) - args.aux_scale * outs["MI_outbound"][1][0] ) loss = (contrib_loss + context_loss) / 2 losses = [ outs["LogL"][0][0], outs["MI_inbound"][0][0], outs["MI_outbound"][0][0], outs["LogL"][1][0], outs["MI_inbound"][1][0], outs["MI_outbound"][1][0], ] # multihead else: # contrib, context losses = [o[0] for o in outs] loss = sum(losses) / len(losses) check_nan(loss) # reporting logL only loss_tensor_log.append( (losses[0] if args.input_type == "all" else loss).detach() ) global_step, grad_acc_step, loss_tensor_log = update_step( args, model, tokenizer, optimizer, loss, losses, eval_iter, global_step, grad_acc_step, loss_tensor_log, ) else: # For compatibility of training loop if args.target_type != "both": input_texts, output_texts = ([input_texts], [output_texts]) input_texts, output_texts = zip(*input_texts), zip(*output_texts) heads = ["contrib", "context"] losses = [] # loop over the two targets for input_text, output_text, head in zip(input_texts, output_texts, heads): tok_input, tok_output, labels = tokenize_batch( input_text, output_text, model, tokenizer, args ) if args.distributed: if (grad_acc_step == 0) or ( (grad_acc_step + 1) % args.gradient_accumulation_steps == 0 ): model.require_forward_param_sync = True model.require_backward_grad_sync = True else: model.require_forward_param_sync = False model.require_backward_grad_sync = False # Auxiliary loss: informativeness if args.input_type == "all": outs = model( input_ids=[t["input_ids"] for t in tok_input], attention_mask=[t["attention_mask"] for t in tok_input], decoder_input_ids=shift_left( tok_output["input_ids"], tokenizer.bos_token_id ), decoder_attention_mask=tok_output["attention_mask"], lm_labels=labels, ) else: outs = model( input_ids=tok_input["input_ids"], attention_mask=tok_input["attention_mask"], decoder_input_ids=shift_left( tok_output["input_ids"], tokenizer.bos_token_id ), decoder_attention_mask=tok_output["attention_mask"], lm_labels=labels, ) if args.input_type == "all": losses += outs[0] if args.target_type == "contrib": if args.use_informativeness: coeff = [ 1, 0, args.aux_scale, ] else: coeff = [ 1, -args.aux_scale, args.aux_scale if not args.use_adaptive_scale else 0, ] elif args.target_type == "context": if args.use_informativeness: coeff = [ 1, args.aux_scale, 0, ] else: coeff = [ 1, args.aux_scale if not args.use_adaptive_scale else 0, -args.aux_scale, ] loss = sum(l * c for l, c in zip(outs[0], coeff)) elif args.use_multi_head: loss = outs[0 if head == "contrib" else 1][0] else: loss = outs[0] check_nan(loss) loss_tensor_log.append( (losses[0] if args.input_type == "all" else loss).detach() ) global_step, grad_acc_step, loss_tensor_log = update_step( args, model, tokenizer, optimizer, loss, losses, eval_iter, global_step, grad_acc_step, loss_tensor_log, )
class KoBartModel(object): """KoBart Model from SKT""" def __init__(self, model: str, device: str): config = BartConfig.from_pretrained("hyunwoongko/kobart") self.model = BartForConditionalGeneration(config).half().eval().to( device) self.model.model.load_state_dict(torch.load( model, map_location=device, )) self.tokenizer = PreTrainedTokenizerFast.from_pretrained( "hyunwoongko/kobart") self.device = device @classmethod def from_pretrained( cls, device: str, model_path: str = "path/to/model.pt", ): """ load pretrained model from disk. this method is equivalent with constructor. Args: device (str): device model_path (str): full model path Returns: (KoBartModel): object of KoBartModel """ return cls(model=model_path, device=device) def tokenize( self, texts: List[str], max_len: int = 1024, ) -> Dict: if isinstance(texts, str): texts = [texts] texts = [f"<s> {text}" for text in texts] eos = self.tokenizer.convert_tokens_to_ids(self.tokenizer.eos_token) eos_list = [eos for _ in range(len(texts))] tokens = self.tokenizer( texts, return_tensors="pt", padding=True, truncation=True, add_special_tokens=False, max_length=max_len - 1, # result + <eos> ) return self.add_bos_eos_tokens(tokens, eos_list) def add_bos_eos_tokens(self, tokens, eos_list): input_ids = tokens["input_ids"] attention_mask = tokens["attention_mask"] token_added_ids, token_added_masks = [], [] for input_id, atn_mask, eos in zip( input_ids, attention_mask, eos_list, ): maximum_idx = [ i for i, val in enumerate(input_id) if val != self.tokenizer.convert_tokens_to_ids("<pad>") ] if len(maximum_idx) == 0: idx_to_add = 0 else: idx_to_add = max(maximum_idx) + 1 eos = torch.tensor([eos], requires_grad=False) additional_atn_mask = torch.tensor([1], requires_grad=False) input_id = torch.cat([ input_id[:idx_to_add], eos, input_id[idx_to_add:], ]).long() atn_mask = torch.cat([ atn_mask[:idx_to_add], additional_atn_mask, atn_mask[idx_to_add:], ]).long() token_added_ids.append(input_id.unsqueeze(0)) token_added_masks.append(atn_mask.unsqueeze(0)) tokens["input_ids"] = torch.cat(token_added_ids, dim=0) tokens["attention_mask"] = torch.cat(token_added_masks, dim=0) return tokens @torch.no_grad() def translate( self, text: str, beam: int = 5, sampling: bool = False, temperature: float = 1.0, sampling_topk: int = -1, sampling_topp: float = -1, length_penalty: float = 1.0, max_len_a: int = 1, max_len_b: int = 50, no_repeat_ngram_size: int = 4, return_tokens: bool = False, bad_words_ids=None, ): """ generate sentence from input sentence. See Also: 1. method and argument names follow fairseq.models.transformer.TransformerModel >>> from fairseq.models.transformer import TransformerModel 2. language codes follow farseq language codes >>> from transformers.tokenization_mbart import FAIRSEQ_LANGUAGE_CODES Args: text (str): input string beam (int): beam size sampling (bool): sampling or not temperature (float): temperature value sampling_topk (int): topk sampling sampling_topp (float): topp sampling probs return_tokens (bool): return tokens or not Returns: (str): generated sentence string (if return_tokens=False) (List[str]): list of generated tokens (if return_tokens=True) """ if isinstance(text, str): texts = [text] else: texts = text tokenized = self.tokenize(texts) input_ids = tokenized["input_ids"] attention_mask = tokenized["attention_mask"] generated = self.model.generate( input_ids.to(self.device), attention_mask=attention_mask.to(self.device), use_cache=True, early_stopping=False, decoder_start_token_id=self.tokenizer.bos_token_id, num_beams=beam, do_sample=sampling, temperature=temperature, top_k=sampling_topk if sampling_topk > 0 else None, top_p=sampling_topp if sampling_topk > 0 else None, no_repeat_ngram_size=no_repeat_ngram_size, bad_words_ids=[[self.tokenizer.convert_tokens_to_ids("<unk>")]] if not bad_words_ids else bad_words_ids + [[self.tokenizer.convert_tokens_to_ids("<unk>")]], length_penalty=length_penalty, max_length=max_len_a * len(input_ids[0]) + max_len_b, ) if return_tokens: output = [ self.tokenizer.convert_ids_to_tokens(_) for _ in generated.tolist() ] return (output[0] if isinstance( text, str, ) else output) else: output = self.tokenizer.batch_decode( generated.tolist(), skip_special_tokens=True, ) return (output[0].strip() if isinstance( text, str, ) else [o.strip() for o in output])
def __init__(self): self.model = BartForConditionalGeneration.from_pretrained("./models/bart-coder") self.tokenizer = BartTokenizer.from_pretrained("facebook/bart-base")
def __init__(self, args): """ currently we use the model `bart-large-cnn` """ self.model = BartForConditionalGeneration.from_pretrained('model_id') self.tokenizer = BartTokenizer.from_pretrained(args['model_id'])
raise NotImplementedError def get_iter(): data = read_qa_data(args.data, eval(args.domains.upper()), split=args.split, has_ret=args.has_ret, use_inp=args.use_inp, load=args.num_options > 0) iter = string_to_tensor(tokenizer, data, max_input_len=max_input_len, max_target_len=max_target_len, max_token_per_batch=args.max_token_per_batch, append_bos=append_bos, append_eos=append_eos, num_options=args.num_options, qa_merge=qa_merge, device=device) return iter print('loading models ...') if args.model == 'facebook/bart-large': model = MyBart.from_pretrained(base_model).to(device) else: if args.model_type == 'bart': model = MyBart.from_pretrained(base_model, state_dict=torch.load(args.model)).to(device) elif args.model_type == 'bart_raw': model = BartForConditionalGeneration.from_pretrained(base_model, state_dict=torch.load(args.model)).to(device) elif args.model_type == 'gpt2': model = MyGPT2.from_pretrained(base_model, state_dict=torch.load(args.model)).to(device) else: raise NotImplementedError os.makedirs(os.path.dirname(args.output), exist_ok=True) if args.task == 'generate': model.eval() with open(args.data, 'r') as fin, open(args.output, 'w') as fout: for l in tqdm(fin): question = l.strip() res = model.generate(torch.LongTensor([[0] + tokenizer.encode(question)]).to(device), num_beams=1, min_length=1, max_length=50, early_stopping=False, num_return_sequences=1) res = [tokenizer.decode(x, skip_special_tokens=True).strip() for x in res] fout.write('{}\n'.format(res[0]))
def get_kobart_for_conditional_generation(): return BartForConditionalGeneration.from_pretrained("hyunwoongko/kobart")
def __init__( self, model_name_or_path, # teacher tokenizer_name, model_cache_dir, input_max_length, target_max_length, summary_column_name, document_column_name, wandb_project, wandb_run_name, student_encoder_layers, student_decoder_layers, **kwargs, ): super().__init__( input_max_length, target_max_length, summary_column_name, document_column_name, wandb_project, wandb_run_name, ) self.tokenizer = BartTokenizer.from_pretrained( tokenizer_name if tokenizer_name else model_name_or_path, cache_dir=model_cache_dir, ) teacher = BartForConditionalGeneration.from_pretrained( model_name_or_path, cache_dir=model_cache_dir, ).eval() student_updates = { "decoder_layers": student_decoder_layers, "encoder_layers": student_encoder_layers, } d_layers_to_copy = self._get_layers_to_copy(student_updates["decoder_layers"], teacher.config.decoder_layers) e_layers_to_copy: List = self._get_layers_to_copy(student_updates["encoder_layers"], teacher.config.encoder_layers) kw = teacher.config.to_diff_dict() kw.update(student_updates) # Copy weights student_cfg = BartConfig(**kw) student = BartForConditionalGeneration(student_cfg) student, _ = self._init_student(student, teacher) self._copy_to_student(d_layers_to_copy, e_layers_to_copy, student_encoder_layers, student_decoder_layers, student, teacher) self.model = student print(student) inputs = self.tokenizer.encode_plus("TEXT TO SUMMARIZE", max_length=1024, return_tensors="pt") # Summarize outputs = self.model.generate( input_ids=inputs['input_ids'], attention_mask=inputs['attention_mask'], max_length=400, min_length=150, length_penalty=2.0, num_beams=4, early_stopping=True ) # Decode summary = self.tokenizer.decode(outputs[0], skip_special_tokens=True, clean_up_tokenization_spaces=False) print(summary)
from flask import Flask, request, render_template from regression import model1, tokenizer_new, tokenize_new import numpy as np from transformers import BartForConditionalGeneration, BartTokenizer, BartConfig import torch config = BartConfig.from_json_file('output_model/hate/config.json') model = BartForConditionalGeneration.from_pretrained('output_model/hate/') tok = BartTokenizer.from_pretrained('output_model/hate/') app = Flask(__name__) app.debug = True @app.route("/", methods=['GET', 'POST']) def index(): if request.method == "POST": name = request.form["name"] hate = " " if (len(name) > 0): if name.split(" ")[-1] == '': a, b, c = tokenize_new([name], tokenizer_new) out = np.round(model1.predict([a, b])[0][0]) if out <= 3: hate = "No Hate detected" elif out > 3 and out <= 5: hate = "LOW" elif out > 5 and out <= 7: hate = "MEDIUM" else: hate = "HIGH"
import argparse from pathlib import Path import torch from tqdm import tqdm from transformers import BartForConditionalGeneration, BartTokenizer torch_device = 'cuda' if torch.cuda.is_available() else 'cpu' DEFAULT_DEVICE = "cuda" if torch.cuda.is_available() else "cpu" SUMMARY_TOKENIZER = BartTokenizer.from_pretrained('bart-large-cnn') SUMMARY_MODEL = BartForConditionalGeneration.from_pretrained('bart-large-cnn') SUMMARY_MODEL.to(torch_device) SUMMARY_MODEL.eval() def clean_text(text): tokens = text.split(" ") if '[SEP]' in tokens: sepind = tokens.index('[SEP]') tokens = tokens[sepind + 1:] txt = ' '.join(tokens) txt = txt.replace(' ##', '') txt = txt.replace('##', '') txt = txt.strip() txt = " ".join(txt.split()) txt = txt.replace(' .', '.') txt = txt.replace('( ', '(') txt = txt.replace(' )', ')')
def test_generate_fp16(self): config, input_ids, batch_size = self._get_config_and_data() attention_mask = input_ids.ne(1).to(torch_device) model = BartForConditionalGeneration(config).eval().to(torch_device).half() model.generate(input_ids, attention_mask=attention_mask, do_sample=False, early_stopping=True)
def Seq2Seq(df): model_type = 'facebook/bart-large' tokenizer = BartTokenizer.from_pretrained(model_type) model = BartModel.from_pretrained(model_type) mask_model = BartForConditionalGeneration.from_pretrained(model_type) sep_token = '</s>' mask_token = '<mask>' mask_id = tokenizer(mask_token, return_tensors='pt')['input_ids'][0][1] sep_id = tokenizer(sep_token, return_tensors='pt')['input_ids'][0][1] optimizer = AdamW(model.parameters()) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) mask_model.to(device) df['mask_text'] = 0 df['auxiliary_text'] = 0 for i in range(len(df)): sentiment = df['sentiment'].iloc[i] aspect = 'sentence' if DPM_type == 'Senti': mask_sent = 'the polarity of the ' + aspect + ' is ' + mask_token + ' ' + sep_token + ' ' auxiliary_sent = 'the polarity of the ' + aspect + ' is ' + sentiment + ' ' + sep_token + ' ' df['mask_text'].iloc[i] = mask_sent + df['text'].iloc[i] df['auxiliary_text'].iloc[i] = auxiliary_sent + df['text'].iloc[i] df['distance'] = 0 df = df.astype('object') for i in range(len(df)): tokenized = df['mask_text'][i:i+1].apply((lambda x: tokenizer.encode(x, add_special_tokens=True, max_length=MAX_LEN, truncation=True))) sep_index = tokenized[i].index(sep_id) mask_index = tokenized[i].index(mask_id) padded = pad_sequences(tokenized, maxlen=MAX_LEN, dtype="long", value=0, truncating="post", padding="post") attention_mask = np.where(padded != 0, 1, 0) input_ids = torch.tensor(padded).to(device) attention_mask = torch.tensor(attention_mask).to(device) with torch.no_grad(): last_hidden_states = model(input_ids, attention_mask=attention_mask) original_mask_embedding = last_hidden_states[0][:, mask_index, :].cpu().numpy() distance = [] for pertubed_index in range(sep_index+1, MAX_LEN): padded = pad_sequences(tokenized, maxlen=MAX_LEN, dtype="long", value=0, truncating="post", padding="post") if padded[0][pertubed_index] != 0 and padded[0][pertubed_index] != sep_id: #print(padded.shape) cur_id = padded[0][pertubed_index] padded[0][pertubed_index] = mask_id cur_embedding = mask_embedding(model, padded, mask_index) d = dist(original_mask_embedding, cur_embedding) distance.append((cur_id, d)) df['distance'].iloc[i] = distance df['perturbed_mask_index'] = 0 df = df.astype('object') for i in range(len(df)): perturbed_mask_index = [] mask_threshold = calculate_threshold(np.array(df['distance'].iloc[i])[:, 1], std_strength) for dis_index in range(len(df['distance'].iloc[i])): if df['distance'].iloc[i][dis_index][1] < mask_threshold: perturbed_mask_index.append(dis_index) df['perturbed_mask_index'].iloc[i] = perturbed_mask_index df['augment_token_id'] = 0 df = df.astype('object') for i in range(len(df)): tokenized = tokenizer.encode(df['auxiliary_text'].iloc[i]) tokenized = torch.Tensor(tokenized).unsqueeze(0).to(torch.int64).to(device) augment_tokenized = tokenizer.encode(df['auxiliary_text'].iloc[i]) mask_tokenized = tokenizer.encode(df['auxiliary_text'].iloc[i]) sep_index = mask_tokenized.index(sep_id) for j in range(len(df['perturbed_mask_index'].iloc[i])): perturbed_mask_index = df['perturbed_mask_index'].iloc[i][j] + sep_index + 1 mask_tokenized[perturbed_mask_index] = mask_id mask_tokenized = torch.Tensor(mask_tokenized).unsqueeze(0).to(torch.int64).to(device) logits = mask_model(mask_tokenized).logits for j in range(len(df['perturbed_mask_index'].iloc[i])): perturbed_mask_index = df['perturbed_mask_index'].iloc[i][j] + sep_index + 1 probs = logits[0, perturbed_mask_index].softmax(dim=0) values, predictions = probs.topk(1) augment_tokenized[perturbed_mask_index] = int(predictions.cpu().numpy()) df['augment_token_id'].iloc[i] = augment_tokenized df['augment_text'] = 0 df = df.astype('object') for i in range(len(df)): sep_index = df['augment_token_id'].iloc[i].index(sep_id) df['augment_text'].iloc[i] = tokenizer.decode(df['augment_token_id'].iloc[i][sep_index+1:-1]) return df
def test_default_generate_kwargs(self): config, input_ids, _ = self._get_config_and_data() model = BartForConditionalGeneration(config).eval().to(torch_device) model.generate(input_ids) model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3)
def load_class(self): # Load the tokenizer. if self.verbose == True: print('Loading {} class...'.format(self.model_name)) if self.model_name == 'bert': # Load BertForSequenceClassification, the pretrained BERT model with a single # linear classification layer on top. self.model = BertForSequenceClassification.from_pretrained( self. model_type, # Use the 12-layer BERT model, with an uncased vocab. # You can increase this for multi-class tasks. num_labels=self.num_labels, output_attentions= False, # Whether the model returns attentions weights. output_hidden_states= False, # Whether the model returns all hidden-states. ) if self.model_name == 'distilbert': self.model = DistilBertForSequenceClassification.from_pretrained( self.model_type, num_labels=self.num_labels, output_attentions=False, output_hidden_states=False, ) if self.model_name == 'albert': self.model = AlbertForSequenceClassification.from_pretrained( self.model_type, num_labels=self.num_labels, output_attentions=False, output_hidden_states=False, ) if self.model_name == 'bart': if self.task == 'classification': self.model = BartForSequenceClassification.from_pretrained( self.model_type, num_labels=self.num_labels, output_attentions=False, output_hidden_states=False, ) if self.task == 'summarize': self.model = BartForConditionalGeneration.from_pretrained( self.model_type) if self.model_name == 'xlnet': self.model = XLNetForSequenceClassification.from_pretrained( self.model_type, num_labels=self.num_labels, output_attentions=False, output_hidden_states=False, ) if self.model_name == 'roberta': self.model = RobertaForSequenceClassification.from_pretrained( self.model_type, num_labels=self.num_labels, output_attentions=False, output_hidden_states=False, ) if self.model_name == 'camenbert': self.model = CamembertForSequenceClassification.from_pretrained( self.model_type, num_labels=self.num_labels, output_attentions=False, output_hidden_states=False, ) if self.model_name == 'flaubert': self.model = FlaubertForSequenceClassification.from_pretrained( self.model_type, num_labels=self.num_labels, output_attentions=False, output_hidden_states=False, ) if self.model_name == 'gpt2': self.model = GPT2LMHeadModel.from_pretrained(self.model_type)
ALLOWED_EXTENSIONS = {'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'} HOSTNAME = '83.212.102.161' app = Flask(__name__) app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER app.config['JSONIFY_PRETTYPRINT_REGULAR'] = True #app.secret_key = "super secret key" #app.config['JSON_AS_ASCII'] = False bootstrap = Bootstrap(app) torch_device = 'cuda' if torch.cuda.is_available() else 'cpu' tokenizer = BartTokenizer.from_pretrained('bart-large-cnn') model = BartForConditionalGeneration.from_pretrained('bart-large-cnn') results = [] articles = [] id = 0 def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS def submit(LONG_TEXT, id, max_words): article_input_ids = tokenizer.batch_encode_plus( [LONG_TEXT], return_tensors='pt', max_length=1024)['input_ids'].to(torch_device)
# Original index ## Base BERT from transformers import BertTokenizer, BertForMaskedLM ## Base XLNet from transformers import XLNetTokenizer, XLNetLMHeadModel xlnet_tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') xlnet_model = XLNetLMHeadModel.from_pretrained('xlnet-base-cased').eval() ## Base XLMRoberta from transformers import XLMRobertaTokenizer, XLMRobertaForMaskedLM xlmroberta_tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-base') xlmroberta_model = XLMRobertaForMaskedLM.from_pretrained( 'xlm-roberta-base').eval() ## Base Bart from transformers import BartTokenizer, BartForConditionalGeneration bart_tokenizer = BartTokenizer.from_pretrained('facebook/bart-large') bart_model = BartForConditionalGeneration.from_pretrained( 'facebook/bart-large').eval() ## Base Eletra from transformers import ElectraTokenizer, ElectraForMaskedLM electra_tokenizer = ElectraTokenizer.from_pretrained( 'google/electra-small-generator') electra_model = ElectraForMaskedLM.from_pretrained( 'google/electra-small-generator').eval() ## Base Roberta from transformers import RobertaTokenizer, RobertaForMaskedLM roberta_tokenizer = RobertaTokenizer.from_pretrained('roberta-base') roberta_model = RobertaForMaskedLM.from_pretrained('roberta-base').eval() ## Bert VNese base uncased from transformers import AutoTokenizer, AutoModelForMaskedLM phobert_tokenizer = AutoTokenizer.from_pretrained("vinai/phobert-base") phobert_model = AutoModelForMaskedLM.from_pretrained("vinai/phobert-base")
def make_model(model_pth=None): model = BartForConditionalGeneration.from_pretrained('facebook/bart-large') if model_pth != None: print("model loaded from {}".format(model_pth)) model.load_state_dict(torch.load(model_pth)) return model
'a', encoding='utf8') as fw: fw.write(' .'.join(i.split('.')).replace('<n>', ' ')) fw.write('\n') print('inference_time: {}'.format(time.time() - start)) if __name__ == '__main__': assert args.dataset in ['cnndm','xsum','newsroom','multi-news','billsum','reddit','wikihow','arxiv','pubmed'],\ '--dataset should be cnndm, xsum, newsroom, multi-news, billsum, reddit, wikihow, arxiv, pubmed' summ_use = None if args.dataset == 'cnndm': if not args.peg: path = 'facebook/bart-large-cnn' summ = BartForConditionalGeneration.from_pretrained( path, use_cache=False) tokenizer = BartTokenizer.from_pretrained( 'facebook/bart-large-cnn') if args.dataset == 'xsum': if not args.peg: path = 'facebook/bart-large-xsum' summ = BartForConditionalGeneration.from_pretrained( path, use_cache=False) summ_use = BartForConditionalGeneration.from_pretrained(path) tokenizer = BartTokenizer.from_pretrained( 'facebook/bart-large-xsum') else: path = 'google/pegasus-xsum' summ = PegasusForConditionalGeneration.from_pretrained( path, use_cache=False) if not args.train: