def load_and_cache_examples(self, examples, evaluate=False, no_cache=False): """ Converts a list of InputExample objects to a TensorDataset containing InputFeatures. Caches the InputFeatures. Utility function for train() and eval() methods. Not intended to be used directly. """ process_count = self.args["process_count"] tokenizer = self.tokenizer output_mode = "classification" args = self.args if not os.path.isdir(self.args["cache_dir"]): os.mkdir(self.args["cache_dir"]) mode = "dev" if evaluate else "train" cached_features_file = os.path.join(args["cache_dir"], "cached_{}_{}_{}_binary".format(mode, args["model_type"], args["max_seq_length"])) if os.path.exists(cached_features_file) and not args["reprocess_input_data"] and not no_cache: features = torch.load(cached_features_file) if not args["silent"]: print(f"Features loaded from cache at {cached_features_file}") else: if not args["silent"]: print(f"Converting to features started.") features = convert_examples_to_features( examples, args["max_seq_length"], tokenizer, output_mode, # XLNet has a CLS token at the end cls_token_at_end=bool(args["model_type"] in ["xlnet"]), cls_token=tokenizer.cls_token, cls_token_segment_id=2 if args["model_type"] in ["xlnet"] else 0, sep_token=tokenizer.sep_token, # RoBERTa uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 sep_token_extra=bool(args["model_type"] in ["roberta"]), # PAD on the left for XLNet pad_on_left=bool(args["model_type"] in ["xlnet"]), pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], pad_token_segment_id=4 if args["model_type"] in ["xlnet"] else 0, process_count=process_count ) if not no_cache: torch.save(features, cached_features_file) all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long) if output_mode == "classification": all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long) elif output_mode == "regression": all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float) dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) return dataset
def load_and_cache_examples(self, examples, evaluate=False, no_cache=False): """ Converts a list of InputExample objects to a TensorDataset containing InputFeatures. Caches the InputFeatures. Utility function for train() and eval() methods. Not intended to be used directly. """ process_count = self.args['process_count'] tokenizer = self.tokenizer output_mode = 'classification' args=self.args if not os.path.isdir(self.args['cache_dir']): os.mkdir(self.args['cache_dir']) mode = 'dev' if evaluate else 'train' cached_features_file = os.path.join(args['cache_dir'], f"cached_{mode}_{args['model_type']}_{args['max_seq_length']}_binary") if os.path.exists(cached_features_file) and not args['reprocess_input_data'] and not no_cache: features = torch.load(cached_features_file) else: features = convert_examples_to_features(examples, args['max_seq_length'], tokenizer, output_mode, # xlnet has a cls token at the end cls_token_at_end=bool(args['model_type'] in ['xlnet']), cls_token=tokenizer.cls_token, cls_token_segment_id=2 if args['model_type'] in ['xlnet'] else 0, sep_token=tokenizer.sep_token, # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 sep_token_extra=bool(args['model_type'] in ['roberta']), # pad on the left for xlnet pad_on_left=bool(args['model_type'] in ['xlnet']), pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], pad_token_segment_id=4 if args['model_type'] in ['xlnet'] else 0, process_count=process_count) if not no_cache: torch.save(features, cached_features_file) all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long) if output_mode == "classification": all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long) elif output_mode == "regression": all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float) dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) return dataset