def load_and_cache_examples(self, examples, evaluate=False, no_cache=False):
"""
Converts a list of InputExample objects to a TensorDataset containing InputFeatures. Caches the InputFeatures.
Utility function for train() and eval() methods. Not intended to be used directly.
"""
process_count = self.args["process_count"]
tokenizer = self.tokenizer
output_mode = "classification"
args = self.args
if not os.path.isdir(self.args["cache_dir"]):
os.mkdir(self.args["cache_dir"])
mode = "dev" if evaluate else "train"
cached_features_file = os.path.join(args["cache_dir"], "cached_{}_{}_{}_binary".format(mode, args["model_type"], args["max_seq_length"]))
if os.path.exists(cached_features_file) and not args["reprocess_input_data"] and not no_cache:
features = torch.load(cached_features_file)
if not args["silent"]:
print(f"Features loaded from cache at {cached_features_file}")
else:
if not args["silent"]:
print(f"Converting to features started.")
features = convert_examples_to_features(
examples,
args["max_seq_length"],
tokenizer,
output_mode,
# XLNet has a CLS token at the end
cls_token_at_end=bool(args["model_type"] in ["xlnet"]),
cls_token=tokenizer.cls_token,
cls_token_segment_id=2 if args["model_type"] in ["xlnet"] else 0,
sep_token=tokenizer.sep_token,
# RoBERTa uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
sep_token_extra=bool(args["model_type"] in ["roberta"]),
# PAD on the left for XLNet
pad_on_left=bool(args["model_type"] in ["xlnet"]),
pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
pad_token_segment_id=4 if args["model_type"] in ["xlnet"] else 0,
process_count=process_count
)
if not no_cache:
torch.save(features, cached_features_file)
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)
dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
return dataset
def load_and_cache_examples(self, examples, evaluate=False, no_cache=False):
"""
Converts a list of InputExample objects to a TensorDataset containing InputFeatures. Caches the InputFeatures.
Utility function for train() and eval() methods. Not intended to be used directly.
"""
process_count = self.args['process_count']
tokenizer = self.tokenizer
output_mode = 'classification'
args=self.args
if not os.path.isdir(self.args['cache_dir']):
os.mkdir(self.args['cache_dir'])
mode = 'dev' if evaluate else 'train'
cached_features_file = os.path.join(args['cache_dir'], f"cached_{mode}_{args['model_type']}_{args['max_seq_length']}_binary")
if os.path.exists(cached_features_file) and not args['reprocess_input_data'] and not no_cache:
features = torch.load(cached_features_file)
else:
features = convert_examples_to_features(examples, args['max_seq_length'], tokenizer, output_mode,
# xlnet has a cls token at the end
cls_token_at_end=bool(args['model_type'] in ['xlnet']),
cls_token=tokenizer.cls_token,
cls_token_segment_id=2 if args['model_type'] in ['xlnet'] else 0,
sep_token=tokenizer.sep_token,
# roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
sep_token_extra=bool(args['model_type'] in ['roberta']),
# pad on the left for xlnet
pad_on_left=bool(args['model_type'] in ['xlnet']),
pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
pad_token_segment_id=4 if args['model_type'] in ['xlnet'] else 0,
process_count=process_count)
if not no_cache:
torch.save(features, cached_features_file)
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
if output_mode == "classification":
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
elif output_mode == "regression":
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)
dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
return dataset