def load_examples(tokenizer, ngram_dict, label_list, text):
examples = combine_example(text)
features = convert_examples_to_features(examples, label_list, 128, tokenizer, ngram_dict)
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
all_valid_ids = torch.tensor([f.valid_ids for f in features], dtype=torch.long)
all_lmask_ids = torch.tensor([f.label_mask for f in features], dtype=torch.long)
all_ngram_ids = torch.tensor([f.ngram_ids for f in features], dtype=torch.long)
all_ngram_positions = torch.tensor([f.ngram_positions for f in features], dtype=torch.long)
all_ngram_lengths = torch.tensor([f.ngram_lengths for f in features], dtype=torch.long)
all_ngram_seg_ids = torch.tensor([f.ngram_seg_ids for f in features], dtype=torch.long)
all_ngram_masks = torch.tensor([f.ngram_masks for f in features], dtype=torch.long)
return TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids, all_ngram_ids,
all_ngram_positions,
all_ngram_lengths, all_ngram_seg_ids, all_ngram_masks, all_valid_ids, all_lmask_ids)
def load_examples(args, tokenizer, ngram_dict, processor, label_list, mode):
if mode == "train":
examples = processor.get_train_examples(args.data_dir)
elif mode == "test":
examples = processor.get_test_examples(args.data_dir)
features = convert_examples_to_features(examples, label_list, args.max_seq_length, tokenizer, ngram_dict)
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
all_valid_ids = torch.tensor([f.valid_ids for f in features], dtype=torch.long)
all_lmask_ids = torch.tensor([f.label_mask for f in features], dtype=torch.long)
all_ngram_ids = torch.tensor([f.ngram_ids for f in features], dtype=torch.long)
all_ngram_positions = torch.tensor([f.ngram_positions for f in features], dtype=torch.long)
all_ngram_lengths = torch.tensor([f.ngram_lengths for f in features], dtype=torch.long)
all_ngram_seg_ids = torch.tensor([f.ngram_seg_ids for f in features], dtype=torch.long)
all_ngram_masks = torch.tensor([f.ngram_masks for f in features], dtype=torch.long)
logger.info('Dataset is ready.')
return TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids, all_ngram_ids,all_ngram_positions,
all_ngram_lengths, all_ngram_seg_ids, all_ngram_masks, all_valid_ids, all_lmask_ids)