def get_squad_data_loader(tokenizer, file, shuffle, args):
examples = read_squad_examples(file, is_training=True, debug=args.debug)
features = convert_examples_to_features_answer_id(
examples,
tokenizer=tokenizer,
max_seq_length=args.max_c_len,
max_query_length=args.max_q_len,
max_ans_length=args.max_q_len,
doc_stride=128,
is_training=True)
all_c_ids = torch.tensor([f.c_ids for f in features], dtype=torch.long)
all_q_ids = torch.tensor([f.q_ids for f in features], dtype=torch.long)
all_tag_ids = torch.tensor([f.tag_ids for f in features], dtype=torch.long)
all_a_ids = (all_tag_ids != 0).long()
all_start_positions = torch.tensor(
[f.noq_start_position for f in features], dtype=torch.long)
all_end_positions = torch.tensor([f.noq_end_position for f in features],
dtype=torch.long)
all_data = TensorDataset(all_c_ids, all_q_ids, all_a_ids,
all_start_positions, all_end_positions)
data_loader = DataLoader(all_data, args.batch_size, shuffle=shuffle)
return data_loader, examples, features
def get_harv_data_loader(tokenizer, file, shuffle, args):
examples = read_squad_examples(file, is_training=True, debug=args.debug)
num_examples = len(examples)
random.shuffle(examples)
examples = examples[:5000]
features = convert_examples_to_features_answer_id(examples,
tokenizer=tokenizer,
max_seq_length=args.max_c_len,
max_query_length=args.max_q_len,
doc_stride=128,
is_training=True)
all_c_ids = torch.tensor([f.c_ids for f in features], dtype=torch.long)
all_data = TensorDataset(all_c_ids)
data_loader = DataLoader(all_data, args.batch_size, shuffle=shuffle)
return data_loader
def get_ext_data_loader(tokenizer, train_folder, shuffle, args):
train_files = [f for f in os.listdir(train_folder) if f.endswith(".gz")]
examples_lst = []
features_lst = []
file_list = args.files.split("_")
for file in train_files:
file_name = file.split(".")[0].lower()
if file_name not in file_list:
print("skip {}".format(file_name))
continue
print("preprocessing {}".format(file_name))
file_path = os.path.join(train_folder, file)
examples = read_examples(file_path, debug=args.debug)
examples_lst.extend(examples)
features = convert_examples_to_features_answer_id(
examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_len,
max_query_length=args.max_query_len,
doc_stride=args.doc_stride,
is_training=True)
features_lst.extend(features)
all_c_ids = torch.tensor([f.c_ids for f in features_lst], dtype=torch.long)
all_q_ids = torch.tensor([f.q_ids for f in features_lst], dtype=torch.long)
all_noq_start_positions = torch.tensor(
[f.noq_start_position for f in features_lst], dtype=torch.long)
all_noq_end_positions = torch.tensor(
[f.noq_end_position for f in features_lst], dtype=torch.long)
train_data = TensorDataset(all_c_ids, all_q_ids, all_noq_start_positions,
all_noq_end_positions)
data_loader = DataLoader(train_data,
shuffle=shuffle,
batch_size=args.batch_size)
return data_loader