def get_data(
model_args,
training_args,
tokenizer,
text_data_path="../data/test_dataset"): # 경로 변경 ../data/test_dataset
"""
get data
Args:
model_args: model arguments
training_args: training arguments
tokenizer: tokenizer
text_data_path: Defaults to "../data/test_dataset"
Returns:
text_data, val_iter, val_dataset, scores
"""
text_data = load_from_disk(text_data_path)
# run_ lasticsearch
if "elastic" in model_args.retrieval_type:
is_sentence_trainformer = False
if "sentence_trainformer" in model_args.retrieval_type:
is_sentence_trainformer = True
# number of text to concat
concat_num = model_args.retrieval_elastic_num
text_data, scores = run_elasticsearch(text_data, concat_num,
model_args,
is_sentence_trainformer)
elif model_args.retrieval_type == "dense":
concat_num = model_args.retrieval_elastic_num
text_data, scores = run_concat_dense_retrival(text_data, concat_num)
column_names = text_data["validation"].column_names
data_collator = (DataCollatorWithPadding(
tokenizer, pad_to_multiple_of=8 if training_args.fp16 else None))
# 데이터 tokenize(mrc 모델안에 들어 갈 수 있도록)
data_processor = DataProcessor(tokenizer)
val_text = text_data["validation"]
val_dataset = data_processor.val_tokenzier(val_text, column_names)
val_iter = DataLoader(val_dataset, collate_fn=data_collator, batch_size=1)
return text_data, val_iter, val_dataset, scores
def get_data(data_args, training_args, tokenizer):
'''train과 validation의 dataloader와 dataset를 반환하는 함수'''
if data_args.dataset_name == 'basic':
if os.path.isdir("../data/train_dataset"):
dataset = load_from_disk("../data/train_dataset")
else:
raise Exception("Set the data path to 'p3-mrc-team-ikyo/data/.'")
elif data_args.dataset_name == 'preprocessed':
if os.path.isfile("../data/preprocess_train.pkl"):
dataset = get_pickle("../data/preprocess_train.pkl")
else:
dataset = make_custom_dataset("../data/preprocess_train.pkl")
elif data_args.dataset_name == 'concat':
if os.path.isfile("../data/concat_train.pkl"):
dataset = get_pickle("../data/concat_train.pkl")
else:
dataset = make_custom_dataset("../data/concat_train.pkl")
elif data_args.dataset_name == 'korquad':
if os.path.isfile("../data/korquad_train.pkl"):
dataset = get_pickle("../data/korquad_train.pkl")
else:
dataset = make_custom_dataset("../data/korquad_train.pkl")
elif data_args.dataset_name == "question_type":
if os.path.isfile("../data/question_type.pkl"):
dataset = get_pickle("../data/question_type.pkl")
else:
dataset = make_custom_dataset("../data/question_type.pkl")
elif data_args.dataset_name == "ai_hub":
if os.path.isfile("../data/ai_hub_dataset.pkl"):
dataset = get_pickle("../data/ai_hub_dataset.pkl")
else:
dataset = make_custom_dataset("../data/ai_hub_dataset.pkl")
elif data_args.dataset_name == "only_korquad":
dataset = load_dataset("squad_kor_v1")
elif data_args.dataset_name == "random_masking":
if os.path.isfile("../data/random_mask_train.pkl"):
dataset = get_pickle("../data/random_mask_train.pkl")
else:
dataset = make_custom_dataset("../data/random_mask_train.pkl")
elif data_args.dataset_name == "token_masking":
if os.path.isfile("../data/concat_token_mask_top_3.pkl"):
dataset = get_pickle("../data/concat_token_mask_top_3.pkl")
else:
dataset = make_mask_dataset("../data/concat_token_mask_top_3.pkl",
tokenizer)
train_dataset = dataset['train']
val_dataset = dataset['validation']
else:
raise Exception(
"dataset_name have to be one of ['basic', 'preprocessed', 'concat', 'korquad', 'only_korquad', 'question_type', 'ai_hub', 'random_masking', 'token_masking']"
)
if data_args.dataset_name != "token_masking":
train_dataset = dataset['train']
val_dataset = dataset['validation']
train_column_names = train_dataset.column_names
val_column_names = val_dataset.column_names
data_processor = DataProcessor(tokenizer, data_args.max_seq_length,
data_args.doc_stride)
train_dataset = data_processor.train_tokenizer(train_dataset,
train_column_names)
val_dataset = data_processor.val_tokenzier(val_dataset,
val_column_names)
data_collator = (DataCollatorWithPadding(
tokenizer, pad_to_multiple_of=8 if training_args.fp16 else None))
train_iter = DataLoader(
train_dataset,
collate_fn=data_collator,
batch_size=training_args.per_device_train_batch_size)
val_iter = DataLoader(val_dataset,
collate_fn=data_collator,
batch_size=training_args.per_device_eval_batch_size)
return dataset, train_iter, val_iter, train_dataset, val_dataset