def load_train_val_examples(args):
all_train_examples = load_glue_examples(train_data_generator,
args.train_file)
# 切分训练集和验证集
# theta 提供split_train_eval_examples辅助函数
from theta.utils import split_train_eval_examples
train_examples, val_examples = split_train_eval_examples(
all_train_examples,
train_rate=args.train_rate,
fold=args.fold,
shuffle=True)
logger.info(f"Loaded {len(train_examples)} train examples, "
f"{len(val_examples)} val examples.")
return train_examples, val_examples
def load_test_examples(args):
test_base_examples = load_glue_examples(test_data_generator,
args.test_file)
logger.info(f"Loaded {len(test_base_examples)} test examples.")
return test_base_examples
def load_eval_examples(eval_file):
eval_examples = load_glue_examples(eval_data_generator, eval_file)
logger.info(f"Loaded {len(eval_examples)} eval examples.")
return eval_examples
def load_train_examples(train_file):
train_examples = load_glue_examples(train_data_generator, train_file)
logger.info(f"Loaded {len(train_examples)} train examples.")
return train_examples
