def _get_testbench(self, task=None):
# TODO Have a proper mock testbench
# Create test bench
testbench_identifier = "test-testbench"
testbench = TestBench(
identifier=testbench_identifier,
task=task,
slices=[
Slice(
dataset=Dataset.load_dataset("snli", split="train[:128]"),
identifier="snli_1",
).filter(lambda example: example["label"] != -1),
Slice(
dataset=Dataset.load_dataset("snli",
split="validation[:128]"),
identifier="snli_2",
).filter(lambda example: example["label"] != -1),
Slice(
dataset=Dataset.load_dataset("snli", split="test[:128]"),
identifier="snli_3",
).filter(lambda example: example["label"] != -1),
],
dataset_id="snli",
)
return testbench
def get_hf_dataset(name: str, version: str = None, split: str = 'test'):
"""Get dataset from Huggingface."""
if version:
return Dataset.load_dataset(name, version, split=split)
return Dataset.load_dataset(name, split=split)