def dataset_num_files_1(tmpdir_factory):
path = tmpdir_factory.mktemp("data").strpath
url = 'file://' + path
data = create_test_dataset(url, range(99), num_files=1)
dataset = SyntheticDataset(url=url, path=path, data=data)
return dataset
def dataset_0_3_8_10_11_20_23(tmpdir_factory):
path = tmpdir_factory.mktemp("data").strpath
url = 'file://' + path
ids = [0, 3, 8, 10, 11, 20, 23]
data = create_test_dataset(url, ids, num_files=1)
dataset = SyntheticDataset(url=url, path=path, data=data)
return dataset
def hello_world_dataset(tmpdir_factory):
path = tmpdir_factory.mktemp("data").strpath
url = 'file://' + path
generate_hello_world_dataset(url)
dataset = SyntheticDataset(url=url, path=path, data=None)
# Generate a dataset
assert os.path.exists(os.path.join(path, '_SUCCESS'))
return dataset
def pycarbon_dataset(tmpdir_factory):
path = tmpdir_factory.mktemp("data").strpath
url = LOCAL_FILE_PREFIX + path
generate_pycarbon_dataset(url)
dataset = SyntheticDataset(url=url, path=path, data=None)
# Generate a dataset
assert os.path.exists(os.path.join(path, '_SUCCESS'))
return dataset
def _dataset_generator():
path = tmpdir_factory.mktemp("data").strpath
url = 'file://' + path
ids = range(0, 99, 5)
data = create_test_dataset(url, ids)
return SyntheticDataset(url=url, path=path, data=data)
def _dataset_generator():
path = tmpdir_factory.mktemp("data").strpath
url = 'file://' + path
ids = [0, 3, 8, 10, 11, 20, 23]
data = create_test_dataset(url, ids, num_files=1)
return SyntheticDataset(url=url, path=path, data=data)
def _dataset_generator():
path = tmpdir_factory.mktemp("data").strpath
url = 'file://' + path
data = create_test_dataset(url, range(99), num_files=1)
return SyntheticDataset(url=url, path=path, data=data)