def test_base_path(self, paths, base_path):
filepaths = _file_utils.flatten_file_trees(paths)
expected_paths = set(
os.path.relpath(path, base_path) for path in filepaths)
dataset = verta.dataset.Path(paths, base_path)
assert set(dataset.list_paths()) == expected_paths
def test_list_paths(self):
data_dir = "modelapi_hypothesis/"
expected_paths = set(_file_utils.walk_files(data_dir))
dataset = verta.dataset.Path(data_dir)
assert set(dataset.list_paths()) == expected_paths
def test_list_paths(self):
data_dir = "modelapi_hypothesis/"
expected_paths = []
for root, _, filenames in os.walk(data_dir):
for filename in filenames:
expected_paths.append(os.path.join(root, filename))
dataset = verta.dataset.Path(data_dir)
assert dataset.list_paths() == expected_paths
def test_list_paths(self):
s3 = pytest.importorskip("boto3").client('s3')
bucket = "verta-starter"
expected_paths = set(
"s3://{}/{}".format(bucket, s3_obj['Key'])
for s3_obj in s3.list_objects_v2(Bucket=bucket)['Contents']
if not s3_obj['Key'].endswith('/') # folder, not object
)
dataset = verta.dataset.S3("s3://{}".format(bucket))
assert set(dataset.list_paths()) == expected_paths
