def test_base_path(self, paths, base_path): filepaths = _file_utils.flatten_file_trees(paths) expected_paths = set( os.path.relpath(path, base_path) for path in filepaths) dataset = verta.dataset.Path(paths, base_path) assert set(dataset.list_paths()) == expected_paths
def test_list_paths(self): data_dir = "modelapi_hypothesis/" expected_paths = set(_file_utils.walk_files(data_dir)) dataset = verta.dataset.Path(data_dir) assert set(dataset.list_paths()) == expected_paths
def test_list_paths(self): data_dir = "modelapi_hypothesis/" expected_paths = [] for root, _, filenames in os.walk(data_dir): for filename in filenames: expected_paths.append(os.path.join(root, filename)) dataset = verta.dataset.Path(data_dir) assert dataset.list_paths() == expected_paths
def test_list_paths(self): s3 = pytest.importorskip("boto3").client('s3') bucket = "verta-starter" expected_paths = set( "s3://{}/{}".format(bucket, s3_obj['Key']) for s3_obj in s3.list_objects_v2(Bucket=bucket)['Contents'] if not s3_obj['Key'].endswith('/') # folder, not object ) dataset = verta.dataset.S3("s3://{}".format(bucket)) assert set(dataset.list_paths()) == expected_paths