def test_scan_no_dimensions(): dataset = Dataset('test-bucket', ['dim1', 'dim2'], prefix='prefix/') with futures.ProcessPoolExecutor(1) as executor: folders = dataset._scan([], [ 'prefix/', ], {}, executor) assert folders == [ 'prefix/', ]
def test_scan_with_prefix(): bucket_name = 'test-bucket' store = InMemoryStore(bucket_name) store.store['prefix1/dir1/subdir1/key1'] = 'value1' store.store['prefix2/dir2/another-dir/key2'] = 'value2' dataset = Dataset(bucket_name, ['dim1', 'dim2'], clauses={'dim1': lambda x: x == 'dir1'}, store=store) with futures.ProcessPoolExecutor(1) as executor: folders = dataset._scan(['dim1', 'dim2',], ['prefix1/',], dataset.clauses, executor) assert list(folders) == ['prefix1/dir1/']
def test_scan_no_clause(): bucket_name = 'test-bucket' store = InMemoryStore(bucket_name) key = 'dir1/dir2/key1' value = 'value1' store.store[key] = value dataset = Dataset(bucket_name, ['dim1', 'dim2'], store=store) with futures.ProcessPoolExecutor(1) as executor: folders = dataset._scan(['dim1', 'subdir'], ['prefix'], {}, executor) assert list(folders) == ['prefix']
def test_scan_with_clause(): bucket_name = 'test-bucket' store = InMemoryStore(bucket_name) store.store['dir1/subdir1/key1'] = 'value1' store.store['dir2/another-dir/key2'] = 'value2' dataset = Dataset(bucket_name, ['dim1', 'dim2'], clauses={'dim1': lambda x: x == 'dir1'}, store=store) with futures.ProcessPoolExecutor(1) as executor: folders = dataset._scan(['dim1', 'dim2'], [''], dataset.clauses, executor) assert list(folders) == ['dir1/']
def test_scan_no_dimensions(): dataset = Dataset('test-bucket', ['dim1', 'dim2'], prefix='prefix/') with futures.ProcessPoolExecutor(1) as executor: folders = dataset._scan([], ['prefix/', ], {}, executor) assert folders == ['prefix/', ]