def test_scan_no_dimensions():
dataset = Dataset('test-bucket', ['dim1', 'dim2'], prefix='prefix/')
with futures.ProcessPoolExecutor(1) as executor:
folders = dataset._scan([], [
'prefix/',
], {}, executor)
assert folders == [
'prefix/',
]
def test_scan_with_prefix():
bucket_name = 'test-bucket'
store = InMemoryStore(bucket_name)
store.store['prefix1/dir1/subdir1/key1'] = 'value1'
store.store['prefix2/dir2/another-dir/key2'] = 'value2'
dataset = Dataset(bucket_name, ['dim1', 'dim2'],
clauses={'dim1': lambda x: x == 'dir1'}, store=store)
with futures.ProcessPoolExecutor(1) as executor:
folders = dataset._scan(['dim1', 'dim2',], ['prefix1/',], dataset.clauses, executor)
assert list(folders) == ['prefix1/dir1/']
def test_scan_no_clause():
bucket_name = 'test-bucket'
store = InMemoryStore(bucket_name)
key = 'dir1/dir2/key1'
value = 'value1'
store.store[key] = value
dataset = Dataset(bucket_name, ['dim1', 'dim2'], store=store)
with futures.ProcessPoolExecutor(1) as executor:
folders = dataset._scan(['dim1', 'subdir'], ['prefix'], {}, executor)
assert list(folders) == ['prefix']
def test_scan_with_clause():
bucket_name = 'test-bucket'
store = InMemoryStore(bucket_name)
store.store['dir1/subdir1/key1'] = 'value1'
store.store['dir2/another-dir/key2'] = 'value2'
dataset = Dataset(bucket_name, ['dim1', 'dim2'],
clauses={'dim1': lambda x: x == 'dir1'}, store=store)
with futures.ProcessPoolExecutor(1) as executor:
folders = dataset._scan(['dim1', 'dim2'], [''], dataset.clauses, executor)
assert list(folders) == ['dir1/']
def test_scan_no_clause():
bucket_name = 'test-bucket'
store = InMemoryStore(bucket_name)
key = 'dir1/dir2/key1'
value = 'value1'
store.store[key] = value
dataset = Dataset(bucket_name, ['dim1', 'dim2'], store=store)
with futures.ProcessPoolExecutor(1) as executor:
folders = dataset._scan(['dim1', 'subdir'], ['prefix'], {}, executor)
assert list(folders) == ['prefix']
def test_scan_no_dimensions():
dataset = Dataset('test-bucket', ['dim1', 'dim2'], prefix='prefix/')
with futures.ProcessPoolExecutor(1) as executor:
folders = dataset._scan([], ['prefix/', ], {}, executor)
assert folders == ['prefix/', ]
