def test_partition_functionality(): sz = 2 a = [1,2,3,4,5] ex = [[1,2],[3,4],[5]] result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a, b in zip(result, ex)]) sz = 3 ex = [[1,2,3],[4,5]] result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a,b in zip(result, ex)]) sz = 1 ex = [[1],[2],[3],[4],[5]] result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a,b in zip(result, ex)]) sz = 6 ex = [[1,2,3,4,5]] result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a,b in zip(result, ex)]) sz = 3 a = [] result = partition(a, sz) assert len(result) == 0
def test_partition_functionality():
def test_partition(a, sz, ex):
result = core.partition(a, sz)
assert len(result) == len(ex)
assert all([a == b for a, b in zip(result, ex)])
a = [1,2,3,4,5]
sz = 2
ex = [[1,2],[3,4],[5]]
test_partition(a, sz, ex)
sz = 3
ex = [[1,2,3],[4,5]]
test_partition(a, sz, ex)
sz = 1
ex = [[1],[2],[3],[4],[5]]
test_partition(a, sz, ex)
sz = 6
ex = [[1,2,3,4,5]]
test_partition(a, sz, ex)
sz = 3
a = []
result = core.partition(a, sz)
assert len(result) == 0
def download(index, path, chunk_size: int=1000, proxy: dict=None):
"""Downloads images with URLs from index dataframe."""
n_cpu = cpu_count()
worker = partial(download_single, path, proxy)
queue = index.to_dict('records')
meta = []
with Pool(n_cpu) as pool:
chunks = partition(queue, chunk_size)
n_chunks = len(chunks)
for i, chunk in enumerate(chunks):
log.info('Downloading chunk %d of %d' % (i+1, n_chunks))
data = [x for x in pool.imap_unordered(worker, chunk) if not x.failed]
meta.extend([asdict(info) for info in data])
return pd.DataFrame(meta)
def test_partition(a, sz, ex): result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a, b in zip(result, ex)])
def test_partition_error_handling():
sz = 0
a = [1,2,3,4,5]
with pytest.raises(ValueError):
partition(a, sz)
def test_partition(self):
result = partition([1, 2, 3, 4, 5], 2)
self.assertEqual(3, len(result))
def test_partition(self):
result = partition([1,2,3,4,5], 2)
self.assertEqual(3, len(result))
def test_partition_error_handling():
sz = 0
a = [1,2,3,4,5]
with pytest.raises(ValueError):
core.partition(a, sz)
def test_partition(a, sz, ex): result = core.partition(a, sz) assert len(result) == len(ex) assert all([a == b for a, b in zip(result, ex)])
