def test_partition_functionality(): sz = 2 a = [1,2,3,4,5] ex = [[1,2],[3,4],[5]] result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a, b in zip(result, ex)]) sz = 3 ex = [[1,2,3],[4,5]] result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a,b in zip(result, ex)]) sz = 1 ex = [[1],[2],[3],[4],[5]] result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a,b in zip(result, ex)]) sz = 6 ex = [[1,2,3,4,5]] result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a,b in zip(result, ex)]) sz = 3 a = [] result = partition(a, sz) assert len(result) == 0
def test_partition_functionality(): def test_partition(a, sz, ex): result = core.partition(a, sz) assert len(result) == len(ex) assert all([a == b for a, b in zip(result, ex)]) a = [1,2,3,4,5] sz = 2 ex = [[1,2],[3,4],[5]] test_partition(a, sz, ex) sz = 3 ex = [[1,2,3],[4,5]] test_partition(a, sz, ex) sz = 1 ex = [[1],[2],[3],[4],[5]] test_partition(a, sz, ex) sz = 6 ex = [[1,2,3,4,5]] test_partition(a, sz, ex) sz = 3 a = [] result = core.partition(a, sz) assert len(result) == 0
def download(index, path, chunk_size: int=1000, proxy: dict=None): """Downloads images with URLs from index dataframe.""" n_cpu = cpu_count() worker = partial(download_single, path, proxy) queue = index.to_dict('records') meta = [] with Pool(n_cpu) as pool: chunks = partition(queue, chunk_size) n_chunks = len(chunks) for i, chunk in enumerate(chunks): log.info('Downloading chunk %d of %d' % (i+1, n_chunks)) data = [x for x in pool.imap_unordered(worker, chunk) if not x.failed] meta.extend([asdict(info) for info in data]) return pd.DataFrame(meta)
def test_partition(a, sz, ex): result = partition(a, sz) assert len(result) == len(ex) assert all([a == b for a, b in zip(result, ex)])
def test_partition_error_handling(): sz = 0 a = [1,2,3,4,5] with pytest.raises(ValueError): partition(a, sz)
def test_partition(self): result = partition([1, 2, 3, 4, 5], 2) self.assertEqual(3, len(result))
def test_partition(self): result = partition([1,2,3,4,5], 2) self.assertEqual(3, len(result))
def test_partition_error_handling(): sz = 0 a = [1,2,3,4,5] with pytest.raises(ValueError): core.partition(a, sz)
def test_partition(a, sz, ex): result = core.partition(a, sz) assert len(result) == len(ex) assert all([a == b for a, b in zip(result, ex)])