def test_no_single_func(self): data = range(10) parallel_func = lambda x: x * 2 # noqa res = list(funnel(data, parallel_func, timeout=10)) self.assertEqual(len(res), 10)
def test_dict_values(self): from bobtools.parallel import funnel data = [{"id": 1}, {"id": 2}] def datagen(): for i in data: yield (i, ) def multi_func(thing): thing["id"] += 1 print(thing) return (thing, ) class Toucher: n_touched = 0 id_sum = 0 def touch(self, thing): self.n_touched += 1 self.id_sum += thing["id"] return self.id_sum toucher = Toucher() res = list(funnel(datagen(), multi_func, toucher.touch)) self.assertEqual(res[-1], sum([thing["id"] + 1 for thing in data]))
def test_assert_minimum_check(self): data = range(10) multi_func = lambda x: x # noqa single_func = lambda x: x # noqa self.assertRaises( ValueError, funnel(data, multi_func, single_func, n_workers=1).__next__)
def test_lambdas_to_generator(self): data = range(10) paralel_func = lambda x: x * 2 # noqa single_func = lambda x: x / 2 # noqa res = list(funnel(data, paralel_func, single_func)) self.assertEqual(len(res), len(range(10))) self.assertAlmostEqual(sum(res), sum(range(10)))
def test_named_arguments(self): def with_named(): for i in range(10): yield {"x": i, "y": 2, "z": 4} parallel_func = lambda x, y=1, z=3: {"x": x * z, "z": 3} # noqa single_func = lambda z=1, y=2, x=3: x / z # noqa res = list(funnel(with_named(), parallel_func, single_func)) expected = [i * 4 / 3 for i in range(10)] self.assertEqual(sum(res), sum(expected)) self.assertEqual(len(res), len(expected))
def test_positional_arguments(self): def with_positional(): for i in range(10): yield (i, 2) def parallel_func(x, y): """pass a positional argument (3) to the single func""" return x * y, 3 single_func = lambda x, y: x / y # noqa res = list(funnel(with_positional(), parallel_func, single_func)) expected = [(i * 2) / 3 for i in range(10)] self.assertAlmostEqual(sum(res), sum(expected)) self.assertEqual(len(res), len(expected))
def test_single_without_output(self): def gen(): for i in range(10): yield i g = gen() parallel_func = lambda x: x # noqa single_func = lambda x: None # noqa res = list( funnel(iterable=g, multi_func=parallel_func, single_func=single_func)) self.assertRaises(StopIteration, g.__next__) self.assertEqual(len(res), 0)