def build_nondivisible_pipeline(self): self.nondivisible = Pipeline("nondivisible") self.nondivisible.add_origin(nondivisible_pipeline_start, num=1) self.nondivisible.add_task(nondivisible_pipeline_task, num=2) self.nondivisible.add_destination(nondivisible_pipeline_destination, num=2)
def build_first_pipeline(self): self.first = Pipeline("first one") self.first.add_origin(first_pipeline_origin) self.first.add_destination(first_pipeline_destination)
def build_divisible_pipeline(self): self.divisible = Pipeline("divisible") self.divisible.add_origin(divisible_pipeline_start) self.divisible.add_destination(divisible_pipeline_dest)
class TestPipeline(unittest.TestCase): def build_first_pipeline(self): self.first = Pipeline("first one") self.first.add_origin(first_pipeline_origin) self.first.add_destination(first_pipeline_destination) def test_first_pipeline(self): self.build_first_pipeline() # ensure the pipeline is started. # trick the pipeline into sending all results to a single queue by registering it manually q = multiprocessing.Queue() Pipeline._messaging.register_pipeline_queue("divisible", q) Pipeline._messaging.register_pipeline_queue("nondivisible", q) self.first.start() self.first.feed(6) # We should get back the set (0, 1, 2, 3, 4, 5) self.first.join() self.assertEqual(set(q.get()[0] for x in range(6)), set(x for x in range(6))) q.close() def build_divisible_pipeline(self): self.divisible = Pipeline("divisible") self.divisible.add_origin(divisible_pipeline_start) self.divisible.add_destination(divisible_pipeline_dest) def test_divisible_pipeline(self): q = multiprocessing.Queue() self.build_divisible_pipeline() Pipeline._messaging.register_pipeline_queue("final", q) self.divisible.start() self.divisible.feed(6) self.divisible.join() # We should get two numbers, 0 * 7 and 1 * 7 self.assertEqual(set(q.get()[0] for x in range(2)), set([0, 7])) q.close() def build_nondivisible_pipeline(self): self.nondivisible = Pipeline("nondivisible") self.nondivisible.add_origin(nondivisible_pipeline_start, num=1) self.nondivisible.add_task(nondivisible_pipeline_task, num=2) self.nondivisible.add_destination(nondivisible_pipeline_destination, num=2) def test_nondivisible_pipeline(self): q = multiprocessing.Queue() self.build_nondivisible_pipeline() Pipeline._messaging.register_pipeline_queue("final", q) self.nondivisible.start() self.nondivisible.feed(6) self.nondivisible.join() # We should get six numbers, 0, 1, 2, 3, 4, 5, each / 2 and * 3 self.assertEqual(set(q.get()[0] for x in range(6)), set(x / 2 * 3 for x in range(6))) q.close() def build_final_pipeline(self): self.final = Pipeline("final") self.final.add_origin(final_pipeline_start) self.final.add_destination(final_pipeline_reduce, num=1) def test_final_pipeline(self): self.build_final_pipeline() self.final.start() for i in range(17): for j in range(20): self.final.feed(j) # We will get a single number; as to its value, it will depend on the exact way the calls get distributed, so # it's impossible to know. It'll be greater than 17*10, at least. self.final.join() value = [x for x in self.final.as_completed()] self.assertGreaterEqual(value[0], 170) def test_assemble_contraption(self): self.build_first_pipeline() self.build_divisible_pipeline() self.build_nondivisible_pipeline() self.build_final_pipeline() self.first.start() self.divisible.start() self.nondivisible.start() self.final.start() self.first.feed(100) self.first.join() self.divisible.join() self.nondivisible.join() self.final.join() result = [x for x in self.final.as_completed()] self.assertGreater(len(result), 0) def test_implicit_as_completed(self): self.build_first_pipeline() self.build_divisible_pipeline() self.build_nondivisible_pipeline() self.build_final_pipeline() self.first.start() self.divisible.start() self.nondivisible.start() self.final.start() self.first.feed(100) self.first.join() self.divisible.join() self.nondivisible.join() result = [x for x in self.final.as_completed()] self.assertGreater(len(result), 0)
def nondivisible_pipeline_destination(number): Pipeline.send("final", number * 3)
def divisible_pipeline_dest(number): Pipeline.send("final", number * 7)
def first_pipeline_destination(number): if number % 3 == 0: Pipeline.send("divisible", number) else: Pipeline.send("nondivisible", number)
def build_final_pipeline(self): self.final = Pipeline("final") self.final.add_origin(final_pipeline_start) self.final.add_destination(final_pipeline_reduce, num=1)