def test_producer_map_consume_with_3_process_plus_1_brach(self):
data = [1, 2, 3, 4, 5]
workflow = Iterable(data)
sync_brach = workflow | StoreAndPickle()
async_branch = workflow | SpawnThread() | Map(add100) | SpawnThread() | StoreAndPickle()
workflow.run()
self.assertEqual(sync_brach.load(), data)
self.assertEqual(async_branch.load(), [d+100 for d in data])
def test_producer_consume(self):
data = [1, 2, 3, 4, 5]
workflow = Iterable(data) | SpawnThread() | StoreAndPickle()
workflow.run()
#workflow ref to StoreAndPickle() instace that is the only leaf of the DAG
actual = workflow.load()
self.assertEqual(actual, data)
def test_producer_map_consume_with_3_process(self):
data = [1, 2, 3, 4, 5]
workflow = Iterable(data) | SpawnThread() | Map(add100) | SpawnThread() | StoreAndPickle()
workflow.run()
#workflow ref to StoreAndPickle() instace that is the only leaf of the DAG
actual = workflow.load()
self.assertEqual(actual, [d+100 for d in data])
def test_lambda_with_func_import(self):
data = [1, 2, 3, 4, 5]
workflow = Iterable(data) | SpawnProcess() | Map(lambda x: add100(x)) | StoreAndPickle()
workflow.run()
#workflow ref to StoreAndPickle() instace that is the only leaf of the DAG
actual = workflow.load()
self.assertEqual(actual, [d+100 for d in data])
def test_producer_consume_2_processes_inline(self):
data = [1, 2, 3, 4, 5]
workflow = Iterable(data) | Parallelize(two_split) | (
Map(add100) | Map(add100)) | Join() | StoreAndPickle()
workflow.run()
#workflow ref to StoreAndPickle() instace that is the only leaf of the DAG
actual = workflow.load()
#need to sort result because with symmetric parallelims order is not guaranteed
self.assertEqual(sorted(actual), [d + 200 for d in data])
def test_producer_consume_10_processes(self):
data = range(20)
parallel = Map(add100)
workflow = Iterable(data) | Parallelize(
ten_split) | parallel | Join() | StoreAndPickle()
workflow.run()
#workflow ref to StoreAndPickle() instace that is the only leaf of the DAG
actual = workflow.load()
#need to sort result because with symmetric parallelims order is not guaranteed
self.assertEqual(sorted(actual), [d + 100 for d in data])
def test_producer_map_consume_with_2_process(self):
data = [1, 2, 3, 4, 5]
#CAUTION!!!!
#Cannot use lambda (e.g. Map(lambda x: x+100)) yet due to pickle problem in multiprocessing lib
#possible solution is to hook the import of pickle im multiprocessing lib
#and substitute with dill.
#See: #http://chimera.labs.oreilly.com/books/1230000000393/ch10.html#_solution_180
workflow = Iterable(data) | SpawnProcess() | Map(lambda x: x+100) | StoreAndPickle()
workflow.run()
#workflow ref to StoreAndPickle() instace that is the only leaf of the DAG
actual = workflow.load()
self.assertEqual(actual, [d+100 for d in data])