def test_nested_task(self): pkg=PackageSource('testpkg', 'pkgfile', 'NONE') exec_task = ExecTask("exec_name", pkg, ('a',), ('b',)) subgraph = G( C(START_TICK, 'a', 1,'a'), C(START_TICK, 'context', 1,'context'), T(1, exec_task, {'name': 'test_exec', 'path':'test_exec'}), C(1, 'b', FINAL_TICK,'b'), ) subgraph_task_tick=subgraph.get_all_ticks()[0] graph=Graph() task=NestedGraphTask(subgraph, 'test_nested') tick=START_TICK+1 graph.add_task(tick, task, {'name':'test_nested','path':'test_nested'}) source=Endpoint(START_TICK, 'a') dest=Endpoint(tick, 'a') graph.connect(source,dest) source=Endpoint(START_TICK, 'context') dest=Endpoint(tick, 'context') graph.connect(source,dest) source=Endpoint(tick, 'b') dest=Endpoint(FINAL_TICK, 'b') graph.connect(source,dest) task.refine(graph, Tick.parse_tick(1), {'context':{},'a':{}}) expected_graph_task_tick=subgraph_task_tick<<Tick.parse_tick(1) expected = G( C(START_TICK, 'a', expected_graph_task_tick,'a'), C(START_TICK, 'context', expected_graph_task_tick,'context'), T(expected_graph_task_tick, exec_task, {'name': 'test_exec', 'path':'test_nested.test_exec'}), C(expected_graph_task_tick, 'b', FINAL_TICK,'b'), ) utils.assert_graph_equal(expected, graph)
def test_nested_task(self): pkg = PackageSource('testpkg', 'pkgfile', 'NONE') exec_task = ExecTask("exec_name", pkg, ('a', ), ('b', )) subgraph = G( C(START_TICK, 'a', 1, 'a'), C(START_TICK, 'context', 1, 'context'), T(1, exec_task, { 'name': 'test_exec', 'path': 'test_exec' }), C(1, 'b', FINAL_TICK, 'b'), ) subgraph_task_tick = subgraph.get_all_ticks()[0] graph = Graph() task = NestedGraphTask(subgraph, 'test_nested') tick = START_TICK + 1 graph.add_task(tick, task, { 'name': 'test_nested', 'path': 'test_nested' }) source = Endpoint(START_TICK, 'a') dest = Endpoint(tick, 'a') graph.connect(source, dest) source = Endpoint(START_TICK, 'context') dest = Endpoint(tick, 'context') graph.connect(source, dest) source = Endpoint(tick, 'b') dest = Endpoint(FINAL_TICK, 'b') graph.connect(source, dest) task.refine(graph, Tick.parse_tick(1), {'context': {}, 'a': {}}) expected_graph_task_tick = subgraph_task_tick << Tick.parse_tick(1) expected = G( C(START_TICK, 'a', expected_graph_task_tick, 'a'), C(START_TICK, 'context', expected_graph_task_tick, 'context'), T(expected_graph_task_tick, exec_task, { 'name': 'test_exec', 'path': 'test_nested.test_exec' }), C(expected_graph_task_tick, 'b', FINAL_TICK, 'b'), ) utils.assert_graph_equal(expected, graph)
def test_parallel_split(self): pkg = PackageSource('testpkg', 'pkgfile', 'NONE') exec_task = ExecTask("exec_name", pkg, ('a', 'b'), ('c', 'd')) subgraph = G( C(START_TICK, 'a', 1, 'a'), C(START_TICK, 'b', 1, 'b'), C(START_TICK, 'context', 1, 'context'), T(1, exec_task, { 'name': 'test_exec', 'path': 'test_exec' }), C(1, 'c', FINAL_TICK, 'c'), C(1, 'd', FINAL_TICK, 'd'), ) subgraph_task_tick = subgraph.get_all_ticks()[0] graph = Graph() task = ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps') tick = START_TICK + 1 graph.add_task(tick, task, {'name': 'ps', 'path': 'ps'}) source = Endpoint(START_TICK, 'a') dest = Endpoint(tick, 'a') graph.connect(source, dest) source = Endpoint(START_TICK, 'b') dest = Endpoint(tick, 'b') graph.connect(source, dest) source = Endpoint(START_TICK, 'context') dest = Endpoint(tick, 'context') graph.connect(source, dest) source = Endpoint(tick, 'cd_list') dest = Endpoint(FINAL_TICK, 'cd_list') graph.connect(source, dest) testdir = tempfile.mkdtemp() workdir = os.path.join(testdir, 'workdir') os.mkdir(workdir) testfile = os.path.join(workdir, 'testfile.pickle') local_workdir = os.path.join(testdir, 'local_workdir') os.mkdir(local_workdir) with open(testfile, 'wb') as f: pickle.dump(['x', 'y', 'z'], f) context = { 'transporter': LocalFileTransporter(), 'workdir': workdir, 'local_workdir': local_workdir } task.refine(graph, Tick.parse_tick(1), { 'b': 'testfile.pickle', 'context': context }) mapTask = _mapPortsTask('b', ['x', 'y', 'z']) map_task_tick = START_TICK + 1 << tick iterations_tick = map_task_tick + 1 it_1 = START_TICK + 1 << iterations_tick it_2 = START_TICK + 2 << iterations_tick it_3 = START_TICK + 3 << iterations_tick task_tick_1 = subgraph_task_tick << it_1 task_tick_2 = subgraph_task_tick << it_2 task_tick_3 = subgraph_task_tick << it_3 reduceTask = _reducePortsTask(3, ['c', 'd'], 'cd_list') reduce_tick = iterations_tick + 1 expected = G( C(START_TICK, 'context', map_task_tick, 'context'), C(START_TICK, 'a', task_tick_1, 'a'), C(START_TICK, 'a', task_tick_2, 'a'), C(START_TICK, 'a', task_tick_3, 'a'), C(START_TICK, 'b', map_task_tick, 'inputlist'), C(START_TICK, 'context', task_tick_1, 'context'), C(START_TICK, 'context', task_tick_2, 'context'), C(START_TICK, 'context', task_tick_3, 'context'), T(map_task_tick, mapTask, { 'name': 'map', 'path': 'ps.map' }), C(map_task_tick, 'b_1', task_tick_1, 'b'), C(map_task_tick, 'b_2', task_tick_2, 'b'), C(map_task_tick, 'b_3', task_tick_3, 'b'), T(task_tick_1, exec_task, { 'name': 'test_exec', 'path': 'ps.iterations.1.test_exec' }), T(task_tick_2, exec_task, { 'name': 'test_exec', 'path': 'ps.iterations.2.test_exec' }), T(task_tick_3, exec_task, { 'name': 'test_exec', 'path': 'ps.iterations.3.test_exec' }), C(task_tick_1, 'c', reduce_tick, 'c_1'), C(task_tick_2, 'c', reduce_tick, 'c_2'), C(task_tick_3, 'c', reduce_tick, 'c_3'), C(task_tick_1, 'd', reduce_tick, 'd_1'), C(task_tick_2, 'd', reduce_tick, 'd_2'), C(task_tick_3, 'd', reduce_tick, 'd_3'), C(START_TICK, 'context', reduce_tick, 'context'), T(reduce_tick, reduceTask, { 'name': 'reduce', 'path': 'ps.reduce' }), C(reduce_tick, 'cd_list', FINAL_TICK, 'cd_list'), ) utils.assert_graph_equal(expected, graph) self.assertTrue( os.path.exists(os.path.join(local_workdir, 'testfile.pickle')))
def test_parallel_split(self): pkg=PackageSource('testpkg', 'pkgfile', 'NONE') exec_task = ExecTask("exec_name", pkg, ('a','b'), ('c','d')) subgraph = G( C(START_TICK, 'a', 1,'a'), C(START_TICK, 'b', 1,'b'), C(START_TICK, 'context', 1,'context'), T(1, exec_task, {'name': 'test_exec', 'path':'test_exec'}), C(1, 'c', FINAL_TICK,'c'), C(1, 'd', FINAL_TICK,'d'), ) subgraph_task_tick=subgraph.get_all_ticks()[0] graph=Graph() task=ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps') tick=START_TICK+1 graph.add_task(tick, task, {'name':'ps','path':'ps'}) source=Endpoint(START_TICK, 'a') dest=Endpoint(tick, 'a') graph.connect(source,dest) source=Endpoint(START_TICK, 'b') dest=Endpoint(tick, 'b') graph.connect(source,dest) source=Endpoint(START_TICK, 'context') dest=Endpoint(tick, 'context') graph.connect(source,dest) source=Endpoint(tick, 'cd_list') dest=Endpoint(FINAL_TICK, 'cd_list') graph.connect(source,dest) testdir=tempfile.mkdtemp() workdir=os.path.join(testdir,'workdir') os.mkdir(workdir) testfile=os.path.join(workdir, 'testfile.pickle') local_workdir=os.path.join(testdir,'local_workdir') os.mkdir(local_workdir) with open(testfile,'wb') as f: pickle.dump(['x','y','z'],f) context={'transporter': LocalFileTransporter(), 'workdir':workdir, 'local_workdir':local_workdir} task.refine(graph, Tick.parse_tick(1), {'b':'testfile.pickle', 'context':context}) mapTask=_mapPortsTask('b', ['x','y','z']) map_task_tick=START_TICK+1<<tick iterations_tick=map_task_tick+1 it_1 = START_TICK + 1 << iterations_tick it_2 = START_TICK + 2 << iterations_tick it_3 = START_TICK + 3 << iterations_tick task_tick_1=subgraph_task_tick<<it_1 task_tick_2=subgraph_task_tick<<it_2 task_tick_3=subgraph_task_tick<<it_3 reduceTask=_reducePortsTask(3, ['c','d'], 'cd_list') reduce_tick=iterations_tick+1 expected = G( C(START_TICK, 'context', map_task_tick,'context'), C(START_TICK, 'a', task_tick_1,'a'), C(START_TICK, 'a', task_tick_2,'a'), C(START_TICK, 'a', task_tick_3,'a'), C(START_TICK, 'b', map_task_tick,'inputlist'), C(START_TICK, 'context', task_tick_1,'context'), C(START_TICK, 'context', task_tick_2,'context'), C(START_TICK, 'context', task_tick_3,'context'), T(map_task_tick, mapTask, {'name': 'map', 'path':'ps.map'}), C(map_task_tick, 'b_1', task_tick_1,'b'), C(map_task_tick, 'b_2', task_tick_2,'b'), C(map_task_tick, 'b_3', task_tick_3,'b'), T(task_tick_1, exec_task, {'name': 'test_exec', 'path':'ps.iterations.1.test_exec'}), T(task_tick_2, exec_task, {'name': 'test_exec', 'path':'ps.iterations.2.test_exec'}), T(task_tick_3, exec_task, {'name': 'test_exec', 'path':'ps.iterations.3.test_exec'}), C(task_tick_1, 'c', reduce_tick,'c_1'), C(task_tick_2, 'c', reduce_tick,'c_2'), C(task_tick_3, 'c', reduce_tick,'c_3'), C(task_tick_1, 'd', reduce_tick,'d_1'), C(task_tick_2, 'd', reduce_tick,'d_2'), C(task_tick_3, 'd', reduce_tick,'d_3'), C(START_TICK, 'context', reduce_tick,'context'), T(reduce_tick, reduceTask, {'name': 'reduce', 'path':'ps.reduce'}), C(reduce_tick, 'cd_list', FINAL_TICK,'cd_list'), ) utils.assert_graph_equal(expected, graph) self.assertTrue(os.path.exists(os.path.join(local_workdir,'testfile.pickle')))