def test_nested_task(self):
pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
exec_task = ExecTask("exec_name", pkg, ('a', ), ('b', ))
subgraph = G(
C(START_TICK, 'a', 1, 'a'),
C(START_TICK, 'context', 1, 'context'),
T(1, exec_task, {
'name': 'test_exec',
'path': 'test_exec'
}),
C(1, 'b', FINAL_TICK, 'b'),
)
subgraph_task_tick = subgraph.get_all_ticks()[0]
graph = Graph()
task = NestedGraphTask(subgraph, 'test_nested')
tick = START_TICK + 1
graph.add_task(tick, task, {
'name': 'test_nested',
'path': 'test_nested'
})
source = Endpoint(START_TICK, 'a')
dest = Endpoint(tick, 'a')
graph.connect(source, dest)
source = Endpoint(START_TICK, 'context')
dest = Endpoint(tick, 'context')
graph.connect(source, dest)
source = Endpoint(tick, 'b')
dest = Endpoint(FINAL_TICK, 'b')
graph.connect(source, dest)
task.refine(graph, Tick.parse_tick(1), {'context': {}, 'a': {}})
expected_graph_task_tick = subgraph_task_tick << Tick.parse_tick(1)
expected = G(
C(START_TICK, 'a', expected_graph_task_tick, 'a'),
C(START_TICK, 'context', expected_graph_task_tick, 'context'),
T(expected_graph_task_tick, exec_task, {
'name': 'test_exec',
'path': 'test_nested.test_exec'
}),
C(expected_graph_task_tick, 'b', FINAL_TICK, 'b'),
)
utils.assert_graph_equal(expected, graph)
def test_parallel_split(self):
pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
exec_task = ExecTask("exec_name", pkg, ('a', 'b'), ('c', 'd'))
subgraph = G(
C(START_TICK, 'a', 1, 'a'),
C(START_TICK, 'b', 1, 'b'),
C(START_TICK, 'context', 1, 'context'),
T(1, exec_task, {
'name': 'test_exec',
'path': 'test_exec'
}),
C(1, 'c', FINAL_TICK, 'c'),
C(1, 'd', FINAL_TICK, 'd'),
)
subgraph_task_tick = subgraph.get_all_ticks()[0]
graph = Graph()
task = ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps')
tick = START_TICK + 1
graph.add_task(tick, task, {'name': 'ps', 'path': 'ps'})
source = Endpoint(START_TICK, 'a')
dest = Endpoint(tick, 'a')
graph.connect(source, dest)
source = Endpoint(START_TICK, 'b')
dest = Endpoint(tick, 'b')
graph.connect(source, dest)
source = Endpoint(START_TICK, 'context')
dest = Endpoint(tick, 'context')
graph.connect(source, dest)
source = Endpoint(tick, 'cd_list')
dest = Endpoint(FINAL_TICK, 'cd_list')
graph.connect(source, dest)
testdir = tempfile.mkdtemp()
workdir = os.path.join(testdir, 'workdir')
os.mkdir(workdir)
testfile = os.path.join(workdir, 'testfile.pickle')
local_workdir = os.path.join(testdir, 'local_workdir')
os.mkdir(local_workdir)
with open(testfile, 'wb') as f:
pickle.dump(['x', 'y', 'z'], f)
context = {
'transporter': LocalFileTransporter(),
'workdir': workdir,
'local_workdir': local_workdir
}
task.refine(graph, Tick.parse_tick(1), {
'b': 'testfile.pickle',
'context': context
})
mapTask = _mapPortsTask('b', ['x', 'y', 'z'])
map_task_tick = START_TICK + 1 << tick
iterations_tick = map_task_tick + 1
it_1 = START_TICK + 1 << iterations_tick
it_2 = START_TICK + 2 << iterations_tick
it_3 = START_TICK + 3 << iterations_tick
task_tick_1 = subgraph_task_tick << it_1
task_tick_2 = subgraph_task_tick << it_2
task_tick_3 = subgraph_task_tick << it_3
reduceTask = _reducePortsTask(3, ['c', 'd'], 'cd_list')
reduce_tick = iterations_tick + 1
expected = G(
C(START_TICK, 'context', map_task_tick, 'context'),
C(START_TICK, 'a', task_tick_1, 'a'),
C(START_TICK, 'a', task_tick_2, 'a'),
C(START_TICK, 'a', task_tick_3, 'a'),
C(START_TICK, 'b', map_task_tick, 'inputlist'),
C(START_TICK, 'context', task_tick_1, 'context'),
C(START_TICK, 'context', task_tick_2, 'context'),
C(START_TICK, 'context', task_tick_3, 'context'),
T(map_task_tick, mapTask, {
'name': 'map',
'path': 'ps.map'
}),
C(map_task_tick, 'b_1', task_tick_1, 'b'),
C(map_task_tick, 'b_2', task_tick_2, 'b'),
C(map_task_tick, 'b_3', task_tick_3, 'b'),
T(task_tick_1, exec_task, {
'name': 'test_exec',
'path': 'ps.iterations.1.test_exec'
}),
T(task_tick_2, exec_task, {
'name': 'test_exec',
'path': 'ps.iterations.2.test_exec'
}),
T(task_tick_3, exec_task, {
'name': 'test_exec',
'path': 'ps.iterations.3.test_exec'
}),
C(task_tick_1, 'c', reduce_tick, 'c_1'),
C(task_tick_2, 'c', reduce_tick, 'c_2'),
C(task_tick_3, 'c', reduce_tick, 'c_3'),
C(task_tick_1, 'd', reduce_tick, 'd_1'),
C(task_tick_2, 'd', reduce_tick, 'd_2'),
C(task_tick_3, 'd', reduce_tick, 'd_3'),
C(START_TICK, 'context', reduce_tick, 'context'),
T(reduce_tick, reduceTask, {
'name': 'reduce',
'path': 'ps.reduce'
}),
C(reduce_tick, 'cd_list', FINAL_TICK, 'cd_list'),
)
utils.assert_graph_equal(expected, graph)
self.assertTrue(
os.path.exists(os.path.join(local_workdir, 'testfile.pickle')))