def test_submit_failed_check(self):
test_executor = TestCmdExecutor(self.config, "1", False, 0, True,
JOB_COMPLETED)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
def expected_error(fail):
self.assertIsInstance(fail, Failure)
self.assertTrue(
fail.value.message.startswith(
"Exception at check_status with reason"))
def unexpected_success(value):
self.fail(
"Unexpected success - failure expected. Return value: %s of type %s"
% (str(value), type(value)))
d.addCallbacks(unexpected_success, expected_error)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
self.assertFalse("summary" in taskprops.keys())
self.assertEqual("test_exec", taskprops['path'])
self.assertEqual("test_exec", taskprops['name'])
def test_nested_task(self):
pkg=PackageSource('testpkg', 'pkgfile', 'NONE')
exec_task = ExecTask("exec_name", pkg, ('a',), ('b',))
subgraph = G(
C(START_TICK, 'a', 1,'a'),
C(START_TICK, 'context', 1,'context'),
T(1, exec_task, {'name': 'test_exec', 'path':'test_exec'}),
C(1, 'b', FINAL_TICK,'b'),
)
subgraph_task_tick=subgraph.get_all_ticks()[0]
graph=Graph()
task=NestedGraphTask(subgraph, 'test_nested')
tick=START_TICK+1
graph.add_task(tick, task, {'name':'test_nested','path':'test_nested'})
source=Endpoint(START_TICK, 'a')
dest=Endpoint(tick, 'a')
graph.connect(source,dest)
source=Endpoint(START_TICK, 'context')
dest=Endpoint(tick, 'context')
graph.connect(source,dest)
source=Endpoint(tick, 'b')
dest=Endpoint(FINAL_TICK, 'b')
graph.connect(source,dest)
task.refine(graph, Tick.parse_tick(1), {'context':{},'a':{}})
expected_graph_task_tick=subgraph_task_tick<<Tick.parse_tick(1)
expected = G(
C(START_TICK, 'a', expected_graph_task_tick,'a'),
C(START_TICK, 'context', expected_graph_task_tick,'context'),
T(expected_graph_task_tick, exec_task, {'name': 'test_exec', 'path':'test_nested.test_exec'}),
C(expected_graph_task_tick, 'b', FINAL_TICK,'b'),
)
utils.assert_graph_equal(expected, graph)
def test_submit_failed_submit(self):
test_executor = TestCmdExecutor(self.config, "1", True, 0, False,
JOB_COMPLETED)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
def expected_error(failure):
self.assertIsInstance(failure, Failure)
self.assertTrue(
failure.value.message.startswith(
"Exception at submit with reason"))
return "Exception has occurred as expected."
def unexpected_success(value):
return Failure(
ValueError(
"Exception did not occur as expected. Value %s found" %
str(value)))
d.addCallback(unexpected_success)
d.addErrback(expected_error)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
self.assertFalse("summary" in taskprops.keys())
self.assertEqual("test_exec", taskprops['path'])
self.assertEqual("test_exec", taskprops['name'])
def test_task_by_property(self):
pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
exec_task1 = ExecTask("exec_name1", pkg, ('a', ), ('b', ))
exec_task2 = ExecTask("exec_name2", pkg, ('a', ), ('b', ))
exec_task3 = ExecTask("exec_name3", pkg, ('a', ), ('b', ))
graph = G(
C(START_TICK, 'a', 1, 'a'),
C(START_TICK, 'context', 1, 'context'),
T(1, exec_task1, {
'name': 'test_exec1',
'path': 'test_exec1',
'prop1': 'a'
}),
T(2, exec_task2, {
'name': 'test_exec2',
'path': 'test_exec2',
'prop1': 'a'
}),
T(3, exec_task3, {
'name': 'test_exec3',
'path': 'test_exec3',
'prop1': 'b'
}),
C(1, 'b', FINAL_TICK, 'b'),
)
self.assertEquals([Tick.parse_tick(1)],
get_ticks_by_property(graph, 'name', 'test_exec1'))
self.assertEquals(
[Tick.parse_tick(1), Tick.parse_tick(2)],
get_ticks_by_property(graph, 'prop1', 'a'))
self.assertEquals([], get_ticks_by_property(graph, 'prop2', 'a'))
def test_submit_error(self):
test_executor = TestCmdExecutor(self.config, "1", False, 0, False,
JOB_ERROR)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
def expected_error(fail):
if isinstance(fail, Failure):
self.assertTrue(True)
else:
self.fail("Type of return value not as expected: %s" %
type(fail))
def unexpected_success(value):
self.fail("Return value not as expected: %s of type %s" %
(str(value), type(value)))
d.addCallbacks(unexpected_success, expected_error)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_ERROR, summary.status)
def test_submit_completed(self):
test_executor=TestCmdExecutor(self.config, "1", False, 0, False, JOB_COMPLETED)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
expected={'c':'test_exec/c.xml'}
self.assertEqual(expected, d.result.result)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_COMPLETED,summary.status)
def test_task_by_property(self):
pkg=PackageSource('testpkg', 'pkgfile', 'NONE')
exec_task1 = ExecTask("exec_name1", pkg, ('a',), ('b',))
exec_task2 = ExecTask("exec_name2", pkg, ('a',), ('b',))
exec_task3 = ExecTask("exec_name3", pkg, ('a',), ('b',))
graph = G(
C(START_TICK, 'a', 1,'a'),
C(START_TICK, 'context', 1,'context'),
T(1, exec_task1, {'name': 'test_exec1', 'path':'test_exec1', 'prop1':'a'}),
T(2, exec_task2, {'name': 'test_exec2', 'path':'test_exec2', 'prop1':'a'}),
T(3, exec_task3, {'name': 'test_exec3', 'path':'test_exec3', 'prop1':'b'}),
C(1, 'b', FINAL_TICK,'b'),
)
self.assertEquals([Tick.parse_tick(1)], get_ticks_by_property(graph, 'name', 'test_exec1'))
self.assertEquals([Tick.parse_tick(1), Tick.parse_tick(2)], get_ticks_by_property(graph, 'prop1', 'a'))
self.assertEquals([], get_ticks_by_property(graph, 'prop2', 'a'))
def test_submit_error_after_some_checks(self):
num_checks=2
test_executor=TestCmdExecutor(self.config, "1", False, 2, False, JOB_ERROR)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
t1=datetime.datetime.now()
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
def expected_error(fail):
if isinstance(fail, Failure):
self.assertTrue(True)
t2=datetime.datetime.now()
self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_ERROR,summary.status)
else:
self.fail("Type of return value not as expected: %s"%type(fail))
def unexpected_success(value):
self.fail("Return value not as expected: %s of type %s"%(str(value),type(value)))
d.addCallbacks(unexpected_success, expected_error)
return d
def test_submit_completed_after_some_checks(self):
num_checks = 2
test_executor = TestCmdExecutor(self.config, "1", False, 2, False,
JOB_COMPLETED)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
t1 = datetime.datetime.now()
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
expected = {'c': 'test_exec/c.xml'}
def check_success(_):
t2 = datetime.datetime.now()
self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
self.assertEqual(expected, d.result.result)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_COMPLETED, summary.status)
d.addCallback(check_success)
return d
def test_evaluate_reduce_task(self):
testdir=tempfile.mkdtemp()
workdir=os.path.join(testdir,'workdir')
os.mkdir(workdir)
local_workdir=os.path.join(testdir,'local_workdir')
os.makedirs(os.path.join(local_workdir,'ps.reduce'))
testfile=os.path.join(local_workdir, 'ps.reduce','cd_list')
with open(testfile,'wb') as f:
pickle.dump([('x_1','y_1'), ('x_2','y_2'), ('x_3','y_3')],f)
context={'transporter': LocalFileTransporter(), 'workdir':workdir, 'local_workdir':local_workdir}
reduce_task=_reducePortsTask(3, ['c','d'], 'cd_list')
reduce_tick=START_TICK+3<<Tick.parse_tick(1)
g = G(
C(START_TICK, 'context', reduce_tick,'context'),
T(reduce_tick, reduce_task, {'name': 'reduce', 'path':'ps.reduce'}),
C(reduce_tick, 'cd_list', FINAL_TICK,'cd_list'),
)
inputs={}
inputs['context']=context
inputs['c_1']='x_1'
inputs['d_1']='y_1'
inputs['c_2']='x_2'
inputs['d_2']='y_2'
inputs['c_3']='x_3'
inputs['d_3']='y_3'
result=[('x_1','y_1'),('x_2','y_2'),('x_3','y_3')]
reduce_task.evaluate(g, reduce_tick, reduce_task, inputs)
outputfile=os.path.join(workdir,'ps.reduce','cd_list')
self.assertTrue(os.path.exists(outputfile))
with open(outputfile,'r') as f:
output=pickle.load(f)
self.assertEqual(result, output)
def test_submit_completed(self):
test_executor = TestCmdExecutor(self.config, "1", False, 0, False,
JOB_COMPLETED)
self.node_callbacks._cmd_executor = test_executor
tick = Tick.parse_tick(1)
d = self.node_callbacks.submit_task(self.g, tick, self.task,
self.inputs)
expected = {'c': 'test_exec/c.xml'}
self.assertEqual(expected, d.result.result)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_COMPLETED, summary.status)
def test_nested_task(self):
pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
exec_task = ExecTask("exec_name", pkg, ('a', ), ('b', ))
subgraph = G(
C(START_TICK, 'a', 1, 'a'),
C(START_TICK, 'context', 1, 'context'),
T(1, exec_task, {
'name': 'test_exec',
'path': 'test_exec'
}),
C(1, 'b', FINAL_TICK, 'b'),
)
subgraph_task_tick = subgraph.get_all_ticks()[0]
graph = Graph()
task = NestedGraphTask(subgraph, 'test_nested')
tick = START_TICK + 1
graph.add_task(tick, task, {
'name': 'test_nested',
'path': 'test_nested'
})
source = Endpoint(START_TICK, 'a')
dest = Endpoint(tick, 'a')
graph.connect(source, dest)
source = Endpoint(START_TICK, 'context')
dest = Endpoint(tick, 'context')
graph.connect(source, dest)
source = Endpoint(tick, 'b')
dest = Endpoint(FINAL_TICK, 'b')
graph.connect(source, dest)
task.refine(graph, Tick.parse_tick(1), {'context': {}, 'a': {}})
expected_graph_task_tick = subgraph_task_tick << Tick.parse_tick(1)
expected = G(
C(START_TICK, 'a', expected_graph_task_tick, 'a'),
C(START_TICK, 'context', expected_graph_task_tick, 'context'),
T(expected_graph_task_tick, exec_task, {
'name': 'test_exec',
'path': 'test_nested.test_exec'
}),
C(expected_graph_task_tick, 'b', FINAL_TICK, 'b'),
)
utils.assert_graph_equal(expected, graph)
def test_submit_failed_check(self):
test_executor=TestCmdExecutor(self.config, "1", False, 0, True, JOB_COMPLETED)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
def expected_error(fail):
self.assertIsInstance(fail, Failure)
self.assertTrue(fail.value.message.startswith("Exception at check_status with reason"))
def unexpected_success(value):
self.fail("Unexpected success - failure expected. Return value: %s of type %s"%(str(value),type(value)))
d.addCallbacks(unexpected_success, expected_error)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
self.assertFalse("summary" in taskprops.keys())
self.assertEqual("test_exec",taskprops['path'])
self.assertEqual("test_exec",taskprops['name'])
def check_success(_):
t2=datetime.datetime.now()
self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
self.assertEqual(expected, d.result.result)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_COMPLETED,summary.status)
def check_success(_):
t2 = datetime.datetime.now()
self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
self.assertEqual(expected, d.result.result)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_COMPLETED, summary.status)
def test_submit_failed_submit(self):
test_executor=TestCmdExecutor(self.config, "1", True, 0, False, JOB_COMPLETED)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
def expected_error(failure):
self.assertIsInstance(failure, Failure)
self.assertTrue(failure.value.message.startswith("Exception at submit with reason"))
return "Exception has occurred as expected."
def unexpected_success(value):
return Failure(ValueError("Exception did not occur as expected. Value %s found"%str(value)))
d.addCallback(unexpected_success)
d.addErrback(expected_error)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
self.assertFalse("summary" in taskprops.keys())
self.assertEqual("test_exec",taskprops['path'])
self.assertEqual("test_exec",taskprops['name'])
def test_eval_fail(self):
g = G(
T(1, tasks.ConstTask(None)),
C(1, "value", FINAL_TICK, "retval")
)
d = self.target.execute(g, {})
self.next_ready()[-1].errback(failure.Failure(MockError()))
f = twistit.extract_failure(d)
self.assertTrue(f.check(traverser.EvaluationError))
self.assertTrue(f.value.cause.check(MockError))
self.assertEqual(Tick.parse_tick(1), f.value.tick)
def test_refine_body_twice(self):
# def f(x):
# while x:
# pass
# return x
with graph_factory():
g = G(
C(START_TICK, "x", 1, "$test"),
C(START_TICK, "x", 1, "x"),
T(1, tasks.WhileTask(False, False, G("body",
C(START_TICK, "x", 1 , "$test"),
C(START_TICK, "x", 1 , "x"),
T(1, tasks.WhileTask(True, False, G("body"), G("else"))),
), G("else"))),
C(START_TICK, "x", FINAL_TICK, "retval")
)
target = g.get_task(START_TICK + 1)
target.refine(g, START_TICK + 1, {"$test":True})
target = g.get_task(Tick.parse_tick((1,1,1)))
target.refine(g, Tick.parse_tick((1,1,1)), {"$test":True})
with graph_factory():
expected = G(
C(START_TICK, "x", (1,2,1), "$test"),
C(START_TICK, "x", (1,2,1), "x"),
T((1,2,1), tasks.WhileTask(True, False, G("body",
C(START_TICK, "x", 1 , "$test"),
C(START_TICK, "x", 1 , "x"),
T(1, tasks.WhileTask(True, False, G("body"), G("else"))),
), G("else"))),
C(START_TICK, "x", FINAL_TICK, "retval")
)
utils.assert_graph_equal(expected, g)
def expected_error(fail):
if isinstance(fail, Failure):
self.assertTrue(True)
t2=datetime.datetime.now()
self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_ERROR,summary.status)
else:
self.fail("Type of return value not as expected: %s"%type(fail))
def test_refine_body_twice(self):
# def f(lst, x):
# for x in lst:
# pass
# return x
with graph_factory():
g = G(
C(START_TICK, "lst", 1, "iterable"),
T(1, tasks.IterTask()),
C(1, "value", 2, "$iterator"),
C(START_TICK, "x", 2, "x"),
T(2, tasks.ForTask(False, False, G("body",
C(START_TICK, "$target",1 , "x"),
C(START_TICK, "$iterator", 1, "$iterator"),
T(1, tasks.ForTask(True, False, G("body"), G("else"))),
C(1, "x", FINAL_TICK, "x")
), G("else"))),
C(2, "x", FINAL_TICK, "retval")
)
it = iter([1,2,3])
target_tick = START_TICK + 2
target = g.get_task(target_tick)
target.refine(g, target_tick, {"$iterator":it})
target_tick = Tick.parse_tick((2,1,2,1))
target = g.get_task(target_tick)
target.refine(g, target_tick, {"$iterator":it})
with graph_factory():
expected = G(
C(START_TICK, "lst", 1, "iterable"),
T(1, tasks.IterTask()),
T((2,1,1), tasks.ConstTask(1)),
T((2,2,1), tasks.ConstTask(2)),
C(1, "value", (2,2,2,1), "$iterator"),
C((2,2,1), "value", (2,2,2,1), "x"),
T((2,2,2,1), tasks.ForTask(True, False, G("body",
C(START_TICK, "$target",1 , "x"),
C(START_TICK, "$iterator", 1, "$iterator"),
T(1, tasks.ForTask(True, False, G("body"), G("else"))),
C(1, "x", FINAL_TICK, "x")
), G("else"))),
C((2,2,2,1), "x", FINAL_TICK, "retval")
)
utils.assert_graph_equal(expected, g)
def expected_error(fail):
if isinstance(fail, Failure):
self.assertTrue(True)
t2 = datetime.datetime.now()
self.assertTrue((t2 - t1).total_seconds() > num_checks * 2.0)
tick = Tick.parse_tick(1)
taskprops = self.g.get_task_properties(tick)
summary = taskprops['summary']
self.assertEqual("test_exec", summary.dfpath)
self.assertEqual("test_exec", summary.outdir)
self.assertEqual("1", summary.pid)
self.assertEqual(JOB_ERROR, summary.status)
else:
self.fail("Type of return value not as expected: %s" %
type(fail))
def test_refine_fail(self):
task = MockTask("in")
g = G(
C(START_TICK, "a", 1, "in"),
T(1, task),
C(1, "out", FINAL_TICK, "retval")
)
d = self.target.execute(g, {"a": "refinedata"})
self.next_refine()[-1].errback(failure.Failure(MockError()))
f = twistit.extract_failure(d)
self.assertTrue(f.check(traverser.RefineError))
self.assertTrue(f.value.cause.check(MockError))
self.assertEqual(Tick.parse_tick(1), f.value.tick)
def test_evaluate_reduce_task(self):
testdir = tempfile.mkdtemp()
workdir = os.path.join(testdir, 'workdir')
os.mkdir(workdir)
local_workdir = os.path.join(testdir, 'local_workdir')
os.makedirs(os.path.join(local_workdir, 'ps.reduce'))
testfile = os.path.join(local_workdir, 'ps.reduce', 'cd_list')
with open(testfile, 'wb') as f:
pickle.dump([('x_1', 'y_1'), ('x_2', 'y_2'), ('x_3', 'y_3')], f)
context = {
'transporter': LocalFileTransporter(),
'workdir': workdir,
'local_workdir': local_workdir
}
reduce_task = _reducePortsTask(3, ['c', 'd'], 'cd_list')
reduce_tick = START_TICK + 3 << Tick.parse_tick(1)
g = G(
C(START_TICK, 'context', reduce_tick, 'context'),
T(reduce_tick, reduce_task, {
'name': 'reduce',
'path': 'ps.reduce'
}),
C(reduce_tick, 'cd_list', FINAL_TICK, 'cd_list'),
)
inputs = {}
inputs['context'] = context
inputs['c_1'] = 'x_1'
inputs['d_1'] = 'y_1'
inputs['c_2'] = 'x_2'
inputs['d_2'] = 'y_2'
inputs['c_3'] = 'x_3'
inputs['d_3'] = 'y_3'
result = [('x_1', 'y_1'), ('x_2', 'y_2'), ('x_3', 'y_3')]
reduce_task.evaluate(g, reduce_tick, reduce_task, inputs)
outputfile = os.path.join(workdir, 'ps.reduce', 'cd_list')
self.assertTrue(os.path.exists(outputfile))
with open(outputfile, 'r') as f:
output = pickle.load(f)
self.assertEqual(result, output)
def test_submit_completed_after_some_checks(self):
num_checks=2
test_executor=TestCmdExecutor(self.config, "1", False, 2, False, JOB_COMPLETED)
self.node_callbacks._cmd_executor=test_executor
tick=Tick.parse_tick(1)
t1=datetime.datetime.now()
d=self.node_callbacks.submit_task(self.g, tick, self.task, self.inputs)
expected={'c':'test_exec/c.xml'}
def check_success(_):
t2=datetime.datetime.now()
self.assertTrue((t2-t1).total_seconds()>num_checks*2.0)
self.assertEqual(expected, d.result.result)
tick=Tick.parse_tick(1)
taskprops=self.g.get_task_properties(tick)
summary=taskprops['summary']
self.assertEqual("test_exec",summary.dfpath)
self.assertEqual("test_exec",summary.outdir)
self.assertEqual("1",summary.pid)
self.assertEqual(JOB_COMPLETED,summary.status)
d.addCallback(check_success)
return d
def test_parallel_split(self):
pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
exec_task = ExecTask("exec_name", pkg, ('a', 'b'), ('c', 'd'))
subgraph = G(
C(START_TICK, 'a', 1, 'a'),
C(START_TICK, 'b', 1, 'b'),
C(START_TICK, 'context', 1, 'context'),
T(1, exec_task, {
'name': 'test_exec',
'path': 'test_exec'
}),
C(1, 'c', FINAL_TICK, 'c'),
C(1, 'd', FINAL_TICK, 'd'),
)
subgraph_task_tick = subgraph.get_all_ticks()[0]
graph = Graph()
task = ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps')
tick = START_TICK + 1
graph.add_task(tick, task, {'name': 'ps', 'path': 'ps'})
source = Endpoint(START_TICK, 'a')
dest = Endpoint(tick, 'a')
graph.connect(source, dest)
source = Endpoint(START_TICK, 'b')
dest = Endpoint(tick, 'b')
graph.connect(source, dest)
source = Endpoint(START_TICK, 'context')
dest = Endpoint(tick, 'context')
graph.connect(source, dest)
source = Endpoint(tick, 'cd_list')
dest = Endpoint(FINAL_TICK, 'cd_list')
graph.connect(source, dest)
testdir = tempfile.mkdtemp()
workdir = os.path.join(testdir, 'workdir')
os.mkdir(workdir)
testfile = os.path.join(workdir, 'testfile.pickle')
local_workdir = os.path.join(testdir, 'local_workdir')
os.mkdir(local_workdir)
with open(testfile, 'wb') as f:
pickle.dump(['x', 'y', 'z'], f)
context = {
'transporter': LocalFileTransporter(),
'workdir': workdir,
'local_workdir': local_workdir
}
task.refine(graph, Tick.parse_tick(1), {
'b': 'testfile.pickle',
'context': context
})
mapTask = _mapPortsTask('b', ['x', 'y', 'z'])
map_task_tick = START_TICK + 1 << tick
iterations_tick = map_task_tick + 1
it_1 = START_TICK + 1 << iterations_tick
it_2 = START_TICK + 2 << iterations_tick
it_3 = START_TICK + 3 << iterations_tick
task_tick_1 = subgraph_task_tick << it_1
task_tick_2 = subgraph_task_tick << it_2
task_tick_3 = subgraph_task_tick << it_3
reduceTask = _reducePortsTask(3, ['c', 'd'], 'cd_list')
reduce_tick = iterations_tick + 1
expected = G(
C(START_TICK, 'context', map_task_tick, 'context'),
C(START_TICK, 'a', task_tick_1, 'a'),
C(START_TICK, 'a', task_tick_2, 'a'),
C(START_TICK, 'a', task_tick_3, 'a'),
C(START_TICK, 'b', map_task_tick, 'inputlist'),
C(START_TICK, 'context', task_tick_1, 'context'),
C(START_TICK, 'context', task_tick_2, 'context'),
C(START_TICK, 'context', task_tick_3, 'context'),
T(map_task_tick, mapTask, {
'name': 'map',
'path': 'ps.map'
}),
C(map_task_tick, 'b_1', task_tick_1, 'b'),
C(map_task_tick, 'b_2', task_tick_2, 'b'),
C(map_task_tick, 'b_3', task_tick_3, 'b'),
T(task_tick_1, exec_task, {
'name': 'test_exec',
'path': 'ps.iterations.1.test_exec'
}),
T(task_tick_2, exec_task, {
'name': 'test_exec',
'path': 'ps.iterations.2.test_exec'
}),
T(task_tick_3, exec_task, {
'name': 'test_exec',
'path': 'ps.iterations.3.test_exec'
}),
C(task_tick_1, 'c', reduce_tick, 'c_1'),
C(task_tick_2, 'c', reduce_tick, 'c_2'),
C(task_tick_3, 'c', reduce_tick, 'c_3'),
C(task_tick_1, 'd', reduce_tick, 'd_1'),
C(task_tick_2, 'd', reduce_tick, 'd_2'),
C(task_tick_3, 'd', reduce_tick, 'd_3'),
C(START_TICK, 'context', reduce_tick, 'context'),
T(reduce_tick, reduceTask, {
'name': 'reduce',
'path': 'ps.reduce'
}),
C(reduce_tick, 'cd_list', FINAL_TICK, 'cd_list'),
)
utils.assert_graph_equal(expected, graph)
self.assertTrue(
os.path.exists(os.path.join(local_workdir, 'testfile.pickle')))
def test_parallel_split(self):
pkg=PackageSource('testpkg', 'pkgfile', 'NONE')
exec_task = ExecTask("exec_name", pkg, ('a','b'), ('c','d'))
subgraph = G(
C(START_TICK, 'a', 1,'a'),
C(START_TICK, 'b', 1,'b'),
C(START_TICK, 'context', 1,'context'),
T(1, exec_task, {'name': 'test_exec', 'path':'test_exec'}),
C(1, 'c', FINAL_TICK,'c'),
C(1, 'd', FINAL_TICK,'d'),
)
subgraph_task_tick=subgraph.get_all_ticks()[0]
graph=Graph()
task=ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps')
tick=START_TICK+1
graph.add_task(tick, task, {'name':'ps','path':'ps'})
source=Endpoint(START_TICK, 'a')
dest=Endpoint(tick, 'a')
graph.connect(source,dest)
source=Endpoint(START_TICK, 'b')
dest=Endpoint(tick, 'b')
graph.connect(source,dest)
source=Endpoint(START_TICK, 'context')
dest=Endpoint(tick, 'context')
graph.connect(source,dest)
source=Endpoint(tick, 'cd_list')
dest=Endpoint(FINAL_TICK, 'cd_list')
graph.connect(source,dest)
testdir=tempfile.mkdtemp()
workdir=os.path.join(testdir,'workdir')
os.mkdir(workdir)
testfile=os.path.join(workdir, 'testfile.pickle')
local_workdir=os.path.join(testdir,'local_workdir')
os.mkdir(local_workdir)
with open(testfile,'wb') as f:
pickle.dump(['x','y','z'],f)
context={'transporter': LocalFileTransporter(), 'workdir':workdir, 'local_workdir':local_workdir}
task.refine(graph, Tick.parse_tick(1), {'b':'testfile.pickle', 'context':context})
mapTask=_mapPortsTask('b', ['x','y','z'])
map_task_tick=START_TICK+1<<tick
iterations_tick=map_task_tick+1
it_1 = START_TICK + 1 << iterations_tick
it_2 = START_TICK + 2 << iterations_tick
it_3 = START_TICK + 3 << iterations_tick
task_tick_1=subgraph_task_tick<<it_1
task_tick_2=subgraph_task_tick<<it_2
task_tick_3=subgraph_task_tick<<it_3
reduceTask=_reducePortsTask(3, ['c','d'], 'cd_list')
reduce_tick=iterations_tick+1
expected = G(
C(START_TICK, 'context', map_task_tick,'context'),
C(START_TICK, 'a', task_tick_1,'a'),
C(START_TICK, 'a', task_tick_2,'a'),
C(START_TICK, 'a', task_tick_3,'a'),
C(START_TICK, 'b', map_task_tick,'inputlist'),
C(START_TICK, 'context', task_tick_1,'context'),
C(START_TICK, 'context', task_tick_2,'context'),
C(START_TICK, 'context', task_tick_3,'context'),
T(map_task_tick, mapTask, {'name': 'map', 'path':'ps.map'}),
C(map_task_tick, 'b_1', task_tick_1,'b'),
C(map_task_tick, 'b_2', task_tick_2,'b'),
C(map_task_tick, 'b_3', task_tick_3,'b'),
T(task_tick_1, exec_task, {'name': 'test_exec', 'path':'ps.iterations.1.test_exec'}),
T(task_tick_2, exec_task, {'name': 'test_exec', 'path':'ps.iterations.2.test_exec'}),
T(task_tick_3, exec_task, {'name': 'test_exec', 'path':'ps.iterations.3.test_exec'}),
C(task_tick_1, 'c', reduce_tick,'c_1'),
C(task_tick_2, 'c', reduce_tick,'c_2'),
C(task_tick_3, 'c', reduce_tick,'c_3'),
C(task_tick_1, 'd', reduce_tick,'d_1'),
C(task_tick_2, 'd', reduce_tick,'d_2'),
C(task_tick_3, 'd', reduce_tick,'d_3'),
C(START_TICK, 'context', reduce_tick,'context'),
T(reduce_tick, reduceTask, {'name': 'reduce', 'path':'ps.reduce'}),
C(reduce_tick, 'cd_list', FINAL_TICK,'cd_list'),
)
utils.assert_graph_equal(expected, graph)
self.assertTrue(os.path.exists(os.path.join(local_workdir,'testfile.pickle')))