def test_nested(self):
r = gf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r_2 = test_utils.TaskOneReturn("root.2")
r.add(r_1, r_2)
r.link(r_1, r_2)
subroot = gf.Flow("subroot")
subroot_r_1 = test_utils.TaskOneReturn("subroot.1")
subroot_r_2 = test_utils.TaskOneReturn("subroot.2")
subroot.add(subroot_r_1, subroot_r_2)
subroot.link(subroot_r_1, subroot_r_2)
r.add(subroot)
r_3 = test_utils.TaskOneReturn("root.3")
r.add(r_3)
r.link(r_2, r_3)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, r_1))
self.assertEqual([['root.1']], _get_scopes(c, r_2))
self.assertEqual([['root.2', 'root.1']], _get_scopes(c, r_3))
self.assertEqual([], _get_scopes(c, subroot_r_1))
self.assertEqual([['subroot.1']], _get_scopes(c, subroot_r_2))
def test_unknown(self):
r = lf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r.add(r_1)
r_2 = test_utils.TaskOneReturn("root.2")
c = compiler.PatternCompiler(r).compile()
self.assertRaises(ValueError, _get_scopes, c, r_2)
def test_unordered_flow_provides_required_values(self):
flow = uf.Flow('uf')
flow.add(utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
flow.add(utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
self.assertEqual(set(['x']), flow.provides)
self.assertEqual(set(['x']), flow.requires)
def test_nested_prior_linear(self):
r = lf.Flow("root")
r.add(test_utils.TaskOneReturn("root.1"),
test_utils.TaskOneReturn("root.2"))
sub_r = lf.Flow("subroot")
sub_r_1 = test_utils.TaskOneReturn("subroot.1")
sub_r.add(sub_r_1)
r.add(sub_r)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([[], ['root.2', 'root.1']], _get_scopes(c, sub_r_1))
def test_single_prior_linear(self):
r = lf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r_2 = test_utils.TaskOneReturn("root.2")
r.add(r_1, r_2)
c = compiler.PatternCompiler(r).compile()
for a in r:
self.assertIn(a, c.execution_graph)
self.assertIsNotNone(c.hierarchy.find(a))
self.assertEqual([], _get_scopes(c, r_1))
self.assertEqual([['root.1']], _get_scopes(c, r_2))
def test_revert_removes_data(self):
flow = lf.Flow('revert-removes').add(
utils.TaskOneReturn(provides='one'),
utils.TaskMultiReturn(provides=('a', 'b', 'c')),
utils.FailingTask(name='fail'))
engine = self._make_engine(flow)
self.assertFailuresRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(engine.storage.fetch_all(), {})
def test_overlap_sibling_expected_result(self):
flow = lf.Flow('flow-1')
flow.add(utils.ProgressingTask(provides='source'))
flow.add(utils.TaskOneReturn(provides='source'))
flow.add(utils.AddOne())
engine = self._make_engine(flow)
engine.run()
results = engine.storage.fetch_all()
self.assertEqual(2, results['result'])
def test_no_visible(self):
r = uf.Flow("root")
atoms = []
for i in range(0, 10):
atoms.append(test_utils.TaskOneReturn("root.%s" % i))
r.add(*atoms)
c = compiler.PatternCompiler(r).compile()
for a in atoms:
self.assertEqual([], _get_scopes(c, a))
def test_execution_pipeline(self):
executor, server = self._start_components([test_utils.TaskOneReturn])
self.assertEqual(0, executor.wait_for_workers(timeout=WAIT_TIMEOUT))
t = test_utils.TaskOneReturn()
f = executor.execute_task(t, uuidutils.generate_uuid(), {})
executor.wait_for_any([f])
t2, _action, result = f.result()
self.assertEqual(1, result)
self.assertEqual(t, t2)
def test_empty(self):
r = lf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r.add(r_1)
c = compiler.PatternCompiler(r).compile()
self.assertIn(r_1, c.execution_graph)
self.assertIsNotNone(c.hierarchy.find(r_1))
walker = sc.ScopeWalker(c, r_1)
scopes = list(walker)
self.assertEqual([], scopes)
def test_graph_linear_scope(self):
r = gf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r_2 = test_utils.TaskOneReturn("root.2")
r.add(r_1, r_2)
r.link(r_1, r_2)
s = lf.Flow("subroot")
s_1 = test_utils.TaskOneReturn("subroot.1")
s_2 = test_utils.TaskOneReturn("subroot.2")
s.add(s_1, s_2)
r.add(s)
t = gf.Flow("subroot2")
t_1 = test_utils.TaskOneReturn("subroot2.1")
t_2 = test_utils.TaskOneReturn("subroot2.2")
t.add(t_1, t_2)
t.link(t_1, t_2)
r.add(t)
r.link(s, t)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, r_1))
self.assertEqual([['root.1']], _get_scopes(c, r_2))
self.assertEqual([], _get_scopes(c, s_1))
self.assertEqual([['subroot.1']], _get_scopes(c, s_2))
self.assertEqual([[], ['subroot.2', 'subroot.1']], _get_scopes(c, t_1))
self.assertEqual([["subroot2.1"], ['subroot.2', 'subroot.1']],
_get_scopes(c, t_2))
def test_execution_pipeline(self):
executor, server = self._start_components([test_utils.TaskOneReturn])
self.assertEqual(0, executor.wait_for_workers(timeout=WAIT_TIMEOUT))
t = test_utils.TaskOneReturn()
progress_callback = lambda *args, **kwargs: None
f = executor.execute_task(t,
uuidutils.generate_uuid(), {},
progress_callback=progress_callback)
waiters.wait_for_any([f])
event, result = f.result()
self.assertEqual(1, result)
self.assertEqual(base_executor.EXECUTED, event)
def test_linear_unordered_scope(self):
r = lf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r_2 = test_utils.TaskOneReturn("root.2")
r.add(r_1, r_2)
u = uf.Flow("subroot")
atoms = []
for i in range(0, 5):
atoms.append(test_utils.TaskOneReturn("subroot.%s" % i))
u.add(*atoms)
r.add(u)
r_3 = test_utils.TaskOneReturn("root.3")
r.add(r_3)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, r_1))
self.assertEqual([['root.1']], _get_scopes(c, r_2))
for a in atoms:
self.assertEqual([[], ['root.2', 'root.1']], _get_scopes(c, a))
scope = _get_scopes(c, r_3)
self.assertEqual(1, len(scope))
first_root = 0
for i, n in enumerate(scope[0]):
if n.startswith('root.'):
first_root = i
break
first_subroot = 0
for i, n in enumerate(scope[0]):
if n.startswith('subroot.'):
first_subroot = i
break
self.assertGreater(first_subroot, first_root)
self.assertEqual(['root.2', 'root.1'], scope[0][-2:])
def test_nested_prior_linear_begin_middle_end(self):
r = lf.Flow("root")
begin_r = test_utils.TaskOneReturn("root.1")
r.add(begin_r, test_utils.TaskOneReturn("root.2"))
middle_r = test_utils.TaskOneReturn("root.3")
r.add(middle_r)
sub_r = lf.Flow("subroot")
sub_r.add(test_utils.TaskOneReturn("subroot.1"),
test_utils.TaskOneReturn("subroot.2"))
r.add(sub_r)
end_r = test_utils.TaskOneReturn("root.4")
r.add(end_r)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, begin_r))
self.assertEqual([['root.2', 'root.1']], _get_scopes(c, middle_r))
self.assertEqual(
[['subroot.2', 'subroot.1', 'root.3', 'root.2', 'root.1']],
_get_scopes(c, end_r))
def test_linear_flow_provides_out_of_order(self):
with self.assertRaises(exceptions.InvariantViolationException):
lf.Flow('lf').add(utils.TaskOneArg('task2'),
utils.TaskOneReturn('task1', provides='x'))
def test_graph_flow_provides_provided_value_other_call(self):
flow = gf.Flow('gf')
flow.add(utils.TaskOneReturn('task1', provides='x'))
flow.add(utils.TaskOneReturn('task2', provides='x'))
self.assertEqual(set(['x']), flow.provides)
def test_unordered_flow_retry_and_task_same_requires_provides(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt', requires=['x']))
flow.add(utils.TaskOneReturn(provides=['x']))
self.assertEqual(set(['x']), flow.requires)
self.assertEqual(set(['x']), flow.provides)
def test_unordered_flow_retry_two_tasks_provide_same_value(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt', provides=['y']))
flow.add(utils.TaskOneReturn('t1', provides=['x']),
utils.TaskOneReturn('t2', provides=['x']))
self.assertEqual(set(['x', 'y']), flow.provides)
def test_graph_flow_retry_and_task_dependency_provide_require(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt', requires=['x']))
flow.add(utils.TaskOneReturn(provides=['x']))
self.assertEqual(set(['x']), flow.provides)
self.assertEqual(set(['x']), flow.requires)
def test_graph_flow_retry_and_task_provide_same_value(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt', provides=['x']))
flow.add(utils.TaskOneReturn('t1', provides=['x']))
self.assertEqual(set(['x']), flow.provides)
def test_linear_flow_provides_same_values(self):
flow = lf.Flow('lf').add(utils.TaskOneReturn(provides='x'))
self.assertRaises(exceptions.DependencyFailure, flow.add,
utils.TaskOneReturn(provides='x'))
def test_unordered_flow_provides_required_values(self):
flow = uf.Flow('uf')
self.assertRaises(exceptions.InvariantViolation, flow.add,
utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
def test_unordered_flow_provides_same_values_one_add(self):
flow = uf.Flow('uf')
self.assertRaises(exceptions.DependencyFailure, flow.add,
utils.TaskOneReturn(provides='x'),
utils.TaskOneReturn(provides='x'))
def _make_dummy_flow(self):
f = linear_flow.Flow('test')
f.add(test_utils.TaskOneReturn("run-1"))
return f
def test_nested_flows_provides_same_values(self):
flow = lf.Flow('lf').add(
uf.Flow('uf').add(utils.TaskOneReturn(provides='x')))
self.assertRaises(exceptions.DependencyFailure, flow.add,
gf.Flow('gf').add(utils.TaskOneReturn(provides='x')))
def test_unordered_flow_provides_required_value_other_call(self):
flow = uf.Flow('uf')
flow.add(utils.TaskOneArg('task2'))
with self.assertRaises(exceptions.InvariantViolationException):
flow.add(utils.TaskOneReturn('task1', provides='x'))
def test_linear_flow_provides_values(self):
flow = lf.Flow('lf').add(
utils.TaskOneReturn('task1', provides='x'),
utils.TaskMultiReturn('task2', provides=['a', 'b', 'c']))
self.assertEquals(flow.requires, set())
self.assertEquals(flow.provides, set(['x', 'a', 'b', 'c']))
def test_graph_flow_provides_provided_value_other_call(self):
flow = gf.Flow('gf')
flow.add(utils.TaskOneReturn('task1', provides='x'))
with self.assertRaises(exceptions.DependencyFailure):
flow.add(utils.TaskOneReturn('task2', provides='x'))
def test_save_as(self):
flow = utils.TaskOneReturn(name='task1', provides='first_data')
engine = self._make_engine(flow)
engine.run()
self.assertEqual({'first_data': 1}, engine.storage.fetch_all())
def test_linear_flow_provides_required_values(self):
flow = lf.Flow('lf').add(utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
self.assertEquals(flow.requires, set())
self.assertEquals(flow.provides, set(['x']))