def test_sanity_run(self):
(flo, history) = self._make_watched_flow("sanity-test")
flo.add(utils.ProvidesRequiresTask("do-this",
provides=['a'],
requires=[]))
flo.add(utils.ProvidesRequiresTask("do-that",
provides=['c'],
requires=['a']))
flo.add(utils.ProvidesRequiresTask("do-other",
provides=['d'],
requires=[]))
flo.add(utils.ProvidesRequiresTask("do-thing",
provides=['e'],
requires=['d']))
self.assertEquals(states.PENDING, flo.state)
context = {}
flo.run(context)
self.assertEquals(states.SUCCESS, flo.state)
self.assertTrue(len(context) > 0)
# Even when running in parallel this will be the required order since
# 'do-that' depends on 'do-this' finishing first.
expected_order = ['do-this', 'do-that']
this_that = [t for t in context[utils.ORDER_KEY]
if t in expected_order]
self.assertEquals(expected_order, this_that)
expected_order = ['do-other', 'do-thing']
this_that = [t for t in context[utils.ORDER_KEY]
if t in expected_order]
self.assertEquals(expected_order, this_that)
def test_simple_resume(self):
(flo, history) = self._make_watched_flow("sanity-test")
f_uuid = flo.add(utils.ProvidesRequiresTask("do-this",
provides=['a'],
requires=[]))
flo.add(utils.ProvidesRequiresTask("do-that",
provides=['c'],
requires=['a']))
def resume_it(flow, ordering):
ran_already = []
not_ran = []
for r in ordering:
if r.uuid == f_uuid:
ran_already.append((r, {
'result': 'b',
'states': [states.SUCCESS],
}))
else:
not_ran.append(r)
return (ran_already, not_ran)
flo.resumer = resume_it
flo.run({})
self.assertEquals('b', flo.results[f_uuid])
self.assertEquals(states.SUCCESS, flo.state)
def test_dependent(self):
r = gf.Flow("root")
customer = test_utils.ProvidesRequiresTask("customer",
provides=['dog'],
requires=[])
washer = test_utils.ProvidesRequiresTask("washer",
requires=['dog'],
provides=['wash'])
dryer = test_utils.ProvidesRequiresTask("dryer",
requires=['dog', 'wash'],
provides=['dry_dog'])
shaved = test_utils.ProvidesRequiresTask("shaver",
requires=['dry_dog'],
provides=['shaved_dog'])
happy_customer = test_utils.ProvidesRequiresTask(
"happy_customer", requires=['shaved_dog'], provides=['happiness'])
r.add(customer, washer, dryer, shaved, happy_customer)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, customer))
self.assertEqual([['washer', 'customer']], _get_scopes(c, dryer))
self.assertEqual([['shaver', 'dryer', 'washer', 'customer']],
_get_scopes(c, happy_customer))
def test_shadow_graph(self):
r = gf.Flow("root")
customer = test_utils.ProvidesRequiresTask("customer",
provides=['dog'],
requires=[])
customer2 = test_utils.ProvidesRequiresTask("customer2",
provides=['dog'],
requires=[])
washer = test_utils.ProvidesRequiresTask("washer",
requires=['dog'],
provides=['wash'])
r.add(customer, washer)
r.add(customer2, resolve_requires=False)
r.link(customer2, washer)
c = compiler.PatternCompiler(r).compile()
# The order currently is *not* guaranteed to be 'customer' before
# 'customer2' or the reverse, since either can occur before the
# washer; since *either* is a valid topological ordering of the
# dependencies...
#
# This may be different after/if the following is resolved:
#
# https://github.com/networkx/networkx/issues/1181 (and a few others)
self.assertEqual(set(['customer', 'customer2']),
set(_get_scopes(c, washer)[0]))
self.assertEqual([], _get_scopes(c, customer2))
self.assertEqual([], _get_scopes(c, customer))
def test_graph_nested_requires(self):
a = test_utils.ProvidesRequiresTask('a', provides=['x'], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
c = test_utils.ProvidesRequiresTask('c', provides=[], requires=['x'])
inner_flo = lf.Flow("test2").add(b, c)
flo = gf.Flow("test").add(a, inner_flo)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(7, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'a', {
'invariant': True
}),
('test2', 'b', {
'invariant': True
}),
('a', 'test2', {
'reasons': set(['x'])
}),
('b', 'c', {
'invariant': True
}),
('c', 'test2[$]', {
'invariant': True
}),
('test2[$]', 'test[$]', {
'invariant': True
}),
])
self.assertItemsEqual(['test'], list(g.no_predecessors_iter()))
self.assertItemsEqual(['test[$]'], list(g.no_successors_iter()))
def test_many_empty_in_graph_flow(self):
flow = gf.Flow('root')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
flow.add(a)
b = lf.Flow('b')
b_0 = test_utils.ProvidesRequiresTask('b.0', provides=[], requires=[])
b_3 = test_utils.ProvidesRequiresTask('b.3', provides=[], requires=[])
b.add(
b_0,
lf.Flow('b.1'),
lf.Flow('b.2'),
b_3,
)
flow.add(b)
c = lf.Flow('c')
c.add(lf.Flow('c.0'), lf.Flow('c.1'), lf.Flow('c.2'))
flow.add(c)
d = test_utils.ProvidesRequiresTask('d', provides=[], requires=[])
flow.add(d)
flow.link(b, d)
flow.link(a, d)
flow.link(c, d)
compilation = compiler.PatternCompiler(flow).compile()
g = compilation.execution_graph
self.assertTrue(g.has_edge(b_0, b_3))
self.assertTrue(g.has_edge(b_3, d))
self.assertEqual(4, len(g))
def test_graph_nested_provides(self):
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=['x'])
b = test_utils.ProvidesRequiresTask('b', provides=['x'], requires=[])
c = test_utils.ProvidesRequiresTask('c', provides=[], requires=[])
inner_flo = lf.Flow("test2").add(b, c)
flo = gf.Flow("test").add(a, inner_flo)
compilation = compiler.PatternCompiler(flo).compile()
graph = compilation.execution_graph
self.assertEqual(5, len(graph))
self.assertItemsEqual(graph.edges(data=True), [
(flo, inner_flo, {
'invariant': True
}),
(inner_flo, b, {
'invariant': True
}),
(b, c, {
'invariant': True
}),
(c, a, {
'reasons': set(['x'])
}),
])
self.assertItemsEqual([flo], graph.no_predecessors_iter())
self.assertItemsEqual([a], graph.no_successors_iter())
def test_no_input_flo(self):
(flo, history) = self._make_watched_flow("sanity-test")
flo.add(utils.ProvidesRequiresTask("do-that",
provides=['c'],
requires=['a']))
flo.add(utils.ProvidesRequiresTask("do-this",
provides=['b'],
requires=['c']))
self.assertRaises(excp.InvalidStateException, flo.run, {})
def test_recache_on_add_no_deps(self):
wf = gw.TargetedFlow("test")
test_1 = utils.ProvidesRequiresTask('test-1', provides=[], requires=[])
wf.add(test_1)
wf.set_target(test_1)
self.assertEqual(1, len(wf.graph))
test_2 = utils.ProvidesRequiresTask('test-2', provides=[], requires=[])
wf.add(test_2)
self.assertEqual(1, len(wf.graph))
def test_recache_on_link(self):
wf = gw.TargetedFlow("test")
test_1 = utils.ProvidesRequiresTask('test-1', provides=[], requires=[])
test_2 = utils.ProvidesRequiresTask('test-2', provides=[], requires=[])
wf.add(test_1, test_2)
wf.set_target(test_1)
self.assertEqual(1, len(wf.graph))
wf.link(test_2, test_1)
self.assertEqual(2, len(wf.graph))
self.assertEqual([(test_2, test_1)], list(wf.graph.edges()))
def test_empty_flow_in_graph_flow(self):
flow = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=['a'], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=['a'])
empty_flow = lf.Flow("empty")
flow.add(a, empty_flow, b)
compilation = compiler.PatternCompiler(flow).compile()
g = compilation.execution_graph
self.assertTrue(g.has_edge(a, b))
def test_flatten_attribute(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1', requires=[], provides=[])
test_2 = utils.ProvidesRequiresTask('test-2', provides=[], requires=[])
wf.add(test_1, test_2)
wf.link(test_1, test_2)
g = fu.flatten(wf)
self.assertEqual(2, len(g))
edge_attrs = gu.get_edge_attrs(g, test_1, test_2)
self.assertTrue(edge_attrs.get('manual'))
self.assertTrue(edge_attrs.get('flatten'))
def test_empty_flow_in_graph_flow_empty_linkage(self):
flow = gf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
empty_flow = lf.Flow("empty")
flow.add(a, empty_flow, b)
flow.link(empty_flow, b)
compilation = compiler.PatternCompiler(flow).compile()
g = compilation.execution_graph
self.assertEqual(0, len(g.edges()))
def test_multi_provider_disallowed(self):
flo = gw.Flow("test-flow", allow_same_inputs=False)
flo.add(
utils.ProvidesRequiresTask('test6', provides=['y'], requires=[]))
flo.add(
utils.ProvidesRequiresTask('test7', provides=['y'], requires=[]))
flo.add(
utils.ProvidesRequiresTask('test8', provides=[], requires=['y']))
self.assertEquals(states.PENDING, flo.state)
self.assertRaises(excp.InvalidStateException, flo.run, {})
self.assertEquals(states.FAILURE, flo.state)
def test_targeted_flow_bad_target(self):
wf = gw.TargetedFlow("test")
test_1 = utils.ProvidesRequiresTask('test-1',
provides=['a'],
requires=[])
test_2 = utils.ProvidesRequiresTask('test-2',
provides=['b'],
requires=['a'])
wf.add(test_1)
self.assertRaisesRegexp(ValueError, '^Item .* not found',
wf.set_target, test_2)
def test_linked_edge_reasons(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1', requires=[], provides=[])
test_2 = utils.ProvidesRequiresTask('test-2', provides=[], requires=[])
wf.add(test_1, test_2)
self.assertFalse(wf.graph.has_edge(test_1, test_2))
wf.link(test_1, test_2)
self.assertTrue(wf.graph.has_edge(test_1, test_2))
edge_attrs = gu.get_edge_attrs(wf.graph, test_1, test_2)
self.assertTrue(len(edge_attrs) > 0)
self.assertTrue(edge_attrs.get('manual'))
def test_graph_dependencies(self):
a = test_utils.ProvidesRequiresTask('a', provides=['x'], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=['x'])
flo = gf.Flow("test").add(a, b)
compilation = compiler.PatternCompiler(flo).compile()
g = compilation.execution_graph
self.assertEqual(2, len(g))
self.assertItemsEqual(g.edges(data=True), [(a, b, {
'reasons': set(['x'])
})])
self.assertItemsEqual([a], g.no_predecessors_iter())
self.assertItemsEqual([b], g.no_successors_iter())
def test_empty_flow_in_linear_flow(self):
flow = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
empty_flow = gf.Flow("empty")
flow.add(a, empty_flow, b)
compilation = compiler.PatternCompiler(flow).compile()
g = compilation.execution_graph
self.assertItemsEqual(g.edges(data=True), [
(a, b, {
'invariant': True
}),
])
def test_empty_flow_in_linear_flow(self):
flo = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
empty_flo = gf.Flow("empty")
flo.add(a, empty_flo, b)
compilation = compiler.PatternCompiler(flo).compile()
graph = compilation.execution_graph
self.assertItemsEqual(graph.edges(), [
(flo, a),
(a, empty_flo),
(empty_flo, b),
])
def test_looping_flow(self):
flo = gw.Flow("test-flow")
flo.add(
utils.ProvidesRequiresTask('test1',
provides=['a', 'b'],
requires=['c', 'd', 'e']))
flo.add(
utils.ProvidesRequiresTask('test2',
provides=['c', 'd', 'e'],
requires=['a', 'b']))
ctx = collections.defaultdict(list)
self.assertEquals(states.PENDING, flo.state)
self.assertRaises(excp.InvalidStateException, flo.run, ctx)
self.assertEquals(states.FAILURE, flo.state)
def test_flow_bad_order(self):
wf = lw.Flow("the-test-action")
wf.add(
utils.ProvidesRequiresTask('test-1',
requires=set(),
provides=['a', 'b']))
# This one should fail to add since it requires 'c'
no_req_task = utils.ProvidesRequiresTask('test-2',
requires=['c'],
provides=[])
wf.add(no_req_task)
e = _make_engine(wf)
self.assertRaises(exc.NotFound, e.run)
def test_graph_nested_provides(self):
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=['x'])
b = test_utils.ProvidesRequiresTask('b', provides=['x'], requires=[])
c = test_utils.ProvidesRequiresTask('c', provides=[], requires=[])
flo = gf.Flow("test").add(a, lf.Flow("test2").add(b, c))
compilation = compiler.PatternCompiler(flo).compile()
g = compilation.execution_graph
self.assertEqual(3, len(g))
self.assertItemsEqual(g.edges(data=True), [(b, c, {
'invariant': True
}), (b, a, {
'reasons': set(['x'])
})])
self.assertItemsEqual([b], g.no_predecessors_iter())
self.assertItemsEqual([a, c], g.no_successors_iter())
def test_empty_flow_in_linear_flow(self):
flo = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
empty_flo = gf.Flow("empty")
flo.add(a, empty_flo, b)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertItemsEqual(g.edges(), [
("lf", "a"),
("a", "empty"),
("empty", "empty[$]"),
("empty[$]", "b"),
("b", "lf[$]"),
])
def test_ordering(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1',
requires=[],
provides=set(['a', 'b']))
test_2 = utils.ProvidesRequiresTask('test-2',
provides=['c'],
requires=['a', 'b'])
test_3 = utils.ProvidesRequiresTask('test-3',
provides=[],
requires=['c'])
wf.add(test_1, test_2, test_3)
self.assertTrue(wf.graph.has_edge(test_1, test_2))
self.assertTrue(wf.graph.has_edge(test_2, test_3))
self.assertEqual(3, len(wf.graph))
self.assertEqual([test_1], list(gu.get_no_predecessors(wf.graph)))
self.assertEqual([test_3], list(gu.get_no_successors(wf.graph)))
def test_many_empty_in_graph_flow(self):
flo = gf.Flow('root')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
flo.add(a)
b = lf.Flow('b')
b_0 = test_utils.ProvidesRequiresTask('b.0', provides=[], requires=[])
b_1 = lf.Flow('b.1')
b_2 = lf.Flow('b.2')
b_3 = test_utils.ProvidesRequiresTask('b.3', provides=[], requires=[])
b.add(b_0, b_1, b_2, b_3)
flo.add(b)
c = lf.Flow('c')
c_0 = lf.Flow('c.0')
c_1 = lf.Flow('c.1')
c_2 = lf.Flow('c.2')
c.add(c_0, c_1, c_2)
flo.add(c)
d = test_utils.ProvidesRequiresTask('d', provides=[], requires=[])
flo.add(d)
flo.link(b, d)
flo.link(a, d)
flo.link(c, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertTrue(g.has_edge('root', 'a'))
self.assertTrue(g.has_edge('root', 'b'))
self.assertTrue(g.has_edge('root', 'c'))
self.assertTrue(g.has_edge('b.0', 'b.1'))
self.assertTrue(g.has_edge('b.1[$]', 'b.2'))
self.assertTrue(g.has_edge('b.2[$]', 'b.3'))
self.assertTrue(g.has_edge('c.0[$]', 'c.1'))
self.assertTrue(g.has_edge('c.1[$]', 'c.2'))
self.assertTrue(g.has_edge('a', 'd'))
self.assertTrue(g.has_edge('b[$]', 'd'))
self.assertTrue(g.has_edge('c[$]', 'd'))
self.assertEqual(20, len(g))
def test_flow_add_order(self):
wf = lw.Flow("the-test-action")
wf.add(
utils.ProvidesRequiresTask('test-1',
requires=set(),
provides=['a', 'b']))
# This one should fail to add since it requires 'c'
self.assertRaises(
exc.InvalidStateException, wf.add,
utils.ProvidesRequiresTask('test-2', requires=['c'], provides=[]))
wf.add(
utils.ProvidesRequiresTask('test-2',
requires=['a', 'b'],
provides=['c', 'd']))
wf.add(
utils.ProvidesRequiresTask('test-3',
requires=['c', 'd'],
provides=[]))
wf.add(
utils.ProvidesRequiresTask('test-4', requires=[], provides=['d']))
wf.add(
utils.ProvidesRequiresTask('test-5', requires=[], provides=['d']))
wf.add(
utils.ProvidesRequiresTask('test-6', requires=['d'], provides=[]))
def test_empty_flow_in_nested_flow(self):
flow = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
flow2 = lf.Flow("lf-2")
c = test_utils.ProvidesRequiresTask('c', provides=[], requires=[])
d = test_utils.ProvidesRequiresTask('d', provides=[], requires=[])
empty_flow = gf.Flow("empty")
flow2.add(c, empty_flow, d)
flow.add(a, flow2, b)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flow).compile())
for u, v in [('lf', 'a'), ('a', 'lf-2'), ('lf-2', 'c'), ('c', 'empty'),
('empty[$]', 'd'), ('d', 'lf-2[$]'), ('lf-2[$]', 'b'),
('b', 'lf[$]')]:
self.assertTrue(g.has_edge(u, v))
def test_no_requires_provider(self):
flo = gw.Flow("test-flow")
flo.add(
utils.ProvidesRequiresTask('test1',
provides=['a', 'b'],
requires=['c', 'd']))
self.assertEquals(states.PENDING, flo.state)
self.assertRaises(excp.InvalidStateException, flo.run, {})
self.assertEquals(states.FAILURE, flo.state)
def test_empty_flow_in_nested_flow(self):
flow = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
flow2 = lf.Flow("lf-2")
c = test_utils.ProvidesRequiresTask('c', provides=[], requires=[])
d = test_utils.ProvidesRequiresTask('d', provides=[], requires=[])
empty_flow = gf.Flow("empty")
flow2.add(c, empty_flow, d)
flow.add(a, flow2, b)
compilation = compiler.PatternCompiler(flow).compile()
g = compilation.execution_graph
self.assertTrue(g.has_edge(a, c))
self.assertTrue(g.has_edge(c, d))
self.assertTrue(g.has_edge(d, b))
def test_shadow_linear(self):
r = lf.Flow("root")
customer = test_utils.ProvidesRequiresTask("customer",
provides=['dog'],
requires=[])
customer2 = test_utils.ProvidesRequiresTask("customer2",
provides=['dog'],
requires=[])
washer = test_utils.ProvidesRequiresTask("washer",
requires=['dog'],
provides=['wash'])
r.add(customer, customer2, washer)
c = compiler.PatternCompiler(r).compile()
# This order is guaranteed...
self.assertEqual(['customer2', 'customer'], _get_scopes(c, washer)[0])