def test_dependent(self):
r = gf.Flow("root")
customer = test_utils.ProvidesRequiresTask("customer",
provides=['dog'],
requires=[])
washer = test_utils.ProvidesRequiresTask("washer",
requires=['dog'],
provides=['wash'])
dryer = test_utils.ProvidesRequiresTask("dryer",
requires=['dog', 'wash'],
provides=['dry_dog'])
shaved = test_utils.ProvidesRequiresTask("shaver",
requires=['dry_dog'],
provides=['shaved_dog'])
happy_customer = test_utils.ProvidesRequiresTask(
"happy_customer", requires=['shaved_dog'], provides=['happiness'])
r.add(customer, washer, dryer, shaved, happy_customer)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, customer))
self.assertEqual([['washer', 'customer']], _get_scopes(c, dryer))
self.assertEqual([['shaver', 'dryer', 'washer', 'customer']],
_get_scopes(c, happy_customer))
def test_retry_in_graph_flow_with_tasks(self):
r = retry.AlwaysRevert("r")
a, b, c = test_utils.make_many(3)
flo = gf.Flow("test", r).add(a, b, c).link(b, c)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertItemsEqual(g.edges(data=True), [
('test', 'r', {
'invariant': True
}),
('r', 'a', {
'invariant': True,
'retry': True
}),
('r', 'b', {
'invariant': True,
'retry': True
}),
('b', 'c', {
'manual': True
}),
('a', 'test[$]', {
'invariant': True
}),
('c', 'test[$]', {
'invariant': True
}),
])
self.assertItemsEqual(['test'], g.no_predecessors_iter())
self.assertItemsEqual(['test[$]'], g.no_successors_iter())
self.assertIs(r, g.node['a']['retry'])
self.assertIs(r, g.node['b']['retry'])
self.assertIs(r, g.node['c']['retry'])
def test_shadow_graph(self):
r = gf.Flow("root")
customer = test_utils.ProvidesRequiresTask("customer",
provides=['dog'],
requires=[])
customer2 = test_utils.ProvidesRequiresTask("customer2",
provides=['dog'],
requires=[])
washer = test_utils.ProvidesRequiresTask("washer",
requires=['dog'],
provides=['wash'])
r.add(customer, washer)
r.add(customer2, resolve_requires=False)
r.link(customer2, washer)
c = compiler.PatternCompiler(r).compile()
# The order currently is *not* guaranteed to be 'customer' before
# 'customer2' or the reverse, since either can occur before the
# washer; since *either* is a valid topological ordering of the
# dependencies...
#
# This may be different after/if the following is resolved:
#
# https://github.com/networkx/networkx/issues/1181 (and a few others)
self.assertEqual(set(['customer', 'customer2']),
set(_get_scopes(c, washer)[0]))
self.assertEqual([], _get_scopes(c, customer2))
self.assertEqual([], _get_scopes(c, customer))
def test_retry_in_unordered_flow_with_tasks(self):
c = retry.AlwaysRevert("c")
a, b = test_utils.make_many(2)
flo = uf.Flow("test", c).add(a, b)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(5, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'c', {
'invariant': True
}),
('c', 'a', {
'invariant': True,
'retry': True
}),
('c', 'b', {
'invariant': True,
'retry': True
}),
('b', 'test[$]', {
'invariant': True
}),
('a', 'test[$]', {
'invariant': True
}),
])
self.assertItemsEqual(['test'], list(g.no_predecessors_iter()))
self.assertItemsEqual(['test[$]'], list(g.no_successors_iter()))
self.assertIs(c, g.node['a']['retry'])
self.assertIs(c, g.node['b']['retry'])
def test_graph_linear_scope(self):
r = gf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r_2 = test_utils.TaskOneReturn("root.2")
r.add(r_1, r_2)
r.link(r_1, r_2)
s = lf.Flow("subroot")
s_1 = test_utils.TaskOneReturn("subroot.1")
s_2 = test_utils.TaskOneReturn("subroot.2")
s.add(s_1, s_2)
r.add(s)
t = gf.Flow("subroot2")
t_1 = test_utils.TaskOneReturn("subroot2.1")
t_2 = test_utils.TaskOneReturn("subroot2.2")
t.add(t_1, t_2)
t.link(t_1, t_2)
r.add(t)
r.link(s, t)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, r_1))
self.assertEqual([['root.1']], _get_scopes(c, r_2))
self.assertEqual([], _get_scopes(c, s_1))
self.assertEqual([['subroot.1']], _get_scopes(c, s_2))
self.assertEqual([[], ['subroot.2', 'subroot.1']], _get_scopes(c, t_1))
self.assertEqual([["subroot2.1"], ['subroot.2', 'subroot.1']],
_get_scopes(c, t_2))
def test_nested(self):
r = gf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r_2 = test_utils.TaskOneReturn("root.2")
r.add(r_1, r_2)
r.link(r_1, r_2)
subroot = gf.Flow("subroot")
subroot_r_1 = test_utils.TaskOneReturn("subroot.1")
subroot_r_2 = test_utils.TaskOneReturn("subroot.2")
subroot.add(subroot_r_1, subroot_r_2)
subroot.link(subroot_r_1, subroot_r_2)
r.add(subroot)
r_3 = test_utils.TaskOneReturn("root.3")
r.add(r_3)
r.link(r_2, r_3)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, r_1))
self.assertEqual([['root.1']], _get_scopes(c, r_2))
self.assertEqual([['root.2', 'root.1']], _get_scopes(c, r_3))
self.assertEqual([], _get_scopes(c, subroot_r_1))
self.assertEqual([['subroot.1']], _get_scopes(c, subroot_r_2))
def test_graph_nested_requires(self):
a = test_utils.ProvidesRequiresTask('a', provides=['x'], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
c = test_utils.ProvidesRequiresTask('c', provides=[], requires=['x'])
inner_flo = lf.Flow("test2").add(b, c)
flo = gf.Flow("test").add(a, inner_flo)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(7, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'a', {
'invariant': True
}),
('test2', 'b', {
'invariant': True
}),
('a', 'test2', {
'reasons': set(['x'])
}),
('b', 'c', {
'invariant': True
}),
('c', 'test2[$]', {
'invariant': True
}),
('test2[$]', 'test[$]', {
'invariant': True
}),
])
self.assertItemsEqual(['test'], list(g.no_predecessors_iter()))
self.assertItemsEqual(['test[$]'], list(g.no_successors_iter()))
def test_graph_links(self):
a, b, c, d = test_utils.make_many(4)
flo = gf.Flow("test")
flo.add(a, b, c, d)
flo.link(a, b)
flo.link(b, c)
flo.link(c, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(6, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'a', {
'invariant': True
}),
('a', 'b', {
'manual': True
}),
('b', 'c', {
'manual': True
}),
('c', 'd', {
'manual': True
}),
('d', 'test[$]', {
'invariant': True
}),
])
self.assertItemsEqual(['test'], g.no_predecessors_iter())
self.assertItemsEqual(['test[$]'], g.no_successors_iter())
def test_graph_nested_graph(self):
a, b, c, d, e, f, g = test_utils.make_many(7)
flo = gf.Flow("test")
flo.add(a, b, c, d)
flo2 = gf.Flow('test2')
flo2.add(e, f, g)
flo.add(flo2)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(11, len(g))
self.assertItemsEqual(g.edges(), [
('test', 'a'),
('test', 'b'),
('test', 'c'),
('test', 'd'),
('test', 'test2'),
('test2', 'e'),
('test2', 'f'),
('test2', 'g'),
('e', 'test2[$]'),
('f', 'test2[$]'),
('g', 'test2[$]'),
('test2[$]', 'test[$]'),
('a', 'test[$]'),
('b', 'test[$]'),
('c', 'test[$]'),
('d', 'test[$]'),
])
def test_graph(self):
a, b, c, d = test_utils.make_many(4)
flo = gf.Flow("test")
flo.add(a, b, c, d)
compilation = compiler.PatternCompiler(flo).compile()
self.assertEqual(6, len(compilation.execution_graph))
self.assertEqual(8, compilation.execution_graph.number_of_edges())
def test_retry_in_nested_flows(self):
c1 = retry.AlwaysRevert("c1")
c2 = retry.AlwaysRevert("c2")
inner_flo = lf.Flow("test2", c2)
flo = lf.Flow("test", c1).add(inner_flo)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(6, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'c1', {
'invariant': True
}),
('c1', 'test2', {
'invariant': True,
'retry': True
}),
('test2', 'c2', {
'invariant': True
}),
('c2', 'test2[$]', {
'invariant': True
}),
('test2[$]', 'test[$]', {
'invariant': True
}),
])
self.assertIs(c1, g.node['c2']['retry'])
self.assertItemsEqual(['test'], list(g.no_predecessors_iter()))
self.assertItemsEqual(['test[$]'], list(g.no_successors_iter()))
def test_unknown(self):
r = lf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r.add(r_1)
r_2 = test_utils.TaskOneReturn("root.2")
c = compiler.PatternCompiler(r).compile()
self.assertRaises(ValueError, _get_scopes, c, r_2)
def test_no_visible(self):
r = uf.Flow("root")
atoms = []
for i in range(0, 10):
atoms.append(test_utils.TaskOneReturn("root.%s" % i))
r.add(*atoms)
c = compiler.PatternCompiler(r).compile()
for a in atoms:
self.assertEqual([], _get_scopes(c, a))
def test_nested_prior_linear(self):
r = lf.Flow("root")
r.add(test_utils.TaskOneReturn("root.1"),
test_utils.TaskOneReturn("root.2"))
sub_r = lf.Flow("subroot")
sub_r_1 = test_utils.TaskOneReturn("subroot.1")
sub_r.add(sub_r_1)
r.add(sub_r)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([[], ['root.2', 'root.1']], _get_scopes(c, sub_r_1))
def test_empty(self):
r = lf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r.add(r_1)
c = compiler.PatternCompiler(r).compile()
self.assertIn(r_1, c.execution_graph)
self.assertIsNotNone(c.hierarchy.find(r_1))
walker = sc.ScopeWalker(c, r_1)
scopes = list(walker)
self.assertEqual([], scopes)
def test_single_prior_linear(self):
r = lf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r_2 = test_utils.TaskOneReturn("root.2")
r.add(r_1, r_2)
c = compiler.PatternCompiler(r).compile()
for a in r:
self.assertIn(a, c.execution_graph)
self.assertIsNotNone(c.hierarchy.find(a))
self.assertEqual([], _get_scopes(c, r_1))
self.assertEqual([['root.1']], _get_scopes(c, r_2))
def test_empty_flow_in_graph_flow_linkage(self):
flow = gf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
empty_flow = lf.Flow("empty")
flow.add(a, empty_flow, b)
flow.link(a, b)
compilation = compiler.PatternCompiler(flow).compile()
g = compilation.execution_graph
self.assertTrue(g.has_edge(a, b))
self.assertTrue(g.has_edge(flow, a))
self.assertTrue(g.has_edge(flow, empty_flow))
def test_empty_flow_in_linear_flow(self):
flo = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
empty_flo = gf.Flow("empty")
flo.add(a, empty_flo, b)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertItemsEqual(g.edges(), [
("lf", "a"),
("a", "empty"),
("empty", "empty[$]"),
("empty[$]", "b"),
("b", "lf[$]"),
])
def test_retries_hierarchy(self):
c1 = retry.AlwaysRevert("c1")
c2 = retry.AlwaysRevert("c2")
a, b, c, d = test_utils.make_many(4)
inner_flo = lf.Flow("test2", c2).add(b, c)
flo = lf.Flow("test", c1).add(a, inner_flo, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(10, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'c1', {
'invariant': True
}),
('c1', 'a', {
'invariant': True,
'retry': True
}),
('a', 'test2', {
'invariant': True
}),
('test2', 'c2', {
'invariant': True
}),
('c2', 'b', {
'invariant': True,
'retry': True
}),
('b', 'c', {
'invariant': True
}),
('c', 'test2[$]', {
'invariant': True
}),
('test2[$]', 'd', {
'invariant': True
}),
('d', 'test[$]', {
'invariant': True
}),
])
self.assertIs(c1, g.node['a']['retry'])
self.assertIs(c1, g.node['d']['retry'])
self.assertIs(c2, g.node['b']['retry'])
self.assertIs(c2, g.node['c']['retry'])
self.assertIs(c1, g.node['c2']['retry'])
self.assertIsNone(g.node['c1'].get('retry'))
def test_many_empty_in_graph_flow(self):
flo = gf.Flow('root')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
flo.add(a)
b = lf.Flow('b')
b_0 = test_utils.ProvidesRequiresTask('b.0', provides=[], requires=[])
b_1 = lf.Flow('b.1')
b_2 = lf.Flow('b.2')
b_3 = test_utils.ProvidesRequiresTask('b.3', provides=[], requires=[])
b.add(b_0, b_1, b_2, b_3)
flo.add(b)
c = lf.Flow('c')
c_0 = lf.Flow('c.0')
c_1 = lf.Flow('c.1')
c_2 = lf.Flow('c.2')
c.add(c_0, c_1, c_2)
flo.add(c)
d = test_utils.ProvidesRequiresTask('d', provides=[], requires=[])
flo.add(d)
flo.link(b, d)
flo.link(a, d)
flo.link(c, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertTrue(g.has_edge('root', 'a'))
self.assertTrue(g.has_edge('root', 'b'))
self.assertTrue(g.has_edge('root', 'c'))
self.assertTrue(g.has_edge('b.0', 'b.1'))
self.assertTrue(g.has_edge('b.1[$]', 'b.2'))
self.assertTrue(g.has_edge('b.2[$]', 'b.3'))
self.assertTrue(g.has_edge('c.0[$]', 'c.1'))
self.assertTrue(g.has_edge('c.1[$]', 'c.2'))
self.assertTrue(g.has_edge('a', 'd'))
self.assertTrue(g.has_edge('b[$]', 'd'))
self.assertTrue(g.has_edge('c[$]', 'd'))
self.assertEqual(20, len(g))
def __init__(self, flow, flow_detail, backend, options):
super(ActionEngine, self).__init__(flow, flow_detail, backend, options)
self._runtime = None
self._compiled = False
self._compilation = None
self._compiler = compiler.PatternCompiler(flow)
self._lock = threading.RLock()
self._storage_ensured = False
self._validated = False
# Retries are not *currently* executed out of the engines process
# or thread (this could change in the future if we desire it to).
self._retry_executor = executor.SerialRetryExecutor()
self._inject_transient = strutils.bool_from_string(
self._options.get('inject_transient', True))
self._gather_statistics = strutils.bool_from_string(
self._options.get('gather_statistics', True))
self._statistics = {}
def test_empty_flow_in_nested_flow(self):
flow = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=[], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=[])
flow2 = lf.Flow("lf-2")
c = test_utils.ProvidesRequiresTask('c', provides=[], requires=[])
d = test_utils.ProvidesRequiresTask('d', provides=[], requires=[])
empty_flow = gf.Flow("empty")
flow2.add(c, empty_flow, d)
flow.add(a, flow2, b)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flow).compile())
for u, v in [('lf', 'a'), ('a', 'lf-2'), ('lf-2', 'c'), ('c', 'empty'),
('empty[$]', 'd'), ('d', 'lf-2[$]'), ('lf-2[$]', 'b'),
('b', 'lf[$]')]:
self.assertTrue(g.has_edge(u, v))
def test_unordered_nested_in_linear(self):
a, b, c, d = test_utils.make_many(4)
inner_flo = uf.Flow('ut').add(b, c)
flo = lf.Flow('lt').add(a, inner_flo, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(8, len(g))
self.assertItemsEqual(g.edges(), [
('lt', 'a'),
('a', 'ut'),
('ut', 'b'),
('ut', 'c'),
('b', 'ut[$]'),
('c', 'ut[$]'),
('ut[$]', 'd'),
('d', 'lt[$]'),
])
def test_shadow_linear(self):
r = lf.Flow("root")
customer = test_utils.ProvidesRequiresTask("customer",
provides=['dog'],
requires=[])
customer2 = test_utils.ProvidesRequiresTask("customer2",
provides=['dog'],
requires=[])
washer = test_utils.ProvidesRequiresTask("washer",
requires=['dog'],
provides=['wash'])
r.add(customer, customer2, washer)
c = compiler.PatternCompiler(r).compile()
# This order is guaranteed...
self.assertEqual(['customer2', 'customer'], _get_scopes(c, washer)[0])
def test_empty_flow_in_graph_flow(self):
flow = lf.Flow('lf')
a = test_utils.ProvidesRequiresTask('a', provides=['a'], requires=[])
b = test_utils.ProvidesRequiresTask('b', provides=[], requires=['a'])
empty_flow = lf.Flow("empty")
flow.add(a, empty_flow, b)
compilation = compiler.PatternCompiler(flow).compile()
g = compilation.execution_graph
self.assertTrue(g.has_edge(flow, a))
self.assertTrue(g.has_edge(a, empty_flow))
empty_flow_successors = list(g.successors(empty_flow))
self.assertEqual(1, len(empty_flow_successors))
empty_flow_terminal = empty_flow_successors[0]
self.assertIs(empty_flow, empty_flow_terminal.flow)
self.assertEqual(compiler.FLOW_END,
g.node[empty_flow_terminal]['kind'])
self.assertTrue(g.has_edge(empty_flow_terminal, b))
def _make_runtime(self, flow, initial_state=None):
compilation = compiler.PatternCompiler(flow).compile()
flow_detail = pu.create_flow_detail(flow)
store = storage.Storage(flow_detail)
nodes_iter = compilation.execution_graph.nodes_iter(data=True)
for node, node_attrs in nodes_iter:
if node_attrs['kind'] in ('task', 'retry'):
store.ensure_atom(node)
if initial_state:
store.set_flow_state(initial_state)
atom_notifier = notifier.Notifier()
task_executor = executor.SerialTaskExecutor()
retry_executor = executor.SerialRetryExecutor()
task_executor.start()
self.addCleanup(task_executor.stop)
r = runtime.Runtime(compilation, store, atom_notifier, task_executor,
retry_executor)
r.compile()
return r
def test_unordered(self):
a, b, c, d = test_utils.make_many(4)
flo = uf.Flow("test")
flo.add(a, b, c, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(6, len(g))
self.assertItemsEqual(g.edges(), [
('test', 'a'),
('test', 'b'),
('test', 'c'),
('test', 'd'),
('a', 'test[$]'),
('b', 'test[$]'),
('c', 'test[$]'),
('d', 'test[$]'),
])
self.assertEqual(set(['test']), set(g.no_predecessors_iter()))
def test_nested_prior_linear_begin_middle_end(self):
r = lf.Flow("root")
begin_r = test_utils.TaskOneReturn("root.1")
r.add(begin_r, test_utils.TaskOneReturn("root.2"))
middle_r = test_utils.TaskOneReturn("root.3")
r.add(middle_r)
sub_r = lf.Flow("subroot")
sub_r.add(test_utils.TaskOneReturn("subroot.1"),
test_utils.TaskOneReturn("subroot.2"))
r.add(sub_r)
end_r = test_utils.TaskOneReturn("root.4")
r.add(end_r)
c = compiler.PatternCompiler(r).compile()
self.assertEqual([], _get_scopes(c, begin_r))
self.assertEqual([['root.2', 'root.1']], _get_scopes(c, middle_r))
self.assertEqual(
[['subroot.2', 'subroot.1', 'root.3', 'root.2', 'root.1']],
_get_scopes(c, end_r))
def test_linear(self):
a, b, c, d = test_utils.make_many(4)
flo = lf.Flow("test")
flo.add(a, b, c)
inner_flo = lf.Flow("sub-test")
inner_flo.add(d)
flo.add(inner_flo)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(8, len(g))
order = list(g.topological_sort())
self.assertEqual(
['test', 'a', 'b', 'c', "sub-test", 'd', "sub-test[$]", 'test[$]'],
order)
self.assertTrue(g.has_edge('c', "sub-test"))
self.assertTrue(g.has_edge("sub-test", 'd'))
self.assertEqual({'invariant': True}, g.get_edge_data("sub-test", 'd'))
self.assertEqual(['test[$]'], list(g.no_successors_iter()))
self.assertEqual(['test'], list(g.no_predecessors_iter()))
def test_unordered_nested(self):
a, b, c, d = test_utils.make_many(4)
flo = uf.Flow("test")
flo.add(a, b)
flo2 = lf.Flow("test2")
flo2.add(c, d)
flo.add(flo2)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(8, len(g))
self.assertItemsEqual(g.edges(), [
('test', 'a'),
('test', 'b'),
('test', 'test2'),
('test2', 'c'),
('c', 'd'),
('d', 'test2[$]'),
('test2[$]', 'test[$]'),
('a', 'test[$]'),
('b', 'test[$]'),
])
