def test_run_iterations_suspended_failure(self):
flow = lf.Flow("root")
sad_tasks = test_utils.make_many(1,
task_cls=test_utils.NastyFailingTask)
flow.add(*sad_tasks)
happy_tasks = test_utils.make_many(
1, task_cls=test_utils.TaskNoRequiresNoReturns, offset=1)
flow.add(*happy_tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
transitions = []
for prior_state, new_state in machine_runner.run_iter(builder.START):
transitions.append((new_state, memory.failures))
if new_state == st.ANALYZING:
runtime.storage.set_flow_state(st.SUSPENDED)
state, failures = transitions[-1]
self.assertEqual(st.SUSPENDED, state)
self.assertEqual([], failures)
self.assertEqual(st.PENDING,
runtime.storage.get_atom_state(happy_tasks[0].name))
self.assertEqual(st.FAILURE,
runtime.storage.get_atom_state(sad_tasks[0].name))
def test_graph_nested_graph(self):
a, b, c, d, e, f, g = test_utils.make_many(7)
flo = gf.Flow("test")
flo.add(a, b, c, d)
flo2 = gf.Flow('test2')
flo2.add(e, f, g)
flo.add(flo2)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(11, len(g))
self.assertItemsEqual(g.edges(), [
('test', 'a'),
('test', 'b'),
('test', 'c'),
('test', 'd'),
('test', 'test2'),
('test2', 'e'),
('test2', 'f'),
('test2', 'g'),
('e', 'test2[$]'),
('f', 'test2[$]'),
('g', 'test2[$]'),
('test2[$]', 'test[$]'),
('a', 'test[$]'),
('b', 'test[$]'),
('c', 'test[$]'),
('d', 'test[$]'),
])
def test_graph(self):
a, b, c, d = test_utils.make_many(4)
flo = gf.Flow("test")
flo.add(a, b, c, d)
compilation = compiler.PatternCompiler(flo).compile()
self.assertEqual(6, len(compilation.execution_graph))
self.assertEqual(8, compilation.execution_graph.number_of_edges())
def test_retry_in_unordered_flow_with_tasks(self):
c = retry.AlwaysRevert("c")
a, b = test_utils.make_many(2)
flo = uf.Flow("test", c).add(a, b)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(5, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'c', {
'invariant': True
}),
('c', 'a', {
'invariant': True,
'retry': True
}),
('c', 'b', {
'invariant': True,
'retry': True
}),
('b', 'test[$]', {
'invariant': True
}),
('a', 'test[$]', {
'invariant': True
}),
])
self.assertItemsEqual(['test'], list(g.no_predecessors_iter()))
self.assertItemsEqual(['test[$]'], list(g.no_successors_iter()))
self.assertIs(c, g.node['a']['retry'])
self.assertIs(c, g.node['b']['retry'])
def test_run_iterations(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(
1, task_cls=test_utils.TaskNoRequiresNoReturns)
flow.add(*tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
it = machine_runner.run_iter(builder.START)
prior_state, new_state = six.next(it)
self.assertEqual(st.RESUMING, new_state)
self.assertEqual(0, len(memory.failures))
prior_state, new_state = six.next(it)
self.assertEqual(st.SCHEDULING, new_state)
self.assertEqual(0, len(memory.failures))
prior_state, new_state = six.next(it)
self.assertEqual(st.WAITING, new_state)
self.assertEqual(0, len(memory.failures))
prior_state, new_state = six.next(it)
self.assertEqual(st.ANALYZING, new_state)
self.assertEqual(0, len(memory.failures))
prior_state, new_state = six.next(it)
self.assertEqual(builder.GAME_OVER, new_state)
self.assertEqual(0, len(memory.failures))
prior_state, new_state = six.next(it)
self.assertEqual(st.SUCCESS, new_state)
self.assertEqual(0, len(memory.failures))
self.assertRaises(StopIteration, six.next, it)
def test_graph_links(self):
a, b, c, d = test_utils.make_many(4)
flo = gf.Flow("test")
flo.add(a, b, c, d)
flo.link(a, b)
flo.link(b, c)
flo.link(c, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(6, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'a', {
'invariant': True
}),
('a', 'b', {
'manual': True
}),
('b', 'c', {
'manual': True
}),
('c', 'd', {
'manual': True
}),
('d', 'test[$]', {
'invariant': True
}),
])
self.assertItemsEqual(['test'], g.no_predecessors_iter())
self.assertItemsEqual(['test[$]'], g.no_successors_iter())
def test_retry_in_graph_flow_with_tasks(self):
r = retry.AlwaysRevert("r")
a, b, c = test_utils.make_many(3)
flo = gf.Flow("test", r).add(a, b, c).link(b, c)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertItemsEqual(g.edges(data=True), [
('test', 'r', {
'invariant': True
}),
('r', 'a', {
'invariant': True,
'retry': True
}),
('r', 'b', {
'invariant': True,
'retry': True
}),
('b', 'c', {
'manual': True
}),
('a', 'test[$]', {
'invariant': True
}),
('c', 'test[$]', {
'invariant': True
}),
])
self.assertItemsEqual(['test'], g.no_predecessors_iter())
self.assertItemsEqual(['test[$]'], g.no_successors_iter())
self.assertIs(r, g.node['a']['retry'])
self.assertIs(r, g.node['b']['retry'])
self.assertIs(r, g.node['c']['retry'])
def test_builder_automatic_process_failure(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(1, task_cls=test_utils.NastyFailingTask)
flow.add(*tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
transitions = list(machine_runner.run_iter(builder.START))
self.assertEqual((builder.GAME_OVER, st.FAILURE), transitions[-1])
self.assertEqual(1, len(memory.failures))
def test_builder_automatic_process_reverted(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(1, task_cls=test_utils.TaskWithFailure)
flow.add(*tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
transitions = list(machine_runner.run_iter(builder.START))
self.assertEqual((builder.GAME_OVER, st.REVERTED), transitions[-1])
self.assertEqual(st.REVERTED,
runtime.storage.get_atom_state(tasks[0].name))
def test_run_iterations_reverted(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(1, task_cls=test_utils.TaskWithFailure)
flow.add(*tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
transitions = list(machine_runner.run_iter(builder.START))
prior_state, new_state = transitions[-1]
self.assertEqual(st.REVERTED, new_state)
self.assertEqual([], memory.failures)
self.assertEqual(st.REVERTED,
runtime.storage.get_atom_state(tasks[0].name))
def test_builder_automatic_process(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(
1, task_cls=test_utils.TaskNoRequiresNoReturns)
flow.add(*tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
transitions = list(machine_runner.run_iter(builder.START))
self.assertEqual((builder.UNDEFINED, st.RESUMING), transitions[0])
self.assertEqual((builder.GAME_OVER, st.SUCCESS), transitions[-1])
self.assertEqual(st.SUCCESS,
runtime.storage.get_atom_state(tasks[0].name))
def test_run_iterations_failure(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(1, task_cls=test_utils.NastyFailingTask)
flow.add(*tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
transitions = list(machine_runner.run_iter(builder.START))
prior_state, new_state = transitions[-1]
self.assertEqual(st.FAILURE, new_state)
self.assertEqual(1, len(memory.failures))
failure = memory.failures[0]
self.assertTrue(failure.check(RuntimeError))
self.assertEqual(st.REVERT_FAILURE,
runtime.storage.get_atom_state(tasks[0].name))
def test_retries_hierarchy(self):
c1 = retry.AlwaysRevert("c1")
c2 = retry.AlwaysRevert("c2")
a, b, c, d = test_utils.make_many(4)
inner_flo = lf.Flow("test2", c2).add(b, c)
flo = lf.Flow("test", c1).add(a, inner_flo, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(10, len(g))
self.assertItemsEqual(g.edges(data=True), [
('test', 'c1', {
'invariant': True
}),
('c1', 'a', {
'invariant': True,
'retry': True
}),
('a', 'test2', {
'invariant': True
}),
('test2', 'c2', {
'invariant': True
}),
('c2', 'b', {
'invariant': True,
'retry': True
}),
('b', 'c', {
'invariant': True
}),
('c', 'test2[$]', {
'invariant': True
}),
('test2[$]', 'd', {
'invariant': True
}),
('d', 'test[$]', {
'invariant': True
}),
])
self.assertIs(c1, g.node['a']['retry'])
self.assertIs(c1, g.node['d']['retry'])
self.assertIs(c2, g.node['b']['retry'])
self.assertIs(c2, g.node['c']['retry'])
self.assertIs(c1, g.node['c2']['retry'])
self.assertIsNone(g.node['c1'].get('retry'))
def test_unordered_nested_in_linear(self):
a, b, c, d = test_utils.make_many(4)
inner_flo = uf.Flow('ut').add(b, c)
flo = lf.Flow('lt').add(a, inner_flo, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(8, len(g))
self.assertItemsEqual(g.edges(), [
('lt', 'a'),
('a', 'ut'),
('ut', 'b'),
('ut', 'c'),
('b', 'ut[$]'),
('c', 'ut[$]'),
('ut[$]', 'd'),
('d', 'lt[$]'),
])
def test_unordered(self):
a, b, c, d = test_utils.make_many(4)
flo = uf.Flow("test")
flo.add(a, b, c, d)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(6, len(g))
self.assertItemsEqual(g.edges(), [
('test', 'a'),
('test', 'b'),
('test', 'c'),
('test', 'd'),
('a', 'test[$]'),
('b', 'test[$]'),
('c', 'test[$]'),
('d', 'test[$]'),
])
self.assertEqual(set(['test']), set(g.no_predecessors_iter()))
def test_builder_expected_transition_occurrences(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(
10, task_cls=test_utils.TaskNoRequiresNoReturns)
flow.add(*tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
transitions = list(machine_runner.run_iter(builder.START))
occurrences = dict((t, transitions.count(t)) for t in transitions)
self.assertEqual(10, occurrences.get((st.SCHEDULING, st.WAITING)))
self.assertEqual(10, occurrences.get((st.WAITING, st.ANALYZING)))
self.assertEqual(9, occurrences.get((st.ANALYZING, st.SCHEDULING)))
self.assertEqual(1, occurrences.get((builder.GAME_OVER, st.SUCCESS)))
self.assertEqual(1, occurrences.get((builder.UNDEFINED, st.RESUMING)))
self.assertEqual(0, len(memory.next_up))
self.assertEqual(0, len(memory.not_done))
self.assertEqual(0, len(memory.failures))
def test_linear(self):
a, b, c, d = test_utils.make_many(4)
flo = lf.Flow("test")
flo.add(a, b, c)
inner_flo = lf.Flow("sub-test")
inner_flo.add(d)
flo.add(inner_flo)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(8, len(g))
order = list(g.topological_sort())
self.assertEqual(
['test', 'a', 'b', 'c', "sub-test", 'd', "sub-test[$]", 'test[$]'],
order)
self.assertTrue(g.has_edge('c', "sub-test"))
self.assertTrue(g.has_edge("sub-test", 'd'))
self.assertEqual({'invariant': True}, g.get_edge_data("sub-test", 'd'))
self.assertEqual(['test[$]'], list(g.no_successors_iter()))
self.assertEqual(['test'], list(g.no_predecessors_iter()))
def test_unordered_nested(self):
a, b, c, d = test_utils.make_many(4)
flo = uf.Flow("test")
flo.add(a, b)
flo2 = lf.Flow("test2")
flo2.add(c, d)
flo.add(flo2)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(8, len(g))
self.assertItemsEqual(g.edges(), [
('test', 'a'),
('test', 'b'),
('test', 'test2'),
('test2', 'c'),
('c', 'd'),
('d', 'test2[$]'),
('test2[$]', 'test[$]'),
('a', 'test[$]'),
('b', 'test[$]'),
])
def test_linear_nested(self):
a, b, c, d = test_utils.make_many(4)
flo = lf.Flow("test")
flo.add(a, b)
inner_flo = uf.Flow("test2")
inner_flo.add(c, d)
flo.add(inner_flo)
g = _replicate_graph_with_names(
compiler.PatternCompiler(flo).compile())
self.assertEqual(8, len(g))
sub_g = g.subgraph(['a', 'b'])
self.assertFalse(sub_g.has_edge('b', 'a'))
self.assertTrue(sub_g.has_edge('a', 'b'))
self.assertEqual({'invariant': True}, sub_g.get_edge_data("a", "b"))
sub_g = g.subgraph(['c', 'd'])
self.assertEqual(0, sub_g.number_of_edges())
# This ensures that c and d do not start executing until after b.
self.assertTrue(g.has_edge('b', 'test2'))
self.assertTrue(g.has_edge('test2', 'c'))
self.assertTrue(g.has_edge('test2', 'd'))
def test_invalid(self):
a, b, c = test_utils.make_many(3)
flo = lf.Flow("test")
flo.add(a, b, c)
flo.add(flo)
self.assertRaises(ValueError, compiler.PatternCompiler(flo).compile)
def test_builder_manual_process(self):
flow = lf.Flow("root")
tasks = test_utils.make_many(
1, task_cls=test_utils.TaskNoRequiresNoReturns)
flow.add(*tasks)
runtime, machine, memory, machine_runner = self._make_machine(
flow, initial_state=st.RUNNING)
self.assertRaises(excp.NotInitialized, machine.process_event, 'poke')
# Should now be pending...
self.assertEqual(st.PENDING,
runtime.storage.get_atom_state(tasks[0].name))
machine.initialize()
self.assertEqual(builder.UNDEFINED, machine.current_state)
self.assertFalse(machine.terminated)
self.assertRaises(excp.NotFound, machine.process_event, 'poke')
last_state = machine.current_state
reaction, terminal = machine.process_event(builder.START)
self.assertFalse(terminal)
self.assertIsNotNone(reaction)
self.assertEqual(st.RESUMING, machine.current_state)
self.assertRaises(excp.NotFound, machine.process_event, 'poke')
last_state = machine.current_state
cb, args, kwargs = reaction
next_event = cb(last_state, machine.current_state, builder.START,
*args, **kwargs)
reaction, terminal = machine.process_event(next_event)
self.assertFalse(terminal)
self.assertIsNotNone(reaction)
self.assertEqual(st.SCHEDULING, machine.current_state)
self.assertRaises(excp.NotFound, machine.process_event, 'poke')
last_state = machine.current_state
cb, args, kwargs = reaction
next_event = cb(last_state, machine.current_state, next_event, *args,
**kwargs)
reaction, terminal = machine.process_event(next_event)
self.assertFalse(terminal)
self.assertEqual(st.WAITING, machine.current_state)
self.assertRaises(excp.NotFound, machine.process_event, 'poke')
# Should now be running...
self.assertEqual(st.RUNNING,
runtime.storage.get_atom_state(tasks[0].name))
last_state = machine.current_state
cb, args, kwargs = reaction
next_event = cb(last_state, machine.current_state, next_event, *args,
**kwargs)
reaction, terminal = machine.process_event(next_event)
self.assertFalse(terminal)
self.assertIsNotNone(reaction)
self.assertEqual(st.ANALYZING, machine.current_state)
self.assertRaises(excp.NotFound, machine.process_event, 'poke')
last_state = machine.current_state
cb, args, kwargs = reaction
next_event = cb(last_state, machine.current_state, next_event, *args,
**kwargs)
reaction, terminal = machine.process_event(next_event)
self.assertFalse(terminal)
self.assertEqual(builder.GAME_OVER, machine.current_state)
# Should now be done...
self.assertEqual(st.SUCCESS,
runtime.storage.get_atom_state(tasks[0].name))
