def test_result_cache_issue(self, cache):
#
# flow1:
#
# Task1
#
# Checks that if there is an issue with retrieving task result from result cache in condition, dispatcher
# will use directly storage.
edge_table = {
'flow1': [{'from': [], 'to': ['Task1'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': 'Task2', 'condition': lambda db, _: db.get('Task1')}]
}
self.init(
edge_table,
storage_mapping={'Storage1': self.MyStorage()},
task2storage_mapping={'Task1': 'Storage1'},
storage2storage_cache={'Storage1': cache}
)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
def test_single_failure_flow_fallback_start(self):
#
# flow1:
#
# Task1 X .... Task3
# | |
# | |
# Task2 Task4
#
# Note:
# Task1 will fail
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true},
{'from': ['Task3'], 'to': ['Task4'], 'condition': self.cond_true}]
}
failures = {
'flow1': {'Task1': {'next:': {}, 'fallback': [['Task3']],
'conditions': [self.cond_true],
'condition_strs': ['cond_true']
}}
}
self.init(edge_table, failures=failures)
node_args = {'foo': 'bar'}
system_state = SystemState(id(self), 'flow1', node_args=node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args == node_args
assert 'Task1' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
# Task1 has failed
task1 = self.get_task('Task1')
self.set_failed(task1)
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args == node_args
assert 'Task1' in self.instantiated_tasks
assert 'Task3' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
task3 = self.get_task('Task3')
assert task3.node_args == node_args
def test_eager_failures(self):
#
# flow1:
# Task1 ------
# | |
# Task2 x Task3
# |
# Task4
#
# Note:
# Task2 marked as eager failure node in the flow, dispatcher will not wait for Task3 to finish
edge_table = {'flow1': [
{'from': [], 'to': ['Task1'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true},
{'from': ['Task2'], 'to': ['Task4'], 'condition': self.cond_true}
]}
self.init(edge_table, eager_failures=({'flow1': ['Task2']}))
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert 'Task1' in self.instantiated_tasks
assert retry is not None
assert system_state.node_args is None
task1 = self.get_task('Task1')
self.set_finished(task1)
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' in self.instantiated_tasks
assert retry is not None
assert system_state.node_args is None
# Manually append new Task1 instance to simplify this test
task2 = self.get_task('Task2')
self.set_failed(task2, ValueError("Some exception raised"))
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
with pytest.raises(FlowError) as flow_error:
system_state.update()
reported_state = flow_error.value.state
assert 'active_nodes' in reported_state.keys()
assert 'finished_nodes' in reported_state.keys()
assert 'failed_nodes' in reported_state.keys()
assert {node['name'] for node in reported_state['active_nodes']} == {'Task3'}
assert set(reported_state['finished_nodes'].keys()) == {'Task1'}
assert set(reported_state['failed_nodes'].keys()) == {'Task2'}
def simulate_flow2(flow2, f2_already_instantiated_tasks):
f2_system_state = flow2.get_initial_system_state()
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert f2_already_instantiated_tasks | {'TaskF2'} == set(
self.instantiated_tasks)
self.set_finished(self.get_task('TaskF2'))
f2_system_state = SystemState(id(self),
'flow2',
state=f2_state_dict,
node_args=f2_system_state.node_args,
selective=f2_selective)
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert f2_already_instantiated_tasks | {'TaskF2', 'Task4'} == set(
self.instantiated_tasks)
assert 'flow3' not in self.instantiated_flows
self.set_finished(self.get_task('Task4'))
f2_system_state = SystemState(id(self),
'flow2',
state=f2_state_dict,
node_args=f2_system_state.node_args,
selective=f2_selective)
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert 'flow3' in self.instantiated_flows
simulate_flow3(self.get_flow('flow3'),
set(self.instantiated_tasks))
f2_system_state = SystemState(id(self),
'flow2',
state=f2_state_dict,
node_args=f2_system_state.node_args,
selective=f2_selective)
f2_retry = f2_system_state.update()
assert f2_retry is None
self.set_finished(flow2, None)
# let's clean after self so we inspect purely flow1 in callee
self.remove_all_tasks_by_name('TaskF2')
self.remove_all_tasks_by_name('Task4')
self.remove_all_flows_by_name('flow2')
def test_single_failure_flow_fallback_true(self):
#
# flow1:
#
# Task1 X .... True
# |
# |
# Task2
#
# Note:
# Task1 will fail
#
edge_table = {
'flow1': [{
'from': ['Task1'],
'to': ['Task2'],
'condition': self.cond_true
}, {
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
failures = {'flow1': {'Task1': {'next:': {}, 'fallback': True}}}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
# Task1 has failed
task1 = self.get_task('Task1')
self.set_failed(task1, KeyError("Some exception raised"))
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
def test_task2flow(self):
#
# flow1:
#
# Task1
# |
# |
# flow2
#
# Note:
# Result of Task1 is not propagated to flow2 as even node_args_from_first is set - but propagate_node_args
# is not set
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}],
'flow2': []
}
self.init(edge_table,
node_args_from_first={'flow1': True},
task2storage_mapping={'Task1': 'Storage1'},
storage_mapping={'Storage1': _MyStorage()})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
# Run without change at first
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert system_state.node_args is None
assert retry is not None
assert 'Task1' in self.instantiated_tasks
# Task1 has finished
task1 = self.get_task('Task1')
task1_result = "propagated result of Task1"
self.set_finished(task1, task1_result)
_MyStorage.result = task1_result
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
flow2 = self.get_flow('flow2')
assert 'flow2' in self.instantiated_flows
assert flow2.node_args is None
assert retry is not None
def test_task2flow(self):
#
# flow1:
#
# Task1
# |
# |
# flow2
#
# Note:
# Result of Task1 is not propagated to flow2 as even node_args_from_first is set - but propagate_node_args
# is not set
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}],
'flow2': []
}
self.init(edge_table,
node_args_from_first={'flow1': True},
task2storage_mapping={'Task1': 'Storage1'},
storage_mapping={'Storage1': _MyStorage()})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
# Run without change at first
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert system_state.node_args is None
assert retry is not None
assert 'Task1' in self.instantiated_tasks
# Task1 has finished
task1 = self.get_task('Task1')
task1_result = "propagated result of Task1"
self.set_finished(task1, task1_result)
_MyStorage.result = task1_result
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
flow2 = self.get_flow('flow2')
assert 'flow2' in self.instantiated_flows
assert flow2.node_args is None
assert retry is not None
def test_foreach_propagate_result(self):
#
# flow1:
#
# Task1
# |
# |
# ---------------------
# | | |
# | | |
# flow2 flow2 ... flow2
#
# Note:
# There will be spawned _FOREACH_COUNT flow2, arguments are passed from foreach function
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true,
'foreach': lambda x, y: range(_FOREACH_COUNT), 'foreach_propagate_result': True},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}],
'flow2': []
}
self.init(edge_table)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
# Task1 has finished
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'flow2' in self.instantiated_flows
tasks_state_dict = [node for node in state_dict['active_nodes'] if node['name'] == 'flow2']
assert len(tasks_state_dict) == _FOREACH_COUNT
# Inspect node_args as we set propagate_result for foreach
all_flow_args = [flow.node_args for flow in self.get_all_flows('flow2')]
assert all_flow_args == list(range(_FOREACH_COUNT))
def test_nowait_in_flow(self):
#
# flow1:
#
# Task1
# |
# ----------
# | |
# Task2 Task3
#
# Note:
# Task3 finishes before Task2 and Task3 is marked as nowait
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task2', 'Task3'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
nowait_nodes = {'flow1': ['Task3']}
self.init(edge_table, nowait_nodes=nowait_nodes)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert 0 in state_dict['waiting_edges']
assert len(state_dict.get('finished_nodes')) == 0
assert len(state_dict.get('active_nodes')) == 1
# Task1 has finished
task1 = self.get_task('Task1')
self.set_finished(task1, 1)
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert 0 in state_dict['waiting_edges']
assert len(state_dict.get('finished_nodes')) == 1
assert len(state_dict.get('active_nodes')) == 1
assert 'Task3' not in state_dict.get('active_nodes')
assert 'Task2' in state_dict['active_nodes'][0]['name']
def test_singe_failure_flow_fallback(self):
#
# flow1:
#
# flow2 X ... Task2
# |
# |
# Task1
#
# Note:
# flow2 will fail
#
edge_table = {
'flow1': [{'from': ['flow2'], 'to': ['Task1'], 'condition': self.cond_true},
{'from': [], 'to': ['flow2'], 'condition': self.cond_true}],
'flow2': []
}
failures = {
'flow1': {'flow2': {'next:': {}, 'fallback': [['Task2']],
'conditions': [self.cond_true],
'condition_strs': ['cond_true']
}}
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'flow2' in self.instantiated_flows
assert 'Task1' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
# flow2 has failed
flow2 = self.get_flow('flow2')
flow_info = {'finished_nodes': {'Task1': ['id_task1']}, 'failed_nodes': {'Task2': ['id_task21']}}
self.set_failed(flow2, FlowError(flow_info))
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'flow2' in self.instantiated_flows
assert 'Task2' in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
def test_multiple_failured_tasks(self):
#
# flow1:
# Task0 Task1 X Task1 X
#
# Note:
# flow1 will have three tasks, Task1 will be instantiated twice. We will provide fallback for Task1
edge_table = {'flow1': [{'from': [], 'to': ['Task0', 'Task1'], 'condition': self.cond_true}]}
failures = {
'flow1': {'Task1': {'next': {}, 'fallback': [True],
'conditions': [self.cond_true],
'condition_strs': ['cond_true']
}},
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert 'Task0' in self.instantiated_tasks
assert 'Task1' in self.instantiated_tasks
assert retry is not None
assert system_state.node_args is None
# Manually append new Task1 instance to simplify this test
task1 = self.get_task('Task1')
self.set_failed(task1, ValueError("Some exception raised"))
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert 'Task1' in state_dict.get('failed_nodes')
assert retry is not None
state_dict['failed_nodes']['Task1'].append('<injected-task1-id>')
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
task0 = self.get_task('Task0')
self.set_finished(task0)
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
assert retry is None
def test_foreach_basic(self):
#
# flow1:
#
# Task1
# |
# |
# ---------------------
# | | |
# | | |
# Task2 Task2 ... Task2
#
# Note:
# There will be spawned _FOREACH_COUNT Task2
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true,
'foreach': lambda x, y: range(_FOREACH_COUNT), 'foreach_propagate_result': False},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
# Task1 has finished
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert len(self.get_all_tasks('Task2')) == _FOREACH_COUNT
tasks_state_dict = [node for node in state_dict['active_nodes'] if node['name'] == 'Task2']
assert len(tasks_state_dict) == _FOREACH_COUNT
def test_nowait_flow(self):
#
# flow1:
#
# flow2
#
# Note:
# flow2 is marked as nowait node
#
edge_table = {
'flow1': [{'from': [], 'to': ['flow2'], 'condition': self.cond_true}],
'flow2': []
}
nowait_nodes = {'flow1': ['flow2']}
self.init(edge_table, nowait_nodes=nowait_nodes)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is None
assert system_state.node_args is None
assert 'flow2' in self.instantiated_flows
assert 'flow2' not in state_dict.get('active_nodes')
assert len(state_dict.get('waiting_edges')) == 0
def test_nowait_flow(self):
#
# flow1:
#
# flow2
#
# Note:
# flow2 is marked as nowait node
#
edge_table = {
'flow1': [{
'from': [],
'to': ['flow2'],
'condition': self.cond_true
}],
'flow2': []
}
nowait_nodes = {'flow1': ['flow2']}
self.init(edge_table, nowait_nodes=nowait_nodes)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is None
assert system_state.node_args is None
assert 'flow2' in self.instantiated_flows
assert 'flow2' not in state_dict.get('active_nodes')
assert len(state_dict.get('waiting_edges')) == 0
def test_retrieve_issue(self):
#
# flow1:
#
# Task1
#
# Checks that if there is an issue with retrieving task result from task storage in condition, dispatcher
# retries flow
edge_table = {
'flow1': [{'from': [], 'to': ['Task1'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': 'Task2', 'condition': lambda db, _: db.get('Task1')}]
}
self.init(
edge_table,
storage_mapping={'Storage1': self.MyStorage()},
task2storage_mapping={'Task1': 'Storage1'}
)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
with pytest.raises(DispatcherRetry) as exc_info:
SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args).update()
assert exc_info.value.keep_state is True
assert exc_info.value.adjust_retry_count is False
def test_async_result_cache_issue(self, cache):
#
# flow1:
#
# Task1
#
# This test ensures that async result cache issues do not affect flow behaviour.
#
edge_table = {
'flow1': [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}],
}
self.init(edge_table, async_result_cache={'flow1': cache})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
# Nothing should be raised here even though the cache fails on get(), flow should continue
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
assert retry is None
def test_async_result_cache_issue(self, cache):
#
# flow1:
#
# Task1
#
# This test ensures that async result cache issues do not affect flow behaviour.
#
edge_table = {
'flow1': [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}],
}
self.init(edge_table, async_result_cache={'flow1': cache})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
# Nothing should be raised here even though the cache fails on get(), flow should continue
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
assert retry is None
def test_foreach_start(self):
#
# flow1:
#
# | | |
# Task1 Task1 ... Task1
#
# Note:
# There will be spawned _FOREACH_COUNT Task2
#
edge_table = {
'flow1': [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true,
'foreach': lambda x, y: range(_FOREACH_COUNT),
'foreach_propagate_result': False
}]
}
self.init(edge_table)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert len(self.get_all_tasks('Task1')) == _FOREACH_COUNT
tasks_state_dict = [
node for node in state_dict['active_nodes']
if node['name'] == 'Task1'
]
assert len(tasks_state_dict) == _FOREACH_COUNT
def test_multiple_failures_from_subflow(self):
#
# flow2:
# Task0 Task1 X Task1 X
#
# flow1:
#
# flow2X..... Task3
#
# Note:
# flow2 will fail with two tasks of type Task1. flow1 defines fallback as True, so
# it should recover from failure
edge_table = {
'flow1': [{
'from': [],
'to': ['flow2'],
'condition': self.cond_true
}],
'flow2': []
} # flow2 handled manually
failures = {
'flow1': {
'flow2': {
'next': {},
'fallback': [True],
'conditions': [self.cond_true],
'condition_strs': ['cond_true']
}
},
'flow2': {}
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert 'flow2' in self.instantiated_flows
assert retry is not None
assert system_state.node_args is None
flow_info = {
'finished_nodes': {
'Task0': ['<id-tak0_0>']
},
'failed_nodes': {
'Task2': ['<id-task1_0>', '<id-task1_1>']
}
}
flow2 = self.get_flow('flow2')
self.set_failed(flow2, FlowError(flow_info))
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
assert retry is None
def test_task2tasks(self):
#
# flow1:
#
# Task1
# |
# --------
# | |
# Task2 Task3
#
# Note:
# Result of Task1 is propagated to flow1 as node_args
#
edge_table = {
'flow1': [{
'from': ['Task1'],
'to': ['Task2', 'Task3'],
'condition': self.cond_true
}, {
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
self.init(edge_table,
node_args_from_first={'flow1': True},
task2storage_mapping={'Task1': 'Storage1'},
storage_mapping={'Storage1': _MyStorage()})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
# Task1 has finished
task1 = self.get_task('Task1')
task1_result = "propagated result of Task1"
self.set_finished(task1, task1_result)
_MyStorage.result = task1_result
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
assert 'Task2' in self.instantiated_tasks
assert 'Task3' in self.instantiated_tasks
task2 = self.get_task('Task2')
task3 = self.get_task('Task3')
assert task2.node_args == task1_result
assert task3.node_args == task1_result
def test_connect_and_configuration(self):
#
# flow1:
#
# Task1
# |
# |
# Task2
#
class MyStorage(DataStorage):
def __init__(self):
pass
def connect(self):
raise ConnectionError()
def disconnect(self):
# called on destruction
pass
def is_connected(self):
# return False so we can test connect()
return False
def store(self, node_args, flow_name, task_name, task_id, result):
# shouldn't be called
raise NotImplementedError()
def retrieve(self, flow_name, task_name, task_id):
# shouldn't be called
raise NotImplementedError()
def _cond_access(db, node_args):
return db.get('Task1') == 0xDEADBEEF
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task2'], 'condition': _cond_access},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
# Task1 has finished, we should access the database
task1 = self.get_task('Task1')
self.set_finished(task1, 1)
Config.storage_mapping = {'Storage1': MyStorage()}
Config.task2storage_mapping = {'Task1': 'Storage1'}
with pytest.raises(ConnectionError):
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
system_state.update()
def test_connect_and_configuration(self):
#
# flow1:
#
# Task1
# |
# |
# Task2
#
class MyStorage(DataStorage):
def __init__(self):
pass
def connect(self):
raise ConnectionError()
def disconnect(self):
# called on destruction
pass
def is_connected(self):
# return False so we can test connect()
return False
def store(self, node_args, flow_name, task_name, task_id, result):
# shouldn't be called
raise NotImplementedError()
def retrieve(self, flow_name, task_name, task_id):
# shouldn't be called
raise NotImplementedError()
def _cond_access(db, node_args):
return db.get('Task1') == 0xDEADBEEF
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task2'], 'condition': _cond_access},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
# Task1 has finished, we should access the database
task1 = self.get_task('Task1')
self.set_finished(task1, 1)
Config.storage_mapping = {'Storage1': MyStorage()}
Config.task2storage_mapping = {'Task1': 'Storage1'}
with pytest.raises(ConnectionError):
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
system_state.update()
def simulate_flow2(flow2, f2_already_instantiated_tasks):
f2_system_state = flow2.get_initial_system_state()
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert f2_already_instantiated_tasks | {'TaskF2'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('TaskF2'))
f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict,
node_args=f2_system_state.node_args, selective=f2_selective)
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert f2_already_instantiated_tasks | {'TaskF2', 'Task4'} == set(self.instantiated_tasks)
assert 'flow3' not in self.instantiated_flows
self.set_finished(self.get_task('Task4'))
f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict,
node_args=f2_system_state.node_args, selective=f2_selective)
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert 'flow3' in self.instantiated_flows
simulate_flow3(self.get_flow('flow3'), set(self.instantiated_tasks))
f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict,
node_args=f2_system_state.node_args, selective=f2_selective)
f2_retry = f2_system_state.update()
assert f2_retry is None
self.set_finished(flow2, None)
# let's clean after self so we inspect purely flow1 in callee
self.remove_all_tasks_by_name('TaskF2')
self.remove_all_tasks_by_name('Task4')
self.remove_all_flows_by_name('flow2')
def test_selective_flow_sybsequent_cyclic(self):
#
# Task1 <-
# / | |
# / | |
# Task3 Task2 --
#
# This is a very special case, where we don't want to keep state in dispatcher, so all tasks will be run
#
edge_table = {
'flow1': [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}, {
'from': ['Task1'],
'to': ['Task2'],
'condition': self.cond_true
}, {
'from': ['Task1'],
'to': ['Task3'],
'condition': self.cond_true
}, {
'from': ['Task2'],
'to': ['Task1'],
'condition': self.cond_true
}]
}
self.init(edge_table)
system_state = self.get_initial_system_state('flow1',
None, ['Task2'],
follow_subflows=False,
run_subsequent=True)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task1'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3'} == set(self.instantiated_tasks)
def test_result_cache_issue(self, cache):
#
# flow1:
#
# Task1
#
# Checks that if there is an issue with retrieving task result from result cache in condition, dispatcher
# will use directly storage.
edge_table = {
'flow1': [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}, {
'from': ['Task1'],
'to': 'Task2',
'condition': lambda db, _: db.get('Task1')
}]
}
self.init(edge_table,
storage_mapping={'Storage1': self.MyStorage()},
task2storage_mapping={'Task1': 'Storage1'},
storage2storage_cache={'Storage1': cache})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
def test_multi_failures(self):
#
# flow1:
#
# Task1 X
# |
# |
# Task2
#
# Note:
# Task1 will fail, Task2 should be run
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
failures = {
'flow1': {'Task1': {'next:': {},
'fallback': [['Task3'], ['Task4', 'Task5'], ['Task6', 'Task7']],
'conditions': [self.cond_true, self.cond_false, self.cond_true],
'condition_strs': ['true', 'false', 'true']}}
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
# Task1 has failed
task1 = self.get_task('Task1')
self.set_failed(task1, KeyError("Some exception raised"))
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
assert 'Task2' not in self.instantiated_tasks
assert 'Task3' in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert 'Task6' in self.instantiated_tasks
assert 'Task7' in self.instantiated_tasks
assert retry is not None
def test_task2tasks(self):
#
# flow1:
#
# Task1
# |
# --------
# | |
# Task2 Task3
#
# Note:
# Result of Task1 is propagated to flow1 as node_args
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task2', 'Task3'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table,
node_args_from_first={'flow1': True},
task2storage_mapping={'Task1': 'Storage1'},
storage_mapping={'Storage1': _MyStorage()})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
# Task1 has finished
task1 = self.get_task('Task1')
task1_result = "propagated result of Task1"
self.set_finished(task1, task1_result)
_MyStorage.result = task1_result
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
assert 'Task2' in self.instantiated_tasks
assert 'Task3' in self.instantiated_tasks
task2 = self.get_task('Task2')
task3 = self.get_task('Task3')
assert task2.node_args == task1_result
assert task3.node_args == task1_result
def test_propagate_parent_flow(self):
#
# flow1:
# Task1
# |
# -------
# | |
# flow2 flow3
#
# Note:
# Arguments are propagated to flow2 but not to flow3. Result of Task1 is not propagated to flow2 nor flow3.
#
edge_table = {
'flow1': [{'from': [], 'to': ['Task1'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['flow2', 'flow3'], 'condition': self.cond_true}],
'flow2': [],
'flow3': []
}
self.init(edge_table, propagate_parent={'flow1': ['flow2']}, propagate_node_args={'flow1': ['flow2']})
args = 123
system_state = SystemState(id(self), 'flow1', args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert 'Task1' in self.instantiated_tasks
assert 'flow2' not in self.instantiated_flows
assert 'flow3' not in self.instantiated_flows
task1 = self.get_task('Task1')
self.set_finished(task1, 0xDEADBEEF)
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
assert 'flow2' in self.instantiated_flows
assert 'flow3' in self.instantiated_flows
flow2 = self.get_flow('flow2')
assert flow2.node_args == args
assert 'Task1' in flow2.parent
assert flow2.parent['Task1'] == task1.task_id
flow3 = self.get_flow('flow3')
assert flow3.node_args is None
assert flow3.parent is None
def test_selective_flow_one_task(self):
#
# flow1:
#
# Task1
# | \
# | \
# Task2 Task4
# |
# |
# Task3
#
# Note: Only Task2 and Task1 should be run
#
edge_table = {
'flow1': [{'from': [], 'to': ['Task1'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['Task4'], 'condition': self.cond_true},
{'from': ['Task2'], 'to': ['Task3'], 'condition': self.cond_true}],
}
node_args = {'foo': 'bar'}
self.init(edge_table)
system_state = self.get_initial_system_state('flow1', node_args, ['Task2'],
follow_subflows=False, run_subsequent=False)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task1'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task2'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
assert retry is None
def test_propagate_parent_true(self):
#
# flow1:
#
# Task1
# |
# |
# flow2
#
# Note:
# Parent are propagated to flow2
#
edge_table = {
'flow1': [{'from': [], 'to': ['Task1'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true}],
'flow2': []
}
self.init(edge_table, propagate_parent={'flow1': True})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'flow2' not in self.instantiated_flows
assert 'Task1' in self.instantiated_tasks
task1 = self.get_task('Task1')
self.set_finished(task1, 0xDEADBEEF)
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
assert system_state.node_args is None
assert 'flow2' in self.instantiated_flows
flow2 = self.get_flow('flow2')
assert flow2.node_args is None
assert 'Task1' in flow2.parent
assert flow2.parent['Task1'] == task1.task_id
def test_result_backend_issue(self):
#
# flow1:
#
# Task1
#
# Checks that if there is an issue with result backend, dispatcher retries as there is no info on how
# to continue
edge_table = {
'flow1': [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
self.init(edge_table,
storage_mapping={'Storage1': self.MyStorage()},
task2storage_mapping={'Task1': 'Storage1'})
node_args = {'Foo': 'bar'}
system_state = SystemState(id(self), 'flow1', node_args=node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
flexmock(AsyncResult).\
should_receive('successful').\
and_raise(ValueError, "Some error raised due to result backed issues")
with pytest.raises(DispatcherRetry) as exc_info:
SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args).update()
assert exc_info.value.keep_state is True
assert exc_info.value.adjust_retry_count is False
def test_result_backend_issue(self):
#
# flow1:
#
# Task1
#
# Checks that if there is an issue with result backend, dispatcher retries as there is no info on how
# to continue
edge_table = {
'flow1': [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(
edge_table,
storage_mapping={'Storage1': self.MyStorage()},
task2storage_mapping={'Task1': 'Storage1'}
)
node_args = {'Foo': 'bar'}
system_state = SystemState(id(self), 'flow1', node_args=node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
task1 = self.get_task('Task1')
self.set_finished(task1, "some result")
flexmock(AsyncResult).\
should_receive('successful').\
and_raise(ValueError, "Some error raised due to result backed issues")
with pytest.raises(DispatcherRetry) as exc_info:
SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args).update()
assert exc_info.value.keep_state is True
assert exc_info.value.adjust_retry_count is False
def test_selective_flow_sybsequent_cyclic(self):
#
# Task1 <-
# / | |
# / | |
# Task3 Task2 --
#
# This is a very special case, where we don't want to keep state in dispatcher, so all tasks will be run
#
edge_table = {
'flow1': [{'from': [], 'to': ['Task1'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true},
{'from': ['Task2'], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
system_state = self.get_initial_system_state('flow1', None, ['Task2'],
follow_subflows=False, run_subsequent=True)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task1'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3'} == set(self.instantiated_tasks)
def test_selective_flow_subflow(self):
#
# flow1:
#
# flow2 Task1
# | |
# | Task2
# | | \
# ------- Task3
# |
# |
# Task4
# |
# |
# Task5
#
# flow2:
#
# TaskF2
# |
# |
# Task4
# | \
# | \
# flow3 flow4
#
# flow3:
#
# Task4
# |
# |
# TaskF3
#
# flow4:
#
# TaskF3
#
# Note: Only Task4 should be run, flow4 is not executed
#
def simulate_flow2(flow2, f2_already_instantiated_tasks):
f2_system_state = flow2.get_initial_system_state()
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert f2_already_instantiated_tasks | {'TaskF2'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('TaskF2'))
f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict,
node_args=f2_system_state.node_args, selective=f2_selective)
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert f2_already_instantiated_tasks | {'TaskF2', 'Task4'} == set(self.instantiated_tasks)
assert 'flow3' not in self.instantiated_flows
self.set_finished(self.get_task('Task4'))
f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict,
node_args=f2_system_state.node_args, selective=f2_selective)
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert 'flow3' in self.instantiated_flows
simulate_flow3(self.get_flow('flow3'), set(self.instantiated_tasks))
f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict,
node_args=f2_system_state.node_args, selective=f2_selective)
f2_retry = f2_system_state.update()
assert f2_retry is None
self.set_finished(flow2, None)
# let's clean after self so we inspect purely flow1 in callee
self.remove_all_tasks_by_name('TaskF2')
self.remove_all_tasks_by_name('Task4')
self.remove_all_flows_by_name('flow2')
def simulate_flow3(flow3, f3_already_instantiated_tasks):
f3_system_state = flow3.get_initial_system_state()
f3_retry = f3_system_state.update()
f3_state_dict = f3_system_state.to_dict()
f3_selective = f3_system_state.selective
assert f3_retry is not None
assert f3_already_instantiated_tasks | {'Task4'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task4'))
f3_system_state = SystemState(id(self), 'flow3', state=f3_state_dict,
node_args=f3_system_state.node_args, selective=f3_selective)
f3_retry = f3_system_state.update()
assert f3_retry is None
self.set_finished(flow3, None)
self.remove_all_flows_by_name('flow3')
edge_table = {
'flow1': [{'from': [], 'to': ['flow2', 'Task1'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true},
{'from': ['flow2', 'Task2'], 'to': ['Task4'], 'condition': self.cond_true},
{'from': ['Task4'], 'to': ['Task5'], 'condition': self.cond_true}],
'flow2': [{'from': [], 'to': ['TaskF2'], 'condition': self.cond_true},
{'from': ['TaskF2'], 'to': ['Task4'], 'condition': self.cond_true},
{'from': ['Task4'], 'to': ['flow4', 'flow3'], 'condition': self.cond_true}],
'flow3': [{'from': [], 'to': ['Task4'], 'condition': self.cond_true},
{'from': ['Task4'], 'to': ['TaskF3'], 'condition': self.cond_true}],
'flow4': [{'from': [], 'to': ['TaskF3']}]
}
self.init(edge_table)
system_state = self.get_initial_system_state('flow1', None, ['Task4'],
follow_subflows=True, run_subsequent=False)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1'} == set(self.instantiated_tasks)
assert {'flow2'} == set(self.instantiated_flows)
simulate_flow2(self.get_flow('flow2'), set(self.instantiated_tasks))
self.set_finished(self.get_task('Task1'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task2'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task4'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task4'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
assert retry is None
assert 'flow4' not in self.instantiated_flows
def test_propagate_compound_mixed(self):
#
# flow1:
#
# flow2 flow3
# | |
# -----------
# |
# TaskX
#
# flow2:
# Not run explicitly, but result finished_nodes is:
# {'Task2': [<task2-id21>],
# 'Task3': [<task3-id21>, <task3-id22>]}
# 'flow4': {'Task2': [<task2-id41>]}
# flow3:
# Not run explicitly, but result finished_nodes is:
# {'Task2': [<task2-id32>],
# 'Task3': [<task3-id31>, <task3-id32>],
# 'flow4': {'Task2': [<task2-id42>]}
#
# We are propagating finished, so we should inspect parent
#
edge_table = {
'flow1': [{'from': [], 'to': ['flow2', 'flow3'], 'condition': self.cond_true},
{'from': ['flow2', 'flow3'], 'to': ['TaskX'], 'condition': self.cond_true}],
'flow2': [],
'flow3': [],
'flow4': []
}
# Make sure propagate_finished is negated propagate_compound_finished
# this is checked in selinon
self.init(edge_table,
propagate_finished={'flow1': ['flow2']},
propagate_compound_finished={'flow1': ['flow3']})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert 'flow2' in self.instantiated_flows
# Create flow4 manually, we will reuse it, but we pretend that there are 2 instances - one run in flow2
# another one in flow3
flow4 = Dispatcher().apply_async(kwargs={'flow_name': 'flow4'}, queue=Config.dispatcher_queues['flow4'])
self.get_task_instance.register_node(flow4)
self.set_finished(flow4, {'finished_nodes': {'Task2': ['<task2-id41']}, 'failed_nodes': {}})
# Create results of flow2 and flow3 manually but they are instantiated by dispatcher
flow2_result = {'finished_nodes': {'Task2': ['<task2-id21>'],
'Task3': ['<task3-id21>', '<task3-id22>'],
'flow4': [flow4.task_id]},
'failed_nodes': {}
}
flow2 = self.get_flow('flow2')
self.set_finished(flow2, flow2_result)
flow3_result = {'finished_nodes': {'Task2': ['<task2-id32>'],
'Task3': ['<task3-id31>', '<task3-id32>'],
'flow4': [flow4.task_id]},
'failed_nodes': {}
}
flow3 = self.get_flow('flow3')
self.set_finished(flow3, flow3_result)
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
assert 'TaskX' in self.instantiated_tasks
task_x_parent = {'flow2': {'Task2': ['<task2-id21>'],
'Task3': ['<task3-id21>', '<task3-id22>'],
'flow4': {'Task2': ['<task2-id41']}},
# flow4 is hidden in flow3 as it is compound, see 'Task2'
'flow3': {'Task2': ['<task2-id41', '<task2-id32>'],
'Task3': ['<task3-id31>', '<task3-id32>']}}
task_x = self.get_task('TaskX')
# we have to check this as a set due to dict randomization
assert 'flow2' in task_x.parent
assert 'flow3' in task_x.parent
assert 'Task2' in task_x.parent['flow2']
assert 'Task3' in task_x.parent['flow2']
assert set(task_x.parent['flow2']['Task2']) == set(task_x_parent['flow2']['Task2'])
assert set(task_x.parent['flow2']['Task3']) == set(task_x_parent['flow2']['Task3'])
assert 'Task2' in task_x.parent['flow3']
assert 'Task3' in task_x.parent['flow3']
assert set(task_x.parent['flow3']['Task2']) == set(task_x_parent['flow3']['Task2'])
assert set(task_x.parent['flow3']['Task3']) == set(task_x_parent['flow3']['Task3'])
assert 'flow4' in task_x.parent['flow2']
assert set(task_x.parent['flow2']['flow4']) == set(task_x_parent['flow2']['flow4'])
def test_propagate_parent_2(self):
#
# flow1:
#
# Task1 Task2
# | |
# flow2 |
# | |
# -----------
# |
# TaskX
#
# flow2:
# Not run explicitly, but result finished_nodes is:
# {'flow3': [<flow3-id>], 'Task2': [<task2-id1>], 'Task3': [<task3-id1>, <task3-id2>]}
# flow3:
# Not run explicitly, but result finished_nodes is:
# {'Task2': [<task2-id2>], 'Task4': [<task4-id1>, <task4-id2>]}
#
# We are propagating finished, so we should inspect parent
#
edge_table = {
'flow1': [{'from': ['flow2', 'Task2'], 'to': ['TaskX'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}],
'flow2': [],
'flow3': []
}
# Make sure propagate_finished is set to False as they are disjoint with propagate_compound_finished;
# this is checked in selinon
self.init(edge_table, propagate_parent=dict.fromkeys(edge_table.keys(), True),
propagate_finished={'flow1': False},
propagate_compound_finished={'flow1': True})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'flow2' not in self.instantiated_flows
# Task1 and Task2 have finished
task1 = self.get_task('Task1')
self.set_finished(task1, None)
task2 = self.get_task('Task2')
self.set_finished(task2, None)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert 'flow2' in self.instantiated_flows
# Create flow3 manually
flow3 = Dispatcher().apply_async(kwargs={'flow_name': 'flow3'}, queue=Config.dispatcher_queues['flow3'])
self.get_task_instance.register_node(flow3)
flow2 = self.get_flow('flow2')
self.set_finished(flow2)
flow2_result = {'finished_nodes': {'flow3': [flow3.task_id],
'Task2': ['<task2-id1>'],
'Task3': ['<task3-id1>', '<task3-id2>']},
'failed_nodes': {}
}
self.set_finished(flow2, flow2_result)
flow3_result = {'finished_nodes': {'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>']},
'failed_nodes': {}
}
self.set_finished(flow3, flow3_result)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
task_x = self.get_task('TaskX')
task_x_parent = {'Task2': task2.task_id,
'flow2': {'Task2': ['<task2-id1>', '<task2-id2>'],
'Task3': ['<task3-id1>', '<task3-id2>'],
'Task4': ['<task4-id1>', '<task4-id2>']
}
}
# we have to check this as a set due to dict randomization
assert 'Task2' in task_x.parent
assert task_x.parent['Task2'] == task_x_parent['Task2']
assert 'flow2' in task_x.parent
assert 'Task2' in task_x.parent['flow2']
assert 'Task3' in task_x.parent['flow2']
assert 'Task4' in task_x.parent['flow2']
assert set(task_x.parent['flow2']['Task2']) == set(task_x_parent['flow2']['Task2'])
assert set(task_x.parent['flow2']['Task3']) == set(task_x_parent['flow2']['Task3'])
assert set(task_x.parent['flow2']['Task4']) == set(task_x_parent['flow2']['Task4'])
def test_propagate_parent(self):
#
# flow1:
#
# Task1 Task2
# | |
# flow2 |
# | |
# -----------
# |
# TaskX
#
# flow2:
# Run explicitly, but result finished_nodes is:
# {'flow3': [<flow3-id>], 'Task2': [<task2-id1>], 'Task3': [<task3-id1>, <task3-id2>]}
# flow3
# Run explicitly, but result finished_nodes is:
# {'Task2': [<task2-id2>], 'Task4': [<task4-id1>, <task4-id2>]}
#
# We are propagating finished, so we should inspect parent
#
edge_table = {
'flow1': [{'from': ['flow2', 'Task2'], 'to': ['TaskX'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}],
'flow2': [],
'flow3': []
}
self.init(edge_table, propagate_parent={'flow1': True}, propagate_finished={'flow1': True})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'flow2' not in self.instantiated_flows
# Task1 and Task2 have finished
task1 = self.get_task('Task1')
self.set_finished(task1, None)
task2 = self.get_task('Task2')
self.set_finished(task2, None)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert 'flow2' in self.instantiated_flows
# Create flow3 manually
flow3 = Dispatcher().apply_async(kwargs={'flow_name': 'flow3'}, queue=Config.dispatcher_queues['flow3'])
self.get_task_instance.register_node(flow3)
flow2 = self.get_flow('flow2')
self.set_finished(flow2)
flow2_result = {'finished_nodes': {'flow3': [flow3.task_id],
'Task2': ['<task2-id1>'],
'Task3': ['<task3-id1>', '<task3-id2>']},
'failed_nodes': {}
}
self.set_finished(flow2, flow2_result)
flow3_result = {'finished_nodes':
{'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>']},
'failed_nodes': {}
}
self.set_finished(flow3, flow3_result)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
task_x = self.get_task('TaskX')
task_x_parent = {'Task2': task2.task_id,
'flow2': {'Task2': ['<task2-id1>'],
'Task3': ['<task3-id1>', '<task3-id2>'],
'flow3': {'Task2': ['<task2-id2>'],
'Task4': ['<task4-id1>', '<task4-id2>'],
}
}
}
assert task_x.parent == task_x_parent
def test_two_failures_fallback_start(self):
#
# flow1:
#
# Task1 X Task2 X
# | |
# | |
# Task3 Task4
#
# Note:
# Task1 will fail, then Task2 will fail; fallback for Task1, Task2 is Task5
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true},
{'from': ['Task2'], 'to': ['Task4'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}],
}
failures = {
'flow1': {'Task1': {'next': {'Task2': {'next': {},
'fallback': [['Task5']],
'conditions': [self.cond_true],
'condition_strs': ['cond_true'],
}},
'fallback': []},
'Task2': {'next': {'Task1': {'next': {},
'fallback': [['Task5']],
'conditions': [self.cond_true],
'condition_strs': ['cond_true'],
}},
'fallback': []}
}
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
task1 = self.get_task('Task1')
self.set_failed(task1, KeyError("Some exception raised"))
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
assert 'Task1' in state_dict['failed_nodes']
# Task2 has failed
task2 = self.get_task('Task2')
self.set_failed(task2)
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
assert 'Task1' not in state_dict['failed_nodes']
assert 'Task2' not in state_dict['failed_nodes']
def test_nowait_in_flow(self):
#
# flow1:
#
# Task1
# |
# ----------
# | |
# Task2 Task3
#
# Note:
# Task3 finishes before Task2 and Task3 is marked as nowait
#
edge_table = {
'flow1': [{
'from': ['Task1'],
'to': ['Task2', 'Task3'],
'condition': self.cond_true
}, {
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
nowait_nodes = {'flow1': ['Task3']}
self.init(edge_table, nowait_nodes=nowait_nodes)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' not in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert 0 in state_dict['waiting_edges']
assert len(state_dict.get('finished_nodes')) == 0
assert len(state_dict.get('active_nodes')) == 1
# Task1 has finished
task1 = self.get_task('Task1')
self.set_finished(task1, 1)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert 0 in state_dict['waiting_edges']
assert len(state_dict.get('finished_nodes')) == 1
assert len(state_dict.get('active_nodes')) == 1
assert 'Task3' not in state_dict.get('active_nodes')
assert 'Task2' in state_dict['active_nodes'][0]['name']
def test_single_failure_finish_wait(self):
#
# flow1:
#
# Task1 Task2 X
# | |
# | |
# Task3 Task4
#
# No fallback defined for Task2
#
# Note:
# Task2 will fail, no fallback defined, Dispatcher should wait for Task1 to finish and
# raise FlowError exception
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true},
{'from': ['Task2'], 'to': ['Task4'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}],
}
failures = {
'flow1': {'Task1': {'next': {'Task2': {'next': {}, 'fallback': []}}, 'fallback': []},
'Task2': {'next': {'Task1': {'next': {}, 'fallback': []}}, 'fallback': []}
}
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
# Task2 has failed
task2 = self.get_task('Task2')
self.set_failed(task2)
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
assert 'Task2' in state_dict['failed_nodes']
# No change so far, still wait
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
assert 'Task2' in state_dict['failed_nodes']
# Task1 has finished successfully
task1 = self.get_task('Task1')
self.set_finished(task1, 0)
# Wait for Task3
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert 'Task3' in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
# Task3 has finished successfully
task3 = self.get_task('Task3')
self.set_finished(task3, 0)
with pytest.raises(FlowError):
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
system_state.update()
def test_selective_flow_two_tasks(self):
#
# flow1:
#
# Task1 Task2 Task3 Task4 Task5
# \ / \ / |
# \ / \ / Task10
# ---------- -------
# | / \
# Task6 Task7 Task8
# | |
# | |
# Task11 Task9
#
# Note: Only Task6 and Task7 should be run
#
edge_table = {
'flow1': [{'from': [], 'to': ['Task1', 'Task2', 'Task3', 'Task4', 'Task5'], 'condition': self.cond_true},
{'from': ['Task1', 'Task2'], 'to': ['Task6'], 'condition': self.cond_true},
{'from': ['Task3', 'Task4'], 'to': ['Task7', 'Task8'], 'condition': self.cond_true},
{'from': ['Task7'], 'to': ['Task9'], 'condition': self.cond_true},
{'from': ['Task5'], 'to': ['Task10'], 'condition': self.cond_true}],
}
self.init(edge_table)
system_state = self.get_initial_system_state('flow1', None, ['Task6', 'Task7'],
follow_subflows=False, run_subsequent=False)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4'} == set(self.instantiated_tasks)
# Task1 has finished
self.set_finished(self.get_task('Task1'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4'} == set(self.instantiated_tasks)
# Task2 has finished
self.set_finished(self.get_task('Task2'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6'} == set(self.instantiated_tasks)
# Task3 has finished
self.set_finished(self.get_task('Task3'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6'} == set(self.instantiated_tasks)
# Task4 has finished
self.set_finished(self.get_task('Task4'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6', 'Task7'} == set(self.instantiated_tasks)
# Task7 has finished
self.set_finished(self.get_task('Task7'), None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6', 'Task7'} == set(self.instantiated_tasks)
# Task6 has finished
task6 = self.get_task('Task6')
self.set_finished(task6, None)
system_state = SystemState(id(self), 'flow1',
state=state_dict, node_args=system_state.node_args, selective=selective)
retry = system_state.update()
assert retry is None
def test_flow_failure(self):
#
# flow1:
#
# flow2 X flow4 X Task1_f1 Task2_f1 X flow5
# | | | | |
# ................................................
# |
# TaskX
#
# flow2:
#
# Task1_f2 Task2_f2 X Task3_f2 flow3
#
# flow3:
#
# Task1_f3 Task2_f3 X Task3_f3 X
#
# flow4:
#
# Task1_f4 Task2_f4 X Task3_f4
#
# flow5:
# Task1_f5 Task2_f5 X
#
# Note:
# All flows fail, we inspect finished propagation in flow1 in case of failure. Tasks marked with X will fail.
#
edge_table = {
'flow1': [{'from': ['flow2', 'flow5', 'flow4', 'Task1_f1', 'Task2_f1'], 'to': ['TaskX'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1_f1', 'Task2_f1', 'flow2', 'flow4', 'flow5'], 'condition': self.cond_true}],
'flow2': [{'from': [], 'to': ['Task1_f2', 'Task2_f2', 'Task3_f2', 'flow3'], 'condition': self.cond_true}],
'flow3': [{'from': [], 'to': ['Task1_f3', 'Task2_f3', 'Task3_f3'], 'condition': self.cond_true}],
'flow4': [{'from': [], 'to': ['Task1_f4', 'Task2_f4', 'Task3_f4'], 'condition': self.cond_true}],
'flow5': [{'from': [], 'to': ['Task1_f5', 'Task2_f5'], 'condition': self.cond_true}]
}
failures = {
'flow1': {
'Task2_f1': {
'next': {
'flow2': {
'next': {
'flow4': {
'next': {},
'fallback': [['TaskX']],
'conditions': [self.cond_true],
'condition_strs': ['cond_true']
},
'fallback': []
},
'fallback': []
},
'fallback': []
},
'fallback': []
},
'fallback': []
},
'fallback': []
}
self.init(edge_table, failures=failures, propagate_parent=dict.fromkeys(edge_table, True))
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'flow2' in self.instantiated_flows
assert 'flow4' in self.instantiated_flows
assert 'flow5' in self.instantiated_flows
assert 'Task1_f1' in self.instantiated_tasks
assert 'Task2_f1' in self.instantiated_tasks
assert 'flow3' not in self.instantiated_flows
# Task1_f1 have finished
task1_f1 = self.get_task('Task1_f1')
self.set_finished(task1_f1, 0)
# Task2_f1 has failed
task2_f1 = self.get_task('Task2_f1')
self.set_failed(task2_f1)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'flow3' not in self.instantiated_flows
# flow2
system_state = SystemState(id(self), 'flow2')
retry = system_state.update()
state_dict_2 = system_state.to_dict()
flow2 = self.get_flow('flow2')
assert retry is not None
assert system_state.node_args is None
assert 'Task1_f2' in self.instantiated_tasks
assert 'Task2_f2' in self.instantiated_tasks
assert 'Task3_f2' in self.instantiated_tasks
assert 'flow3' in self.instantiated_flows
# Task1_f2, Task3_f2 have finished
task1_f2 = self.get_task('Task1_f2')
self.set_finished(task1_f2, 1)
task3_f2 = self.get_task('Task3_f2')
self.set_finished(task3_f2, 2)
# Task2_f2 has failed
task2_f2 = self.get_task('Task2_f2')
self.set_failed(task2_f2)
system_state = SystemState(id(self), 'flow2', state=state_dict_2, node_args=system_state.node_args)
retry = system_state.update()
state_dict_2 = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
# flow 3
system_state = SystemState(id(self), 'flow3')
retry = system_state.update()
state_dict_3 = system_state.to_dict()
flow3 = self.get_flow('flow3')
assert retry is not None
assert system_state.node_args is None
assert 'Task1_f3' in self.instantiated_tasks
assert 'Task2_f3' in self.instantiated_tasks
assert 'Task3_f3' in self.instantiated_tasks
# Task1_f3 has finished
task1_f3 = self.get_task('Task1_f3')
self.set_finished(task1_f3, 3)
# Task2_f3, Task2_f3 have failed
task2_f3 = self.get_task('Task2_f3')
self.set_failed(task2_f3)
task3_f3 = self.get_task('Task3_f3')
self.set_failed(task3_f3)
try:
system_state = SystemState(id(self), 'flow3', state=state_dict_3, node_args=system_state.node_args)
system_state.update()
except FlowError as exc:
self.set_failed(flow3, exc)
else:
assert not "Expected FlowError not raised"
# flow4
system_state = SystemState(id(self), 'flow4')
retry = system_state.update()
state_dict_4 = system_state.to_dict()
flow4 = self.get_flow('flow4')
assert retry is not None
assert system_state.node_args is None
assert 'Task1_f4' in self.instantiated_tasks
assert 'Task2_f4' in self.instantiated_tasks
assert 'Task3_f4' in self.instantiated_tasks
# Task1_f2, Task3_f2 have finished
task1_f4 = self.get_task('Task1_f4')
self.set_finished(task1_f4, 1)
task3_f4 = self.get_task('Task3_f4')
self.set_finished(task3_f4, 4)
# Task2_f4 has failed
task2_f4 = self.get_task('Task2_f4')
self.set_failed(task2_f4)
try:
system_state = SystemState(id(self), 'flow4', state=state_dict_4, node_args=system_state.node_args)
system_state.update()
except FlowError as exc:
self.set_failed(flow4, exc)
else:
assert not "Expected FlowError not raised"
# flow5
system_state = SystemState(id(self), 'flow5')
retry = system_state.update()
state_dict_5 = system_state.to_dict()
flow5 = self.get_flow('flow5')
assert retry is not None
assert system_state.node_args is None
assert 'Task1_f5' in self.instantiated_tasks
assert 'Task2_f5' in self.instantiated_tasks
# Task1_f5, Task2_f5 have finished
task1_f5 = self.get_task('Task1_f5')
self.set_finished(task1_f5, 1)
task2_f5 = self.get_task('Task2_f5')
self.set_finished(task2_f5, 2)
system_state = SystemState(id(self), 'flow5', state=state_dict_5, node_args=system_state.node_args)
retry = system_state.update()
assert retry is None
state_info = {
'finished_nodes': system_state.to_dict()['finished_nodes'],
'failed_nodes': system_state.to_dict()['failed_nodes']
}
self.set_finished(flow5, state_info)
# Back to flow2, all child finished
try:
system_state = SystemState(id(self), 'flow2', state=state_dict_2, node_args=system_state.node_args)
system_state.update()
except FlowError as exc:
self.set_failed(flow2, exc)
else:
assert not "Expected FlowError not raised"
# finally inspect flow1 result and finished that is propagated at fallback to TaskX
assert 'TaskX' not in self.instantiated_tasks
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
system_state.update()
assert 'TaskX' in self.instantiated_tasks
task_x = self.get_task('TaskX')
assert task_x.parent is None
def test_selective_flow_one_task(self):
#
# flow1:
#
# Task1
# | \
# | \
# Task2 Task4
# |
# |
# Task3
#
# Note: Only Task2 and Task1 should be run
#
edge_table = {
'flow1': [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}, {
'from': ['Task1'],
'to': ['Task2'],
'condition': self.cond_true
}, {
'from': ['Task1'],
'to': ['Task4'],
'condition': self.cond_true
}, {
'from': ['Task2'],
'to': ['Task3'],
'condition': self.cond_true
}],
}
node_args = {'foo': 'bar'}
self.init(edge_table)
system_state = self.get_initial_system_state('flow1',
node_args, ['Task2'],
follow_subflows=False,
run_subsequent=False)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task1'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task2'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
assert retry is None
def test_two_failures_no_fallback(self):
#
# flow1:
#
# Task1 X Task2 X
# | |
# | |
# Task3 Task4
#
# Note:
# Task1 will fail, then Task2 will fail
#
edge_table = {
'flow1': [{'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true},
{'from': ['Task2'], 'to': ['Task4'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}],
}
failures = {
'flow1': {'Task1': {'next': {'Task2': {'next': {}, 'fallback': []}}, 'fallback': []},
'Task2': {'next': {'Task1': {'next': {}, 'fallback': []}}, 'fallback': []}
}
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
task1 = self.get_task('Task1')
self.set_failed(task1)
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
assert 'Task1' in state_dict['failed_nodes']
# No change so far
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'Task3' not in self.instantiated_tasks
assert 'Task4' not in self.instantiated_tasks
assert 'Task5' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 2
assert 0 in state_dict['waiting_edges']
assert 1 in state_dict['waiting_edges']
assert 'Task1' in state_dict['failed_nodes']
# Task2 has failed
task2 = self.get_task('Task2')
self.set_failed(task2)
with pytest.raises(FlowError):
system_state = SystemState(id(self), 'flow1', state=state_dict,
node_args=system_state.node_args)
system_state.update()
def test_selective_flow_subflow(self):
#
# flow1:
#
# flow2 Task1
# | |
# | Task2
# | | \
# ------- Task3
# |
# |
# Task4
# |
# |
# Task5
#
# flow2:
#
# TaskF2
# |
# |
# Task4
# | \
# | \
# flow3 flow4
#
# flow3:
#
# Task4
# |
# |
# TaskF3
#
# flow4:
#
# TaskF3
#
# Note: Only Task4 should be run, flow4 is not executed
#
def simulate_flow2(flow2, f2_already_instantiated_tasks):
f2_system_state = flow2.get_initial_system_state()
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert f2_already_instantiated_tasks | {'TaskF2'} == set(
self.instantiated_tasks)
self.set_finished(self.get_task('TaskF2'))
f2_system_state = SystemState(id(self),
'flow2',
state=f2_state_dict,
node_args=f2_system_state.node_args,
selective=f2_selective)
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert f2_already_instantiated_tasks | {'TaskF2', 'Task4'} == set(
self.instantiated_tasks)
assert 'flow3' not in self.instantiated_flows
self.set_finished(self.get_task('Task4'))
f2_system_state = SystemState(id(self),
'flow2',
state=f2_state_dict,
node_args=f2_system_state.node_args,
selective=f2_selective)
f2_retry = f2_system_state.update()
f2_state_dict = f2_system_state.to_dict()
f2_selective = f2_system_state.selective
assert f2_retry is not None
assert 'flow3' in self.instantiated_flows
simulate_flow3(self.get_flow('flow3'),
set(self.instantiated_tasks))
f2_system_state = SystemState(id(self),
'flow2',
state=f2_state_dict,
node_args=f2_system_state.node_args,
selective=f2_selective)
f2_retry = f2_system_state.update()
assert f2_retry is None
self.set_finished(flow2, None)
# let's clean after self so we inspect purely flow1 in callee
self.remove_all_tasks_by_name('TaskF2')
self.remove_all_tasks_by_name('Task4')
self.remove_all_flows_by_name('flow2')
def simulate_flow3(flow3, f3_already_instantiated_tasks):
f3_system_state = flow3.get_initial_system_state()
f3_retry = f3_system_state.update()
f3_state_dict = f3_system_state.to_dict()
f3_selective = f3_system_state.selective
assert f3_retry is not None
assert f3_already_instantiated_tasks | {'Task4'} == set(
self.instantiated_tasks)
self.set_finished(self.get_task('Task4'))
f3_system_state = SystemState(id(self),
'flow3',
state=f3_state_dict,
node_args=f3_system_state.node_args,
selective=f3_selective)
f3_retry = f3_system_state.update()
assert f3_retry is None
self.set_finished(flow3, None)
self.remove_all_flows_by_name('flow3')
edge_table = {
'flow1': [{
'from': [],
'to': ['flow2', 'Task1'],
'condition': self.cond_true
}, {
'from': ['Task1'],
'to': ['Task2'],
'condition': self.cond_true
}, {
'from': ['flow2', 'Task2'],
'to': ['Task4'],
'condition': self.cond_true
}, {
'from': ['Task4'],
'to': ['Task5'],
'condition': self.cond_true
}],
'flow2': [{
'from': [],
'to': ['TaskF2'],
'condition': self.cond_true
}, {
'from': ['TaskF2'],
'to': ['Task4'],
'condition': self.cond_true
}, {
'from': ['Task4'],
'to': ['flow4', 'flow3'],
'condition': self.cond_true
}],
'flow3': [{
'from': [],
'to': ['Task4'],
'condition': self.cond_true
}, {
'from': ['Task4'],
'to': ['TaskF3'],
'condition': self.cond_true
}],
'flow4': [{
'from': [],
'to': ['TaskF3']
}]
}
self.init(edge_table)
system_state = self.get_initial_system_state('flow1',
None, ['Task4'],
follow_subflows=True,
run_subsequent=False)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1'} == set(self.instantiated_tasks)
assert {'flow2'} == set(self.instantiated_flows)
simulate_flow2(self.get_flow('flow2'), set(self.instantiated_tasks))
self.set_finished(self.get_task('Task1'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task2'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task4'} == set(self.instantiated_tasks)
self.set_finished(self.get_task('Task4'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
assert retry is None
assert 'flow4' not in self.instantiated_flows
def test_selective_flow_two_tasks(self):
#
# flow1:
#
# Task1 Task2 Task3 Task4 Task5
# \ / \ / |
# \ / \ / Task10
# ---------- -------
# | / \
# Task6 Task7 Task8
# | |
# | |
# Task11 Task9
#
# Note: Only Task6 and Task7 should be run
#
edge_table = {
'flow1': [{
'from': [],
'to': ['Task1', 'Task2', 'Task3', 'Task4', 'Task5'],
'condition': self.cond_true
}, {
'from': ['Task1', 'Task2'],
'to': ['Task6'],
'condition': self.cond_true
}, {
'from': ['Task3', 'Task4'],
'to': ['Task7', 'Task8'],
'condition': self.cond_true
}, {
'from': ['Task7'],
'to': ['Task9'],
'condition': self.cond_true
}, {
'from': ['Task5'],
'to': ['Task10'],
'condition': self.cond_true
}],
}
self.init(edge_table)
system_state = self.get_initial_system_state('flow1',
None, ['Task6', 'Task7'],
follow_subflows=False,
run_subsequent=False)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3',
'Task4'} == set(self.instantiated_tasks)
# Task1 has finished
self.set_finished(self.get_task('Task1'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3',
'Task4'} == set(self.instantiated_tasks)
# Task2 has finished
self.set_finished(self.get_task('Task2'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4',
'Task6'} == set(self.instantiated_tasks)
# Task3 has finished
self.set_finished(self.get_task('Task3'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4',
'Task6'} == set(self.instantiated_tasks)
# Task4 has finished
self.set_finished(self.get_task('Task4'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6',
'Task7'} == set(self.instantiated_tasks)
# Task7 has finished
self.set_finished(self.get_task('Task7'), None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
state_dict = system_state.to_dict()
selective = system_state.selective
assert retry is not None
assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6',
'Task7'} == set(self.instantiated_tasks)
# Task6 has finished
task6 = self.get_task('Task6')
self.set_finished(task6, None)
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args,
selective=selective)
retry = system_state.update()
assert retry is None
