def test_flow_error_retry(self):
def my_retry(max_retries, exc):
assert max_retries == 0
assert json.loads(str(exc)) == json.loads(str(raised_exc))
raise RuntimeError() # we re-raise as stated in Celery doc
flow_name = 'flow1'
edge_table = {
flow_name: [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
self.init(edge_table)
state_dict = {
'finished_nodes': {},
'failed_nodes': {},
'active_nodes': []
}
raised_exc = FlowError(state_dict)
flexmock(SystemState).should_receive('update').and_raise(
raised_exc) # we will retry
flexmock(SystemState).should_receive('to_dict').and_return(state_dict)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').replace_with(my_retry)
with pytest.raises(RuntimeError):
dispatcher.run(flow_name)
def test_multiple_failures_from_subflow(self):
#
# flow2:
# Task0 Task1 X Task1 X
#
# flow1:
#
# flow2X..... Task3
#
# Note:
# flow2 will fail with two tasks of type Task1. flow1 defines fallback as True, so
# it should recover from failure
edge_table = {
'flow1': [{
'from': [],
'to': ['flow2'],
'condition': self.cond_true
}],
'flow2': []
} # flow2 handled manually
failures = {
'flow1': {
'flow2': {
'next': {},
'fallback': [True],
'conditions': [self.cond_true],
'condition_strs': ['cond_true']
}
},
'flow2': {}
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert 'flow2' in self.instantiated_flows
assert retry is not None
assert system_state.node_args is None
flow_info = {
'finished_nodes': {
'Task0': ['<id-tak0_0>']
},
'failed_nodes': {
'Task2': ['<id-task1_0>', '<id-task1_1>']
}
}
flow2 = self.get_flow('flow2')
self.set_failed(flow2, FlowError(flow_info))
system_state = SystemState(id(self),
'flow1',
state=state_dict,
node_args=system_state.node_args)
retry = system_state.update()
assert retry is None
def test_singe_failure_flow_fallback(self):
#
# flow1:
#
# flow2 X ... Task2
# |
# |
# Task1
#
# Note:
# flow2 will fail
#
edge_table = {
'flow1': [{'from': ['flow2'], 'to': ['Task1'], 'condition': self.cond_true},
{'from': [], 'to': ['flow2'], 'condition': self.cond_true}],
'flow2': []
}
failures = {
'flow1': {'flow2': {'next:': {}, 'fallback': [['Task2']],
'conditions': [self.cond_true],
'condition_strs': ['cond_true']
}}
}
self.init(edge_table, failures=failures)
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'flow2' in self.instantiated_flows
assert 'Task1' not in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
# flow2 has failed
flow2 = self.get_flow('flow2')
flow_info = {'finished_nodes': {'Task1': ['id_task1']}, 'failed_nodes': {'Task2': ['id_task21']}}
self.set_failed(flow2, FlowError(flow_info))
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'flow2' in self.instantiated_flows
assert 'Task2' in self.instantiated_tasks
assert len(state_dict.get('waiting_edges')) == 1
assert state_dict['waiting_edges'][0] == 0
def test_selinon_retry(self):
def my_retry(kwargs=None,
max_retries=None,
countdown=None,
queue=None,
exc=None):
kwargs = kwargs or {}
assert kwargs.get('flow_name') == flow_name
assert kwargs.get('node_args') == node_args
assert 'parent' in kwargs
assert 'selective' in kwargs
assert 'state' in kwargs
assert max_retries is None
assert countdown == 5
assert queue == 'queue_flow1'
assert exc is None
raise RuntimeError() # we re-raise as stated in Celery doc
flow_name = 'flow1'
node_args = {'foo': 'bar'}
edge_table = {
flow_name: [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
self.init(edge_table,
retry_countdown={'flow1': 5},
max_retry={'flow1': 1})
state_dict = {
'finished_nodes': {
'Task1': ['<task1-id>']
},
'failed_nodes': {}
}
flexmock(SystemState).should_receive('update').and_raise(
FlowError(state_dict))
dispatcher = Dispatcher()
flexmock(dispatcher).should_receive('retry').replace_with(my_retry)
dispatcher.request = RequestMock()
with pytest.raises(RuntimeError):
dispatcher.run(flow_name, node_args)