def test_flow_error_retry(self): def my_retry(max_retries, exc): assert max_retries == 0 assert json.loads(str(exc)) == json.loads(str(raised_exc)) raise RuntimeError() # we re-raise as stated in Celery doc flow_name = 'flow1' edge_table = { flow_name: [{ 'from': [], 'to': ['Task1'], 'condition': self.cond_true }] } self.init(edge_table) state_dict = { 'finished_nodes': {}, 'failed_nodes': {}, 'active_nodes': [] } raised_exc = FlowError(state_dict) flexmock(SystemState).should_receive('update').and_raise( raised_exc) # we will retry flexmock(SystemState).should_receive('to_dict').and_return(state_dict) dispatcher = Dispatcher() dispatcher.request = RequestMock() flexmock(dispatcher).should_receive('retry').replace_with(my_retry) with pytest.raises(RuntimeError): dispatcher.run(flow_name)
def test_multiple_failures_from_subflow(self): # # flow2: # Task0 Task1 X Task1 X # # flow1: # # flow2X..... Task3 # # Note: # flow2 will fail with two tasks of type Task1. flow1 defines fallback as True, so # it should recover from failure edge_table = { 'flow1': [{ 'from': [], 'to': ['flow2'], 'condition': self.cond_true }], 'flow2': [] } # flow2 handled manually failures = { 'flow1': { 'flow2': { 'next': {}, 'fallback': [True], 'conditions': [self.cond_true], 'condition_strs': ['cond_true'] } }, 'flow2': {} } self.init(edge_table, failures=failures) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert 'flow2' in self.instantiated_flows assert retry is not None assert system_state.node_args is None flow_info = { 'finished_nodes': { 'Task0': ['<id-tak0_0>'] }, 'failed_nodes': { 'Task2': ['<id-task1_0>', '<id-task1_1>'] } } flow2 = self.get_flow('flow2') self.set_failed(flow2, FlowError(flow_info)) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() assert retry is None
def test_singe_failure_flow_fallback(self): # # flow1: # # flow2 X ... Task2 # | # | # Task1 # # Note: # flow2 will fail # edge_table = { 'flow1': [{'from': ['flow2'], 'to': ['Task1'], 'condition': self.cond_true}, {'from': [], 'to': ['flow2'], 'condition': self.cond_true}], 'flow2': [] } failures = { 'flow1': {'flow2': {'next:': {}, 'fallback': [['Task2']], 'conditions': [self.cond_true], 'condition_strs': ['cond_true'] }} } self.init(edge_table, failures=failures) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert 'Task1' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 # flow2 has failed flow2 = self.get_flow('flow2') flow_info = {'finished_nodes': {'Task1': ['id_task1']}, 'failed_nodes': {'Task2': ['id_task21']}} self.set_failed(flow2, FlowError(flow_info)) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert 'Task2' in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0
def test_selinon_retry(self): def my_retry(kwargs=None, max_retries=None, countdown=None, queue=None, exc=None): kwargs = kwargs or {} assert kwargs.get('flow_name') == flow_name assert kwargs.get('node_args') == node_args assert 'parent' in kwargs assert 'selective' in kwargs assert 'state' in kwargs assert max_retries is None assert countdown == 5 assert queue == 'queue_flow1' assert exc is None raise RuntimeError() # we re-raise as stated in Celery doc flow_name = 'flow1' node_args = {'foo': 'bar'} edge_table = { flow_name: [{ 'from': [], 'to': ['Task1'], 'condition': self.cond_true }] } self.init(edge_table, retry_countdown={'flow1': 5}, max_retry={'flow1': 1}) state_dict = { 'finished_nodes': { 'Task1': ['<task1-id>'] }, 'failed_nodes': {} } flexmock(SystemState).should_receive('update').and_raise( FlowError(state_dict)) dispatcher = Dispatcher() flexmock(dispatcher).should_receive('retry').replace_with(my_retry) dispatcher.request = RequestMock() with pytest.raises(RuntimeError): dispatcher.run(flow_name, node_args)