def test_run(self): flow_name = 'flow1' edge_table = { flow_name: [{ 'from': [], 'to': ['Task1'], 'condition': self.cond_true }] } self.init(edge_table) state_dict = { 'failed_nodes': {}, 'finished_nodes': { 'Task1': ['<task-id>'] }, 'active_nodes': [] } flexmock(SystemState).should_receive('update').and_return(None) flexmock(SystemState).should_receive('to_dict').and_return(state_dict) dispatcher = Dispatcher() dispatcher.request = RequestMock() assert dispatcher.run(flow_name) == state_dict
def test_selinon_retry(self): def my_retry(kwargs, max_retries, countdown, queue): assert kwargs.get('flow_name') == flow_name assert kwargs.get('node_args') == node_args assert 'parent' in kwargs assert kwargs.get('retried_count') == 1 assert max_retries == 1 assert countdown == 5 assert queue == 'queue_flow1' raise RuntimeError() # we re-raise as stated in Celery doc flow_name = 'flow1' node_args = {'foo': 'bar'} edge_table = { flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}] } self.init(edge_table, retry_countdown={'flow1': 5}, max_retry={'flow1': 1}) state_dict = {'finished_nodes': {'Task1': ['<task1-id>']}, 'failed_nodes': {}} flexmock(SystemState).should_receive('update').and_raise(FlowError(json.dumps(state_dict))) dispatcher = Dispatcher() dispatcher.request = RequestMock() flexmock(dispatcher).should_receive('retry').replace_with(my_retry) with pytest.raises(RuntimeError): dispatcher.run(flow_name, node_args)
def test_flow_error_retry(self): def my_retry(max_retries, exc): assert max_retries == 0 assert json.loads(str(exc)) == json.loads(str(raised_exc)) raise RuntimeError() # we re-raise as stated in Celery doc flow_name = 'flow1' edge_table = { flow_name: [{ 'from': [], 'to': ['Task1'], 'condition': self.cond_true }] } self.init(edge_table) state_dict = { 'finished_nodes': {}, 'failed_nodes': {}, 'active_nodes': [] } raised_exc = FlowError(state_dict) flexmock(SystemState).should_receive('update').and_raise( raised_exc) # we will retry flexmock(SystemState).should_receive('to_dict').and_return(state_dict) dispatcher = Dispatcher() dispatcher.request = RequestMock() flexmock(dispatcher).should_receive('retry').replace_with(my_retry) with pytest.raises(RuntimeError): dispatcher.run(flow_name)
def test_selinon_retry(self): def my_retry(kwargs=None, max_retries=None, countdown=None, queue=None, exc=None): kwargs = kwargs or {} assert kwargs.get('flow_name') == flow_name assert kwargs.get('node_args') == node_args assert 'parent' in kwargs assert 'selective' in kwargs assert 'state' in kwargs assert max_retries is None assert countdown == 5 assert queue == 'queue_flow1' assert exc is None raise RuntimeError() # we re-raise as stated in Celery doc flow_name = 'flow1' node_args = {'foo': 'bar'} edge_table = { flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}] } self.init(edge_table, retry_countdown={'flow1': 5}, max_retry={'flow1': 1}) state_dict = {'finished_nodes': {'Task1': ['<task1-id>']}, 'failed_nodes': {}} flexmock(SystemState).should_receive('update').and_raise(FlowError(state_dict)) dispatcher = Dispatcher() flexmock(dispatcher).should_receive('retry').replace_with(my_retry) dispatcher.request = RequestMock() with pytest.raises(RuntimeError): dispatcher.run(flow_name, node_args)
def test_run(self): flow_name = 'flow1' edge_table = { flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}] } self.init(edge_table) state_dict = {'failed_nodes': {}, 'finished_nodes': {'Task1': ['<task-id>']}, 'active_nodes': []} flexmock(SystemState).should_receive('update').and_return(None) flexmock(SystemState).should_receive('to_dict').and_return(state_dict) dispatcher = Dispatcher() dispatcher.request = RequestMock() assert dispatcher.run(flow_name) == state_dict
def test_dispacher_error(self): flow_name = 'flow1' edge_table = { flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}] } self.init(edge_table) exc = KeyError("some exception in dispatcher") flexmock(SystemState).should_receive('update').and_raise(exc) dispatcher = Dispatcher() dispatcher.request = RequestMock() flexmock(dispatcher).should_receive('retry').and_return(exc) # We should improve this by actually checking exception value with pytest.raises(KeyError): dispatcher.run(flow_name)
def test_retry(self): def my_retry(args, kwargs, countdown, queue): assert args == [] assert countdown == 2 assert queue == 'queue_flow1' state = kwargs.pop('state') assert set(state.keys()) == { 'active_nodes', 'failed_nodes', 'finished_nodes', 'waiting_edges', 'triggered_edges' } assert len(state['active_nodes']) == 1 assert set(state['active_nodes'][0].keys()) == {'id', 'name'} assert not state['failed_nodes'] assert not state['finished_nodes'] assert not state['waiting_edges'] assert len(state['triggered_edges']) == 1 assert kwargs == { 'flow_name': 'flow1', 'migration_version': 0, 'node_args': None, 'parent': None, 'retried_count': 0, 'retry': 2, 'selective': False } # Celery will raise Celery's retry causing dispatcher to retry, but let's simulate # RuntimeError here for testing raise RuntimeError() flow_name = 'flow1' edge_table = { flow_name: [{ 'from': [], 'to': ['Task1'], 'condition': self.cond_true }] } self.init(edge_table) dispatcher = Dispatcher() dispatcher.request = RequestMock() flexmock(dispatcher).should_receive('retry').replace_with(my_retry) with pytest.raises(RuntimeError): dispatcher.run(flow_name)
def test_flow_error(self): flow_name = 'flow1' edge_table = { flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}] } self.init(edge_table) state_dict = {'failed_nodes': {'Task1': ['<task1-id>']}, 'finished_nodes': {}} flexmock(SystemState).should_receive('update').and_raise(FlowError(json.dumps(state_dict))) flexmock(SystemState).should_receive('to_dict').and_return(state_dict) dispatcher = Dispatcher() dispatcher.request = RequestMock() flexmock(dispatcher).should_receive('retry').and_return(FlowError) # We should improve this by actually checking exception value with pytest.raises(FlowError): dispatcher.run(flow_name)
def test_retry(self): def my_retry(args, kwargs, countdown, queue): assert args == [] assert countdown == 2 assert queue == 'queue_flow1' state = kwargs.pop('state') assert set(state.keys()) == { 'active_nodes', 'failed_nodes', 'finished_nodes', 'waiting_edges', 'triggered_edges' } assert len(state['active_nodes']) == 1 assert set(state['active_nodes'][0].keys()) == {'id', 'name'} assert not state['failed_nodes'] assert not state['finished_nodes'] assert not state['waiting_edges'] assert len(state['triggered_edges']) == 1 assert kwargs == { 'flow_name': 'flow1', 'migration_version': 0, 'node_args': None, 'parent': None, 'retried_count': 0, 'retry': 2, 'selective': False } # Celery will raise Celery's retry causing dispatcher to retry, but let's simulate # RuntimeError here for testing raise RuntimeError() flow_name = 'flow1' edge_table = { flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}] } self.init(edge_table) dispatcher = Dispatcher() dispatcher.request = RequestMock() flexmock(dispatcher).should_receive('retry').replace_with(my_retry) with pytest.raises(RuntimeError): dispatcher.run(flow_name)
def test_flow_error_retry(self): def my_retry(max_retries, exc): assert max_retries == 0 assert json.loads(str(exc)) == json.loads(str(raised_exc)) raise RuntimeError() # we re-raise as stated in Celery doc flow_name = 'flow1' edge_table = { flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}] } self.init(edge_table) state_dict = {'finished_nodes': {}, 'failed_nodes': {}, 'active_nodes': []} raised_exc = FlowError(state_dict) flexmock(SystemState).should_receive('update').and_raise(raised_exc) # we will retry flexmock(SystemState).should_receive('to_dict').and_return(state_dict) dispatcher = Dispatcher() dispatcher.request = RequestMock() flexmock(dispatcher).should_receive('retry').replace_with(my_retry) with pytest.raises(RuntimeError): dispatcher.run(flow_name)
def test_propagate_compound_mixed(self): # # flow1: # # flow2 flow3 # | | # ----------- # | # TaskX # # flow2: # Not run explicitly, but result finished_nodes is: # {'Task2': [<task2-id21>], # 'Task3': [<task3-id21>, <task3-id22>]} # 'flow4': {'Task2': [<task2-id41>]} # flow3: # Not run explicitly, but result finished_nodes is: # {'Task2': [<task2-id32>], # 'Task3': [<task3-id31>, <task3-id32>], # 'flow4': {'Task2': [<task2-id42>]} # # We are propagating finished, so we should inspect parent # edge_table = { 'flow1': [{'from': [], 'to': ['flow2', 'flow3'], 'condition': self.cond_true}, {'from': ['flow2', 'flow3'], 'to': ['TaskX'], 'condition': self.cond_true}], 'flow2': [], 'flow3': [], 'flow4': [] } # Make sure propagate_finished is negated propagate_compound_finished # this is checked in selinon self.init(edge_table, propagate_finished={'flow1': ['flow2']}, propagate_compound_finished={'flow1': ['flow3']}) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert 'flow2' in self.instantiated_flows # Create flow4 manually, we will reuse it, but we pretend that there are 2 instances - one run in flow2 # another one in flow3 flow4 = Dispatcher().apply_async(kwargs={'flow_name': 'flow4'}, queue=Config.dispatcher_queues['flow4']) self.get_task_instance.register_node(flow4) self.set_finished(flow4, {'finished_nodes': {'Task2': ['<task2-id41']}, 'failed_nodes': {}}) # Create results of flow2 and flow3 manually but they are instantiated by dispatcher flow2_result = {'finished_nodes': {'Task2': ['<task2-id21>'], 'Task3': ['<task3-id21>', '<task3-id22>'], 'flow4': [flow4.task_id]}, 'failed_nodes': {} } flow2 = self.get_flow('flow2') self.set_finished(flow2, flow2_result) flow3_result = {'finished_nodes': {'Task2': ['<task2-id32>'], 'Task3': ['<task3-id31>', '<task3-id32>'], 'flow4': [flow4.task_id]}, 'failed_nodes': {} } flow3 = self.get_flow('flow3') self.set_finished(flow3, flow3_result) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() assert retry is not None assert 'TaskX' in self.instantiated_tasks task_x_parent = {'flow2': {'Task2': ['<task2-id21>'], 'Task3': ['<task3-id21>', '<task3-id22>'], 'flow4': {'Task2': ['<task2-id41']}}, # flow4 is hidden in flow3 as it is compound, see 'Task2' 'flow3': {'Task2': ['<task2-id41', '<task2-id32>'], 'Task3': ['<task3-id31>', '<task3-id32>']}} task_x = self.get_task('TaskX') # we have to check this as a set due to dict randomization assert 'flow2' in task_x.parent assert 'flow3' in task_x.parent assert 'Task2' in task_x.parent['flow2'] assert 'Task3' in task_x.parent['flow2'] assert set(task_x.parent['flow2']['Task2']) == set(task_x_parent['flow2']['Task2']) assert set(task_x.parent['flow2']['Task3']) == set(task_x_parent['flow2']['Task3']) assert 'Task2' in task_x.parent['flow3'] assert 'Task3' in task_x.parent['flow3'] assert set(task_x.parent['flow3']['Task2']) == set(task_x_parent['flow3']['Task2']) assert set(task_x.parent['flow3']['Task3']) == set(task_x_parent['flow3']['Task3']) assert 'flow4' in task_x.parent['flow2'] assert set(task_x.parent['flow2']['flow4']) == set(task_x_parent['flow2']['flow4'])
def test_propagate_parent_2(self): # # flow1: # # Task1 Task2 # | | # flow2 | # | | # ----------- # | # TaskX # # flow2: # Not run explicitly, but result finished_nodes is: # {'flow3': [<flow3-id>], 'Task2': [<task2-id1>], 'Task3': [<task3-id1>, <task3-id2>]} # flow3: # Not run explicitly, but result finished_nodes is: # {'Task2': [<task2-id2>], 'Task4': [<task4-id1>, <task4-id2>]} # # We are propagating finished, so we should inspect parent # edge_table = { 'flow1': [{'from': ['flow2', 'Task2'], 'to': ['TaskX'], 'condition': self.cond_true}, {'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true}, {'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}], 'flow2': [], 'flow3': [] } # Make sure propagate_finished is set to False as they are disjoint with propagate_compound_finished; # this is checked in selinon self.init(edge_table, propagate_parent=dict.fromkeys(edge_table.keys(), True), propagate_finished={'flow1': False}, propagate_compound_finished={'flow1': True}) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'flow2' not in self.instantiated_flows # Task1 and Task2 have finished task1 = self.get_task('Task1') self.set_finished(task1, None) task2 = self.get_task('Task2') self.set_finished(task2, None) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert 'flow2' in self.instantiated_flows # Create flow3 manually flow3 = Dispatcher().apply_async(kwargs={'flow_name': 'flow3'}, queue=Config.dispatcher_queues['flow3']) self.get_task_instance.register_node(flow3) flow2 = self.get_flow('flow2') self.set_finished(flow2) flow2_result = {'finished_nodes': {'flow3': [flow3.task_id], 'Task2': ['<task2-id1>'], 'Task3': ['<task3-id1>', '<task3-id2>']}, 'failed_nodes': {} } self.set_finished(flow2, flow2_result) flow3_result = {'finished_nodes': {'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>']}, 'failed_nodes': {} } self.set_finished(flow3, flow3_result) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() assert retry is not None task_x = self.get_task('TaskX') task_x_parent = {'Task2': task2.task_id, 'flow2': {'Task2': ['<task2-id1>', '<task2-id2>'], 'Task3': ['<task3-id1>', '<task3-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>'] } } # we have to check this as a set due to dict randomization assert 'Task2' in task_x.parent assert task_x.parent['Task2'] == task_x_parent['Task2'] assert 'flow2' in task_x.parent assert 'Task2' in task_x.parent['flow2'] assert 'Task3' in task_x.parent['flow2'] assert 'Task4' in task_x.parent['flow2'] assert set(task_x.parent['flow2']['Task2']) == set(task_x_parent['flow2']['Task2']) assert set(task_x.parent['flow2']['Task3']) == set(task_x_parent['flow2']['Task3']) assert set(task_x.parent['flow2']['Task4']) == set(task_x_parent['flow2']['Task4'])
def test_propagate_parent(self): # # flow1: # # Task1 Task2 # | | # flow2 | # | | # ----------- # | # TaskX # # flow2: # Run explicitly, but result finished_nodes is: # {'flow3': [<flow3-id>], 'Task2': [<task2-id1>], 'Task3': [<task3-id1>, <task3-id2>]} # flow3 # Run explicitly, but result finished_nodes is: # {'Task2': [<task2-id2>], 'Task4': [<task4-id1>, <task4-id2>]} # # We are propagating finished, so we should inspect parent # edge_table = { 'flow1': [{'from': ['flow2', 'Task2'], 'to': ['TaskX'], 'condition': self.cond_true}, {'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true}, {'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}], 'flow2': [], 'flow3': [] } self.init(edge_table, propagate_parent={'flow1': True}, propagate_finished={'flow1': True}) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'flow2' not in self.instantiated_flows # Task1 and Task2 have finished task1 = self.get_task('Task1') self.set_finished(task1, None) task2 = self.get_task('Task2') self.set_finished(task2, None) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert 'flow2' in self.instantiated_flows # Create flow3 manually flow3 = Dispatcher().apply_async(kwargs={'flow_name': 'flow3'}, queue=Config.dispatcher_queues['flow3']) self.get_task_instance.register_node(flow3) flow2 = self.get_flow('flow2') self.set_finished(flow2) flow2_result = {'finished_nodes': {'flow3': [flow3.task_id], 'Task2': ['<task2-id1>'], 'Task3': ['<task3-id1>', '<task3-id2>']}, 'failed_nodes': {} } self.set_finished(flow2, flow2_result) flow3_result = {'finished_nodes': {'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>']}, 'failed_nodes': {} } self.set_finished(flow3, flow3_result) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() assert retry is not None task_x = self.get_task('TaskX') task_x_parent = {'Task2': task2.task_id, 'flow2': {'Task2': ['<task2-id1>'], 'Task3': ['<task3-id1>', '<task3-id2>'], 'flow3': {'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>'], } } } assert task_x.parent == task_x_parent