def test_run(self):
flow_name = 'flow1'
edge_table = {
flow_name: [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
self.init(edge_table)
state_dict = {
'failed_nodes': {},
'finished_nodes': {
'Task1': ['<task-id>']
},
'active_nodes': []
}
flexmock(SystemState).should_receive('update').and_return(None)
flexmock(SystemState).should_receive('to_dict').and_return(state_dict)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
assert dispatcher.run(flow_name) == state_dict
def test_selinon_retry(self):
def my_retry(kwargs, max_retries, countdown, queue):
assert kwargs.get('flow_name') == flow_name
assert kwargs.get('node_args') == node_args
assert 'parent' in kwargs
assert kwargs.get('retried_count') == 1
assert max_retries == 1
assert countdown == 5
assert queue == 'queue_flow1'
raise RuntimeError() # we re-raise as stated in Celery doc
flow_name = 'flow1'
node_args = {'foo': 'bar'}
edge_table = {
flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table, retry_countdown={'flow1': 5}, max_retry={'flow1': 1})
state_dict = {'finished_nodes': {'Task1': ['<task1-id>']}, 'failed_nodes': {}}
flexmock(SystemState).should_receive('update').and_raise(FlowError(json.dumps(state_dict)))
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').replace_with(my_retry)
with pytest.raises(RuntimeError):
dispatcher.run(flow_name, node_args)
def test_flow_error_retry(self):
def my_retry(max_retries, exc):
assert max_retries == 0
assert json.loads(str(exc)) == json.loads(str(raised_exc))
raise RuntimeError() # we re-raise as stated in Celery doc
flow_name = 'flow1'
edge_table = {
flow_name: [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
self.init(edge_table)
state_dict = {
'finished_nodes': {},
'failed_nodes': {},
'active_nodes': []
}
raised_exc = FlowError(state_dict)
flexmock(SystemState).should_receive('update').and_raise(
raised_exc) # we will retry
flexmock(SystemState).should_receive('to_dict').and_return(state_dict)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').replace_with(my_retry)
with pytest.raises(RuntimeError):
dispatcher.run(flow_name)
def test_selinon_retry(self):
def my_retry(kwargs=None, max_retries=None, countdown=None, queue=None, exc=None):
kwargs = kwargs or {}
assert kwargs.get('flow_name') == flow_name
assert kwargs.get('node_args') == node_args
assert 'parent' in kwargs
assert 'selective' in kwargs
assert 'state' in kwargs
assert max_retries is None
assert countdown == 5
assert queue == 'queue_flow1'
assert exc is None
raise RuntimeError() # we re-raise as stated in Celery doc
flow_name = 'flow1'
node_args = {'foo': 'bar'}
edge_table = {
flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table, retry_countdown={'flow1': 5}, max_retry={'flow1': 1})
state_dict = {'finished_nodes': {'Task1': ['<task1-id>']}, 'failed_nodes': {}}
flexmock(SystemState).should_receive('update').and_raise(FlowError(state_dict))
dispatcher = Dispatcher()
flexmock(dispatcher).should_receive('retry').replace_with(my_retry)
dispatcher.request = RequestMock()
with pytest.raises(RuntimeError):
dispatcher.run(flow_name, node_args)
def test_run(self):
flow_name = 'flow1'
edge_table = {
flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
state_dict = {'failed_nodes': {}, 'finished_nodes': {'Task1': ['<task-id>']}, 'active_nodes': []}
flexmock(SystemState).should_receive('update').and_return(None)
flexmock(SystemState).should_receive('to_dict').and_return(state_dict)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
assert dispatcher.run(flow_name) == state_dict
def test_dispacher_error(self):
flow_name = 'flow1'
edge_table = {
flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
exc = KeyError("some exception in dispatcher")
flexmock(SystemState).should_receive('update').and_raise(exc)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').and_return(exc)
# We should improve this by actually checking exception value
with pytest.raises(KeyError):
dispatcher.run(flow_name)
def test_dispacher_error(self):
flow_name = 'flow1'
edge_table = {
flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
exc = KeyError("some exception in dispatcher")
flexmock(SystemState).should_receive('update').and_raise(exc)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').and_return(exc)
# We should improve this by actually checking exception value
with pytest.raises(KeyError):
dispatcher.run(flow_name)
def test_retry(self):
def my_retry(args, kwargs, countdown, queue):
assert args == []
assert countdown == 2
assert queue == 'queue_flow1'
state = kwargs.pop('state')
assert set(state.keys()) == {
'active_nodes', 'failed_nodes', 'finished_nodes',
'waiting_edges', 'triggered_edges'
}
assert len(state['active_nodes']) == 1
assert set(state['active_nodes'][0].keys()) == {'id', 'name'}
assert not state['failed_nodes']
assert not state['finished_nodes']
assert not state['waiting_edges']
assert len(state['triggered_edges']) == 1
assert kwargs == {
'flow_name': 'flow1',
'migration_version': 0,
'node_args': None,
'parent': None,
'retried_count': 0,
'retry': 2,
'selective': False
}
# Celery will raise Celery's retry causing dispatcher to retry, but let's simulate
# RuntimeError here for testing
raise RuntimeError()
flow_name = 'flow1'
edge_table = {
flow_name: [{
'from': [],
'to': ['Task1'],
'condition': self.cond_true
}]
}
self.init(edge_table)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').replace_with(my_retry)
with pytest.raises(RuntimeError):
dispatcher.run(flow_name)
def test_flow_error(self):
flow_name = 'flow1'
edge_table = {
flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
state_dict = {'failed_nodes': {'Task1': ['<task1-id>']}, 'finished_nodes': {}}
flexmock(SystemState).should_receive('update').and_raise(FlowError(json.dumps(state_dict)))
flexmock(SystemState).should_receive('to_dict').and_return(state_dict)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').and_return(FlowError)
# We should improve this by actually checking exception value
with pytest.raises(FlowError):
dispatcher.run(flow_name)
def test_retry(self):
def my_retry(args, kwargs, countdown, queue):
assert args == []
assert countdown == 2
assert queue == 'queue_flow1'
state = kwargs.pop('state')
assert set(state.keys()) == {
'active_nodes',
'failed_nodes',
'finished_nodes',
'waiting_edges',
'triggered_edges'
}
assert len(state['active_nodes']) == 1
assert set(state['active_nodes'][0].keys()) == {'id', 'name'}
assert not state['failed_nodes']
assert not state['finished_nodes']
assert not state['waiting_edges']
assert len(state['triggered_edges']) == 1
assert kwargs == {
'flow_name': 'flow1',
'migration_version': 0,
'node_args': None,
'parent': None,
'retried_count': 0,
'retry': 2,
'selective': False
}
# Celery will raise Celery's retry causing dispatcher to retry, but let's simulate
# RuntimeError here for testing
raise RuntimeError()
flow_name = 'flow1'
edge_table = {
flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').replace_with(my_retry)
with pytest.raises(RuntimeError):
dispatcher.run(flow_name)
def test_flow_error_retry(self):
def my_retry(max_retries, exc):
assert max_retries == 0
assert json.loads(str(exc)) == json.loads(str(raised_exc))
raise RuntimeError() # we re-raise as stated in Celery doc
flow_name = 'flow1'
edge_table = {
flow_name: [{'from': [], 'to': ['Task1'], 'condition': self.cond_true}]
}
self.init(edge_table)
state_dict = {'finished_nodes': {}, 'failed_nodes': {}, 'active_nodes': []}
raised_exc = FlowError(state_dict)
flexmock(SystemState).should_receive('update').and_raise(raised_exc) # we will retry
flexmock(SystemState).should_receive('to_dict').and_return(state_dict)
dispatcher = Dispatcher()
dispatcher.request = RequestMock()
flexmock(dispatcher).should_receive('retry').replace_with(my_retry)
with pytest.raises(RuntimeError):
dispatcher.run(flow_name)
def test_propagate_compound_mixed(self):
#
# flow1:
#
# flow2 flow3
# | |
# -----------
# |
# TaskX
#
# flow2:
# Not run explicitly, but result finished_nodes is:
# {'Task2': [<task2-id21>],
# 'Task3': [<task3-id21>, <task3-id22>]}
# 'flow4': {'Task2': [<task2-id41>]}
# flow3:
# Not run explicitly, but result finished_nodes is:
# {'Task2': [<task2-id32>],
# 'Task3': [<task3-id31>, <task3-id32>],
# 'flow4': {'Task2': [<task2-id42>]}
#
# We are propagating finished, so we should inspect parent
#
edge_table = {
'flow1': [{'from': [], 'to': ['flow2', 'flow3'], 'condition': self.cond_true},
{'from': ['flow2', 'flow3'], 'to': ['TaskX'], 'condition': self.cond_true}],
'flow2': [],
'flow3': [],
'flow4': []
}
# Make sure propagate_finished is negated propagate_compound_finished
# this is checked in selinon
self.init(edge_table,
propagate_finished={'flow1': ['flow2']},
propagate_compound_finished={'flow1': ['flow3']})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict = system_state.to_dict()
assert retry is not None
assert 'flow2' in self.instantiated_flows
# Create flow4 manually, we will reuse it, but we pretend that there are 2 instances - one run in flow2
# another one in flow3
flow4 = Dispatcher().apply_async(kwargs={'flow_name': 'flow4'}, queue=Config.dispatcher_queues['flow4'])
self.get_task_instance.register_node(flow4)
self.set_finished(flow4, {'finished_nodes': {'Task2': ['<task2-id41']}, 'failed_nodes': {}})
# Create results of flow2 and flow3 manually but they are instantiated by dispatcher
flow2_result = {'finished_nodes': {'Task2': ['<task2-id21>'],
'Task3': ['<task3-id21>', '<task3-id22>'],
'flow4': [flow4.task_id]},
'failed_nodes': {}
}
flow2 = self.get_flow('flow2')
self.set_finished(flow2, flow2_result)
flow3_result = {'finished_nodes': {'Task2': ['<task2-id32>'],
'Task3': ['<task3-id31>', '<task3-id32>'],
'flow4': [flow4.task_id]},
'failed_nodes': {}
}
flow3 = self.get_flow('flow3')
self.set_finished(flow3, flow3_result)
system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
assert 'TaskX' in self.instantiated_tasks
task_x_parent = {'flow2': {'Task2': ['<task2-id21>'],
'Task3': ['<task3-id21>', '<task3-id22>'],
'flow4': {'Task2': ['<task2-id41']}},
# flow4 is hidden in flow3 as it is compound, see 'Task2'
'flow3': {'Task2': ['<task2-id41', '<task2-id32>'],
'Task3': ['<task3-id31>', '<task3-id32>']}}
task_x = self.get_task('TaskX')
# we have to check this as a set due to dict randomization
assert 'flow2' in task_x.parent
assert 'flow3' in task_x.parent
assert 'Task2' in task_x.parent['flow2']
assert 'Task3' in task_x.parent['flow2']
assert set(task_x.parent['flow2']['Task2']) == set(task_x_parent['flow2']['Task2'])
assert set(task_x.parent['flow2']['Task3']) == set(task_x_parent['flow2']['Task3'])
assert 'Task2' in task_x.parent['flow3']
assert 'Task3' in task_x.parent['flow3']
assert set(task_x.parent['flow3']['Task2']) == set(task_x_parent['flow3']['Task2'])
assert set(task_x.parent['flow3']['Task3']) == set(task_x_parent['flow3']['Task3'])
assert 'flow4' in task_x.parent['flow2']
assert set(task_x.parent['flow2']['flow4']) == set(task_x_parent['flow2']['flow4'])
def test_propagate_parent_2(self):
#
# flow1:
#
# Task1 Task2
# | |
# flow2 |
# | |
# -----------
# |
# TaskX
#
# flow2:
# Not run explicitly, but result finished_nodes is:
# {'flow3': [<flow3-id>], 'Task2': [<task2-id1>], 'Task3': [<task3-id1>, <task3-id2>]}
# flow3:
# Not run explicitly, but result finished_nodes is:
# {'Task2': [<task2-id2>], 'Task4': [<task4-id1>, <task4-id2>]}
#
# We are propagating finished, so we should inspect parent
#
edge_table = {
'flow1': [{'from': ['flow2', 'Task2'], 'to': ['TaskX'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}],
'flow2': [],
'flow3': []
}
# Make sure propagate_finished is set to False as they are disjoint with propagate_compound_finished;
# this is checked in selinon
self.init(edge_table, propagate_parent=dict.fromkeys(edge_table.keys(), True),
propagate_finished={'flow1': False},
propagate_compound_finished={'flow1': True})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'flow2' not in self.instantiated_flows
# Task1 and Task2 have finished
task1 = self.get_task('Task1')
self.set_finished(task1, None)
task2 = self.get_task('Task2')
self.set_finished(task2, None)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert 'flow2' in self.instantiated_flows
# Create flow3 manually
flow3 = Dispatcher().apply_async(kwargs={'flow_name': 'flow3'}, queue=Config.dispatcher_queues['flow3'])
self.get_task_instance.register_node(flow3)
flow2 = self.get_flow('flow2')
self.set_finished(flow2)
flow2_result = {'finished_nodes': {'flow3': [flow3.task_id],
'Task2': ['<task2-id1>'],
'Task3': ['<task3-id1>', '<task3-id2>']},
'failed_nodes': {}
}
self.set_finished(flow2, flow2_result)
flow3_result = {'finished_nodes': {'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>']},
'failed_nodes': {}
}
self.set_finished(flow3, flow3_result)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
task_x = self.get_task('TaskX')
task_x_parent = {'Task2': task2.task_id,
'flow2': {'Task2': ['<task2-id1>', '<task2-id2>'],
'Task3': ['<task3-id1>', '<task3-id2>'],
'Task4': ['<task4-id1>', '<task4-id2>']
}
}
# we have to check this as a set due to dict randomization
assert 'Task2' in task_x.parent
assert task_x.parent['Task2'] == task_x_parent['Task2']
assert 'flow2' in task_x.parent
assert 'Task2' in task_x.parent['flow2']
assert 'Task3' in task_x.parent['flow2']
assert 'Task4' in task_x.parent['flow2']
assert set(task_x.parent['flow2']['Task2']) == set(task_x_parent['flow2']['Task2'])
assert set(task_x.parent['flow2']['Task3']) == set(task_x_parent['flow2']['Task3'])
assert set(task_x.parent['flow2']['Task4']) == set(task_x_parent['flow2']['Task4'])
def test_propagate_parent(self):
#
# flow1:
#
# Task1 Task2
# | |
# flow2 |
# | |
# -----------
# |
# TaskX
#
# flow2:
# Run explicitly, but result finished_nodes is:
# {'flow3': [<flow3-id>], 'Task2': [<task2-id1>], 'Task3': [<task3-id1>, <task3-id2>]}
# flow3
# Run explicitly, but result finished_nodes is:
# {'Task2': [<task2-id2>], 'Task4': [<task4-id1>, <task4-id2>]}
#
# We are propagating finished, so we should inspect parent
#
edge_table = {
'flow1': [{'from': ['flow2', 'Task2'], 'to': ['TaskX'], 'condition': self.cond_true},
{'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true},
{'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}],
'flow2': [],
'flow3': []
}
self.init(edge_table, propagate_parent={'flow1': True}, propagate_finished={'flow1': True})
system_state = SystemState(id(self), 'flow1')
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert system_state.node_args is None
assert 'Task1' in self.instantiated_tasks
assert 'Task2' in self.instantiated_tasks
assert 'flow2' not in self.instantiated_flows
# Task1 and Task2 have finished
task1 = self.get_task('Task1')
self.set_finished(task1, None)
task2 = self.get_task('Task2')
self.set_finished(task2, None)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
state_dict_1 = system_state.to_dict()
assert retry is not None
assert 'flow2' in self.instantiated_flows
# Create flow3 manually
flow3 = Dispatcher().apply_async(kwargs={'flow_name': 'flow3'}, queue=Config.dispatcher_queues['flow3'])
self.get_task_instance.register_node(flow3)
flow2 = self.get_flow('flow2')
self.set_finished(flow2)
flow2_result = {'finished_nodes': {'flow3': [flow3.task_id],
'Task2': ['<task2-id1>'],
'Task3': ['<task3-id1>', '<task3-id2>']},
'failed_nodes': {}
}
self.set_finished(flow2, flow2_result)
flow3_result = {'finished_nodes':
{'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>']},
'failed_nodes': {}
}
self.set_finished(flow3, flow3_result)
system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args)
retry = system_state.update()
assert retry is not None
task_x = self.get_task('TaskX')
task_x_parent = {'Task2': task2.task_id,
'flow2': {'Task2': ['<task2-id1>'],
'Task3': ['<task3-id1>', '<task3-id2>'],
'flow3': {'Task2': ['<task2-id2>'],
'Task4': ['<task4-id1>', '<task4-id2>'],
}
}
}
assert task_x.parent == task_x_parent
