def test_single_failure_finish_wait(self): # # flow1: # # Task1 Task2 X # | | # | | # Task3 Task4 # # No fallback defined for Task2 # # Note: # Task2 will fail, no fallback defined, Dispatcher should wait for Task1 to finish and # raise FlowError exception edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true }, { 'from': ['Task2'], 'to': ['Task4'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true }], } failures = { 'flow1': { 'Task1': { 'next': { 'Task2': { 'next': {}, 'fallback': [] } }, 'fallback': [] }, 'Task2': { 'next': { 'Task1': { 'next': {}, 'fallback': [] } }, 'fallback': [] } } } self.init(edge_table, failures=failures) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] # Task2 has failed task2 = self.get_task('Task2') self.set_failed(task2) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] assert 'Task2' in state_dict['failed_nodes'] # No change so far, still wait system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] assert 'Task2' in state_dict['failed_nodes'] # Task1 has finished successfully task1 = self.get_task('Task1') self.set_finished(task1, 0) # Wait for Task3 system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) system_state.update() state_dict = system_state.to_dict() assert retry is not None assert 'Task3' in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks # Task3 has finished successfully task3 = self.get_task('Task3') self.set_finished(task3, 0) with pytest.raises(FlowError): system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) system_state.update()
def test_flow_failure(self): # # flow1: # # flow2 X flow4 X Task1_f1 Task2_f1 X flow5 # | | | | | # ................................................ # | # TaskX # # flow2: # # Task1_f2 Task2_f2 X Task3_f2 flow3 # # flow3: # # Task1_f3 Task2_f3 X Task3_f3 X # # flow4: # # Task1_f4 Task2_f4 X Task3_f4 # # flow5: # Task1_f5 Task2_f5 X # # Note: # All flows fail, we inspect finished propagation in flow1 in case of failure. Tasks marked with X will fail. # edge_table = { 'flow1': [{ 'from': ['flow2', 'flow5', 'flow4', 'Task1_f1', 'Task2_f1'], 'to': ['TaskX'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1_f1', 'Task2_f1', 'flow2', 'flow4', 'flow5'], 'condition': self.cond_true }], 'flow2': [{ 'from': [], 'to': ['Task1_f2', 'Task2_f2', 'Task3_f2', 'flow3'], 'condition': self.cond_true }], 'flow3': [{ 'from': [], 'to': ['Task1_f3', 'Task2_f3', 'Task3_f3'], 'condition': self.cond_true }], 'flow4': [{ 'from': [], 'to': ['Task1_f4', 'Task2_f4', 'Task3_f4'], 'condition': self.cond_true }], 'flow5': [{ 'from': [], 'to': ['Task1_f5', 'Task2_f5'], 'condition': self.cond_true }] } failures = { 'flow1': { 'Task2_f1': { 'next': { 'flow2': { 'next': { 'flow4': { 'next': {}, 'fallback': [['TaskX']], 'conditions': [self.cond_true], 'condition_strs': ['cond_true'] }, 'fallback': [] }, 'fallback': [] }, 'fallback': [] }, 'fallback': [] }, 'fallback': [] }, 'fallback': [] } self.init(edge_table, failures=failures, propagate_parent=dict.fromkeys(edge_table, True)) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert 'flow4' in self.instantiated_flows assert 'flow5' in self.instantiated_flows assert 'Task1_f1' in self.instantiated_tasks assert 'Task2_f1' in self.instantiated_tasks assert 'flow3' not in self.instantiated_flows # Task1_f1 have finished task1_f1 = self.get_task('Task1_f1') self.set_finished(task1_f1, 0) # Task2_f1 has failed task2_f1 = self.get_task('Task2_f1') self.set_failed(task2_f1) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'flow3' not in self.instantiated_flows # flow2 system_state = SystemState(id(self), 'flow2') retry = system_state.update() state_dict_2 = system_state.to_dict() flow2 = self.get_flow('flow2') assert retry is not None assert system_state.node_args is None assert 'Task1_f2' in self.instantiated_tasks assert 'Task2_f2' in self.instantiated_tasks assert 'Task3_f2' in self.instantiated_tasks assert 'flow3' in self.instantiated_flows # Task1_f2, Task3_f2 have finished task1_f2 = self.get_task('Task1_f2') self.set_finished(task1_f2, 1) task3_f2 = self.get_task('Task3_f2') self.set_finished(task3_f2, 2) # Task2_f2 has failed task2_f2 = self.get_task('Task2_f2') self.set_failed(task2_f2) system_state = SystemState(id(self), 'flow2', state=state_dict_2, node_args=system_state.node_args) retry = system_state.update() state_dict_2 = system_state.to_dict() assert retry is not None assert system_state.node_args is None # flow 3 system_state = SystemState(id(self), 'flow3') retry = system_state.update() state_dict_3 = system_state.to_dict() flow3 = self.get_flow('flow3') assert retry is not None assert system_state.node_args is None assert 'Task1_f3' in self.instantiated_tasks assert 'Task2_f3' in self.instantiated_tasks assert 'Task3_f3' in self.instantiated_tasks # Task1_f3 has finished task1_f3 = self.get_task('Task1_f3') self.set_finished(task1_f3, 3) # Task2_f3, Task2_f3 have failed task2_f3 = self.get_task('Task2_f3') self.set_failed(task2_f3) task3_f3 = self.get_task('Task3_f3') self.set_failed(task3_f3) try: system_state = SystemState(id(self), 'flow3', state=state_dict_3, node_args=system_state.node_args) system_state.update() except FlowError as exc: self.set_failed(flow3, exc) else: assert not "Expected FlowError not raised" # flow4 system_state = SystemState(id(self), 'flow4') retry = system_state.update() state_dict_4 = system_state.to_dict() flow4 = self.get_flow('flow4') assert retry is not None assert system_state.node_args is None assert 'Task1_f4' in self.instantiated_tasks assert 'Task2_f4' in self.instantiated_tasks assert 'Task3_f4' in self.instantiated_tasks # Task1_f2, Task3_f2 have finished task1_f4 = self.get_task('Task1_f4') self.set_finished(task1_f4, 1) task3_f4 = self.get_task('Task3_f4') self.set_finished(task3_f4, 4) # Task2_f4 has failed task2_f4 = self.get_task('Task2_f4') self.set_failed(task2_f4) try: system_state = SystemState(id(self), 'flow4', state=state_dict_4, node_args=system_state.node_args) system_state.update() except FlowError as exc: self.set_failed(flow4, exc) else: assert not "Expected FlowError not raised" # flow5 system_state = SystemState(id(self), 'flow5') retry = system_state.update() state_dict_5 = system_state.to_dict() flow5 = self.get_flow('flow5') assert retry is not None assert system_state.node_args is None assert 'Task1_f5' in self.instantiated_tasks assert 'Task2_f5' in self.instantiated_tasks # Task1_f5, Task2_f5 have finished task1_f5 = self.get_task('Task1_f5') self.set_finished(task1_f5, 1) task2_f5 = self.get_task('Task2_f5') self.set_finished(task2_f5, 2) system_state = SystemState(id(self), 'flow5', state=state_dict_5, node_args=system_state.node_args) retry = system_state.update() assert retry is None state_info = { 'finished_nodes': system_state.to_dict()['finished_nodes'], 'failed_nodes': system_state.to_dict()['failed_nodes'] } self.set_finished(flow5, state_info) # Back to flow2, all child finished try: system_state = SystemState(id(self), 'flow2', state=state_dict_2, node_args=system_state.node_args) system_state.update() except FlowError as exc: self.set_failed(flow2, exc) else: assert not "Expected FlowError not raised" # finally inspect flow1 result and finished that is propagated at fallback to TaskX assert 'TaskX' not in self.instantiated_tasks system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) system_state.update() assert 'TaskX' in self.instantiated_tasks task_x = self.get_task('TaskX') assert task_x.parent is None
def test_foreach_propagate_result(self): # # flow1: # # Task1 # | # | # --------------------- # | | | # | | | # flow2 flow2 ... flow2 # # Note: # There will be spawned _FOREACH_COUNT flow2, arguments are passed from foreach function # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true, 'foreach': lambda x, y: range(_FOREACH_COUNT), 'foreach_propagate_result': True }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }], 'flow2': [] } self.init(edge_table) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks # Task1 has finished task1 = self.get_task('Task1') self.set_finished(task1, "some result") system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'flow2' in self.instantiated_flows tasks_state_dict = [ node for node in state_dict['active_nodes'] if node['name'] == 'flow2' ] assert len(tasks_state_dict) == _FOREACH_COUNT # Inspect node_args as we set propagate_result for foreach all_flow_args = [ flow.node_args for flow in self.get_all_flows('flow2') ] assert all_flow_args == list(range(_FOREACH_COUNT))
def test_two_failures_fallback_start(self): # # flow1: # # Task1 X Task2 X # | | # | | # Task3 Task4 # # Note: # Task1 will fail, then Task2 will fail; fallback for Task1, Task2 is Task5 # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true }, { 'from': ['Task2'], 'to': ['Task4'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true }], } failures = { 'flow1': { 'Task1': { 'next': { 'Task2': { 'next': {}, 'fallback': [['Task5']], 'conditions': [self.cond_true], 'condition_strs': ['cond_true'], } }, 'fallback': [] }, 'Task2': { 'next': { 'Task1': { 'next': {}, 'fallback': [['Task5']], 'conditions': [self.cond_true], 'condition_strs': ['cond_true'], } }, 'fallback': [] } } } self.init(edge_table, failures=failures) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] task1 = self.get_task('Task1') self.set_failed(task1, KeyError("Some exception raised")) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] assert 'Task1' in state_dict['failed_nodes'] # Task2 has failed task2 = self.get_task('Task2') self.set_failed(task2) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] assert 'Task1' not in state_dict['failed_nodes'] assert 'Task2' not in state_dict['failed_nodes']
def test_retrieve(self): # # flow1: # # Task1 # | # | # Task2 # class MyStorage(DataStorage): def __init__(self): pass def connect(self): # shouldn't be called raise NotImplementedError() def disconnect(self): # called on destruction pass def is_connected(self): # return True so we can test retrieve() return True def store(self, node_args, flow_name, task_name, task_id, result): # shouldn't be called raise NotImplementedError() def retrieve(self, flow_name, task_name, task_id): assert flow_name == 'flow1' assert task_name == 'Task1' assert task_id == task1.task_id return 0xDEADBEEF def _cond_access(db, node_args): return db.get('Task1') == 0xDEADBEEF edge_table = { 'flow1': [{'from': ['Task1'], 'to': ['Task2'], 'condition': _cond_access}, {'from': [], 'to': ['Task1'], 'condition': self.cond_true}] } self.init(edge_table) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None # Task1 has finished, we should access the database task1 = self.get_task('Task1') self.set_finished(task1, 1) Config.storage_mapping = {'Storage1': MyStorage()} Config.task2storage_mapping = {'Task1': 'Storage1'} system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) system_state.update() assert 'Task2' in self.instantiated_tasks
def test_single_failure_flow_fallback_start2(self): # # flow1: # # Task1 X .... Task3, Task4 # | # | # Task2 # # Task3 Task4 # | | # | | # Task5 Task6 # # Note: # Task1 will fail # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }, { 'from': ['Task3'], 'to': ['Task5'], 'condition': self.cond_true }, { 'from': ['Task4'], 'to': ['Task6'], 'condition': self.cond_true }] } failures = { 'flow1': { 'Task1': { 'next:': {}, 'fallback': ['Task3', 'Task4'] } } } self.init(edge_table, failures=failures) node_args = {'foo': 'bar'} system_state = SystemState(id(self), 'flow1', node_args=node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args == node_args assert 'Task1' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 # Task1 has failed task1 = self.get_task('Task1') self.set_failed(task1, KeyError("Some exception raised")) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args == node_args assert 'Task1' in self.instantiated_tasks assert 'Task3' in self.instantiated_tasks assert 'Task4' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 task3 = self.get_task('Task3') task4 = self.get_task('Task4') assert task3.node_args == node_args assert task4.node_args == node_args # Now let's finish Task3 and Task4 to see that they correctly continue self.set_finished(task3) self.set_finished(task4) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args == node_args assert 'Task1' in self.instantiated_tasks assert 'Task3' in self.instantiated_tasks assert 'Task4' in self.instantiated_tasks assert 'Task5' in self.instantiated_tasks assert 'Task6' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 3 assert state_dict['waiting_edges'][0] == 0 assert 2 in state_dict['waiting_edges'] assert 3 in state_dict['waiting_edges'] assert 'Task3' in state_dict['finished_nodes'] assert 'Task4' in state_dict['finished_nodes'] # Finish the flow! task5 = self.get_task('Task5') task6 = self.get_task('Task6') self.set_finished(task5) self.set_finished(task6) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() assert retry is None
def test_task2flow(self): # # flow1: # # Task1 # | # | # flow2 # # Note: # Result of Task1 is not propagated to flow2 as even node_args_from_first is set - but propagate_node_args # is not set # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }], 'flow2': [] } self.init(edge_table, node_args_from_first={'flow1': True}, task2storage_mapping={'Task1': 'Storage1'}, storage_mapping={'Storage1': _MyStorage()}) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks # Run without change at first system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert system_state.node_args is None assert retry is not None assert 'Task1' in self.instantiated_tasks # Task1 has finished task1 = self.get_task('Task1') task1_result = "propagated result of Task1" self.set_finished(task1, task1_result) _MyStorage.result = task1_result system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() flow2 = self.get_flow('flow2') assert 'flow2' in self.instantiated_flows assert flow2.node_args is None assert retry is not None
def test_selective_flow_subflow(self): # # flow1: # # flow2 Task1 # | | # | Task2 # | | \ # ------- Task3 # | # | # Task4 # | # | # Task5 # # flow2: # # TaskF2 # | # | # Task4 # | \ # | \ # flow3 flow4 # # flow3: # # Task4 # | # | # TaskF3 # # flow4: # # TaskF3 # # Note: Only Task4 should be run, flow4 is not executed # def simulate_flow2(flow2, f2_already_instantiated_tasks): f2_system_state = flow2.get_initial_system_state() f2_retry = f2_system_state.update() f2_state_dict = f2_system_state.to_dict() f2_selective = f2_system_state.selective assert f2_retry is not None assert f2_already_instantiated_tasks | {'TaskF2'} == set( self.instantiated_tasks) self.set_finished(self.get_task('TaskF2')) f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict, node_args=f2_system_state.node_args, selective=f2_selective) f2_retry = f2_system_state.update() f2_state_dict = f2_system_state.to_dict() f2_selective = f2_system_state.selective assert f2_retry is not None assert f2_already_instantiated_tasks | {'TaskF2', 'Task4'} == set( self.instantiated_tasks) assert 'flow3' not in self.instantiated_flows self.set_finished(self.get_task('Task4')) f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict, node_args=f2_system_state.node_args, selective=f2_selective) f2_retry = f2_system_state.update() f2_state_dict = f2_system_state.to_dict() f2_selective = f2_system_state.selective assert f2_retry is not None assert 'flow3' in self.instantiated_flows simulate_flow3(self.get_flow('flow3'), set(self.instantiated_tasks)) f2_system_state = SystemState(id(self), 'flow2', state=f2_state_dict, node_args=f2_system_state.node_args, selective=f2_selective) f2_retry = f2_system_state.update() assert f2_retry is None self.set_finished(flow2, None) # let's clean after self so we inspect purely flow1 in callee self.remove_all_tasks_by_name('TaskF2') self.remove_all_tasks_by_name('Task4') self.remove_all_flows_by_name('flow2') def simulate_flow3(flow3, f3_already_instantiated_tasks): f3_system_state = flow3.get_initial_system_state() f3_retry = f3_system_state.update() f3_state_dict = f3_system_state.to_dict() f3_selective = f3_system_state.selective assert f3_retry is not None assert f3_already_instantiated_tasks | {'Task4'} == set( self.instantiated_tasks) self.set_finished(self.get_task('Task4')) f3_system_state = SystemState(id(self), 'flow3', state=f3_state_dict, node_args=f3_system_state.node_args, selective=f3_selective) f3_retry = f3_system_state.update() assert f3_retry is None self.set_finished(flow3, None) self.remove_all_flows_by_name('flow3') edge_table = { 'flow1': [{ 'from': [], 'to': ['flow2', 'Task1'], 'condition': self.cond_true }, { 'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true }, { 'from': ['flow2', 'Task2'], 'to': ['Task4'], 'condition': self.cond_true }, { 'from': ['Task4'], 'to': ['Task5'], 'condition': self.cond_true }], 'flow2': [{ 'from': [], 'to': ['TaskF2'], 'condition': self.cond_true }, { 'from': ['TaskF2'], 'to': ['Task4'], 'condition': self.cond_true }, { 'from': ['Task4'], 'to': ['flow4', 'flow3'], 'condition': self.cond_true }], 'flow3': [{ 'from': [], 'to': ['Task4'], 'condition': self.cond_true }, { 'from': ['Task4'], 'to': ['TaskF3'], 'condition': self.cond_true }], 'flow4': [{ 'from': [], 'to': ['TaskF3'] }] } self.init(edge_table) system_state = self.get_initial_system_state('flow1', None, ['Task4'], follow_subflows=True, run_subsequent=False) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1'} == set(self.instantiated_tasks) assert {'flow2'} == set(self.instantiated_flows) simulate_flow2(self.get_flow('flow2'), set(self.instantiated_tasks)) self.set_finished(self.get_task('Task1'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2'} == set(self.instantiated_tasks) self.set_finished(self.get_task('Task2'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2', 'Task4'} == set(self.instantiated_tasks) self.set_finished(self.get_task('Task4'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() assert retry is None assert 'flow4' not in self.instantiated_flows
def test_propagate_parent_2(self): # # flow1: # # Task1 Task2 # | | # flow2 | # | | # ----------- # | # TaskX # # flow2: # Not run explicitly, but result finished_nodes is: # {'flow3': [<flow3-id>], 'Task2': [<task2-id1>], 'Task3': [<task3-id1>, <task3-id2>]} # flow3: # Not run explicitly, but result finished_nodes is: # {'Task2': [<task2-id2>], 'Task4': [<task4-id1>, <task4-id2>]} # # We are propagating finished, so we should inspect parent # edge_table = { 'flow1': [{'from': ['flow2', 'Task2'], 'to': ['TaskX'], 'condition': self.cond_true}, {'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true}, {'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}], 'flow2': [], 'flow3': [] } # Make sure propagate_finished is set to False as they are disjoint with propagate_compound_finished; # this is checked in selinon self.init(edge_table, propagate_parent=dict.fromkeys(edge_table.keys(), True), propagate_finished={'flow1': False}, propagate_compound_finished={'flow1': True}) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'flow2' not in self.instantiated_flows # Task1 and Task2 have finished task1 = self.get_task('Task1') self.set_finished(task1, None) task2 = self.get_task('Task2') self.set_finished(task2, None) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert 'flow2' in self.instantiated_flows # Create flow3 manually flow3 = Dispatcher().apply_async(kwargs={'flow_name': 'flow3'}, queue=Config.dispatcher_queues['flow3']) self.get_task_instance.register_node(flow3) flow2 = self.get_flow('flow2') self.set_finished(flow2) flow2_result = {'finished_nodes': {'flow3': [flow3.task_id], 'Task2': ['<task2-id1>'], 'Task3': ['<task3-id1>', '<task3-id2>']}, 'failed_nodes': {} } self.set_finished(flow2, flow2_result) flow3_result = {'finished_nodes': {'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>']}, 'failed_nodes': {} } self.set_finished(flow3, flow3_result) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() assert retry is not None task_x = self.get_task('TaskX') task_x_parent = {'Task2': task2.task_id, 'flow2': {'Task2': ['<task2-id1>', '<task2-id2>'], 'Task3': ['<task3-id1>', '<task3-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>'] } } # we have to check this as a set due to dict randomization assert 'Task2' in task_x.parent assert task_x.parent['Task2'] == task_x_parent['Task2'] assert 'flow2' in task_x.parent assert 'Task2' in task_x.parent['flow2'] assert 'Task3' in task_x.parent['flow2'] assert 'Task4' in task_x.parent['flow2'] assert set(task_x.parent['flow2']['Task2']) == set(task_x_parent['flow2']['Task2']) assert set(task_x.parent['flow2']['Task3']) == set(task_x_parent['flow2']['Task3']) assert set(task_x.parent['flow2']['Task4']) == set(task_x_parent['flow2']['Task4'])
def test_propagate_compound_mixed(self): # # flow1: # # flow2 flow3 # | | # ----------- # | # TaskX # # flow2: # Not run explicitly, but result finished_nodes is: # {'Task2': [<task2-id21>], # 'Task3': [<task3-id21>, <task3-id22>]} # 'flow4': {'Task2': [<task2-id41>]} # flow3: # Not run explicitly, but result finished_nodes is: # {'Task2': [<task2-id32>], # 'Task3': [<task3-id31>, <task3-id32>], # 'flow4': {'Task2': [<task2-id42>]} # # We are propagating finished, so we should inspect parent # edge_table = { 'flow1': [{'from': [], 'to': ['flow2', 'flow3'], 'condition': self.cond_true}, {'from': ['flow2', 'flow3'], 'to': ['TaskX'], 'condition': self.cond_true}], 'flow2': [], 'flow3': [], 'flow4': [] } # Make sure propagate_finished is negated propagate_compound_finished # this is checked in selinon self.init(edge_table, propagate_finished={'flow1': ['flow2']}, propagate_compound_finished={'flow1': ['flow3']}) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert 'flow2' in self.instantiated_flows # Create flow4 manually, we will reuse it, but we pretend that there are 2 instances - one run in flow2 # another one in flow3 flow4 = Dispatcher().apply_async(kwargs={'flow_name': 'flow4'}, queue=Config.dispatcher_queues['flow4']) self.get_task_instance.register_node(flow4) self.set_finished(flow4, {'finished_nodes': {'Task2': ['<task2-id41']}, 'failed_nodes': {}}) # Create results of flow2 and flow3 manually but they are instantiated by dispatcher flow2_result = {'finished_nodes': {'Task2': ['<task2-id21>'], 'Task3': ['<task3-id21>', '<task3-id22>'], 'flow4': [flow4.task_id]}, 'failed_nodes': {} } flow2 = self.get_flow('flow2') self.set_finished(flow2, flow2_result) flow3_result = {'finished_nodes': {'Task2': ['<task2-id32>'], 'Task3': ['<task3-id31>', '<task3-id32>'], 'flow4': [flow4.task_id]}, 'failed_nodes': {} } flow3 = self.get_flow('flow3') self.set_finished(flow3, flow3_result) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() assert retry is not None assert 'TaskX' in self.instantiated_tasks task_x_parent = {'flow2': {'Task2': ['<task2-id21>'], 'Task3': ['<task3-id21>', '<task3-id22>'], 'flow4': {'Task2': ['<task2-id41']}}, # flow4 is hidden in flow3 as it is compound, see 'Task2' 'flow3': {'Task2': ['<task2-id41', '<task2-id32>'], 'Task3': ['<task3-id31>', '<task3-id32>']}} task_x = self.get_task('TaskX') # we have to check this as a set due to dict randomization assert 'flow2' in task_x.parent assert 'flow3' in task_x.parent assert 'Task2' in task_x.parent['flow2'] assert 'Task3' in task_x.parent['flow2'] assert set(task_x.parent['flow2']['Task2']) == set(task_x_parent['flow2']['Task2']) assert set(task_x.parent['flow2']['Task3']) == set(task_x_parent['flow2']['Task3']) assert 'Task2' in task_x.parent['flow3'] assert 'Task3' in task_x.parent['flow3'] assert set(task_x.parent['flow3']['Task2']) == set(task_x_parent['flow3']['Task2']) assert set(task_x.parent['flow3']['Task3']) == set(task_x_parent['flow3']['Task3']) assert 'flow4' in task_x.parent['flow2'] assert set(task_x.parent['flow2']['flow4']) == set(task_x_parent['flow2']['flow4'])
def test_propagate_parent(self): # # flow1: # # Task1 Task2 # | | # flow2 | # | | # ----------- # | # TaskX # # flow2: # Run explicitly, but result finished_nodes is: # {'flow3': [<flow3-id>], 'Task2': [<task2-id1>], 'Task3': [<task3-id1>, <task3-id2>]} # flow3 # Run explicitly, but result finished_nodes is: # {'Task2': [<task2-id2>], 'Task4': [<task4-id1>, <task4-id2>]} # # We are propagating finished, so we should inspect parent # edge_table = { 'flow1': [{'from': ['flow2', 'Task2'], 'to': ['TaskX'], 'condition': self.cond_true}, {'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true}, {'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true}], 'flow2': [], 'flow3': [] } self.init(edge_table, propagate_parent={'flow1': True}, propagate_finished={'flow1': True}) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'flow2' not in self.instantiated_flows # Task1 and Task2 have finished task1 = self.get_task('Task1') self.set_finished(task1, None) task2 = self.get_task('Task2') self.set_finished(task2, None) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() state_dict_1 = system_state.to_dict() assert retry is not None assert 'flow2' in self.instantiated_flows # Create flow3 manually flow3 = Dispatcher().apply_async(kwargs={'flow_name': 'flow3'}, queue=Config.dispatcher_queues['flow3']) self.get_task_instance.register_node(flow3) flow2 = self.get_flow('flow2') self.set_finished(flow2) flow2_result = {'finished_nodes': {'flow3': [flow3.task_id], 'Task2': ['<task2-id1>'], 'Task3': ['<task3-id1>', '<task3-id2>']}, 'failed_nodes': {} } self.set_finished(flow2, flow2_result) flow3_result = {'finished_nodes': {'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>']}, 'failed_nodes': {} } self.set_finished(flow3, flow3_result) system_state = SystemState(id(self), 'flow1', state=state_dict_1, node_args=system_state.node_args) retry = system_state.update() assert retry is not None task_x = self.get_task('TaskX') task_x_parent = {'Task2': task2.task_id, 'flow2': {'Task2': ['<task2-id1>'], 'Task3': ['<task3-id1>', '<task3-id2>'], 'flow3': {'Task2': ['<task2-id2>'], 'Task4': ['<task4-id1>', '<task4-id2>'], } } } assert task_x.parent == task_x_parent
def test_simple_flow(self): # # flow1: # # Task1 # | # | # flow2 # | # | # Task2 # # Note: # Result of Task1 is not propagated to flow2, parent are not propagated as well # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['flow2'], 'condition': self.cond_true }, { 'from': ['flow2'], 'to': ['Task2'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }], 'flow2': [] } queues = { 'Task1': 'mytask1queue', 'Task2': 'mytask2queue', } flow2_queue = 'flow2_queue' dispatcher_queues = {'flow2': flow2_queue, 'flow1': 'flow1_queue'} self.init(edge_table, task_queues=queues, dispatcher_queues=dispatcher_queues) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'flow2' not in self.instantiated_flows assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 # check task's queue configuration task1 = self.get_task('Task1') assert task1.queue == queues.get(task1.task_name) # No change system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'flow2' not in self.instantiated_flows assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 # Task1 has finished task1_result = [1, 2, 3, 4] self.set_finished(task1, task1_result) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() flow2 = self.get_flow('flow2') assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert flow2.node_args is None assert flow2.flow_name == 'flow2' assert flow2.parent is None assert 'Task1' in self.instantiated_tasks assert 'flow2' in self.instantiated_flows assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 assert len(state_dict['finished_nodes']) == 1 assert len(state_dict['active_nodes']) == 1 # No change system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert flow2.node_args is None assert flow2.flow_name == 'flow2' assert flow2.parent is None assert 'Task1' in self.instantiated_tasks assert 'flow2' in self.instantiated_flows assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 assert len(state_dict['finished_nodes']) == 1 assert len(state_dict['active_nodes']) == 1 # check flow queue propagation flow2 = self.get_flow('flow2') assert flow2.queue == flow2_queue # flow2 has finished self.set_finished(flow2, {'TaskSubflow': [1]}) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() flow2 = self.get_flow('flow2') assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert flow2.node_args is None assert flow2.flow_name == 'flow2' assert flow2.parent is None assert 'Task1' in self.instantiated_tasks assert 'flow2' in self.instantiated_flows assert 'Task2' in self.instantiated_tasks assert flow2.node_args is None assert flow2.parent is None assert len(state_dict.get('waiting_edges')) == 2 assert state_dict['waiting_edges'][0] == 0 assert len(state_dict['finished_nodes']) == 2 assert len(state_dict['active_nodes']) == 1 # No change system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert 'Task1' in self.instantiated_tasks assert 'flow2' in self.instantiated_flows assert 'Task2' in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert state_dict['waiting_edges'][0] == 0 assert len(state_dict['finished_nodes']) == 2 assert len(state_dict['active_nodes']) == 1 # Task2 has finished task2 = self.get_task('Task2') self.set_finished(task2) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is None assert system_state.node_args is None assert task2.node_args is None assert 'flow2' in self.instantiated_flows assert 'Task1' in self.instantiated_tasks assert 'flow2' in self.instantiated_flows assert 'Task2' in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert state_dict['waiting_edges'][0] == 0 assert len(state_dict['finished_nodes']) == 3 assert len(state_dict['active_nodes']) == 0
def test_foreach_basic(self): # # flow1: # # Task1 # | # | # --------------------- # | | | # | | | # Task2 Task2 ... Task2 # # Note: # There will be spawned _FOREACH_COUNT Task2 # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true, 'foreach': lambda x, y: range(_FOREACH_COUNT), 'foreach_propagate_result': False }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }] } self.init(edge_table) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks # Task1 has finished task1 = self.get_task('Task1') self.set_finished(task1, "some result") system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert len(self.get_all_tasks('Task2')) == _FOREACH_COUNT tasks_state_dict = [ node for node in state_dict['active_nodes'] if node['name'] == 'Task2' ] assert len(tasks_state_dict) == _FOREACH_COUNT
def test_multi_failures(self): # # flow1: # # Task1 X # | # | # Task2 # # Note: # Task1 will fail, Task2 should be run # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }] } failures = { 'flow1': { 'Task1': { 'next:': {}, 'fallback': [['Task3'], ['Task4', 'Task5'], ['Task6', 'Task7']], 'conditions': [self.cond_true, self.cond_false, self.cond_true], 'condition_strs': ['true', 'false', 'true'] } } } self.init(edge_table, failures=failures) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 # Task1 has failed task1 = self.get_task('Task1') self.set_failed(task1, KeyError("Some exception raised")) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() assert 'Task2' not in self.instantiated_tasks assert 'Task3' in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert 'Task6' in self.instantiated_tasks assert 'Task7' in self.instantiated_tasks assert retry is not None
def test_eager_failures(self): # # flow1: # Task1 ------ # | | # Task2 x Task3 # | # Task4 # # Note: # Task2 marked as eager failure node in the flow, dispatcher will not wait for Task3 to finish edge_table = { 'flow1': [{ 'from': [], 'to': ['Task1'], 'condition': self.cond_true }, { 'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true }, { 'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true }, { 'from': ['Task2'], 'to': ['Task4'], 'condition': self.cond_true }] } self.init(edge_table, eager_failures=({'flow1': ['Task2']})) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert 'Task1' in self.instantiated_tasks assert retry is not None assert system_state.node_args is None task1 = self.get_task('Task1') self.set_finished(task1) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' in self.instantiated_tasks assert retry is not None assert system_state.node_args is None # Manually append new Task1 instance to simplify this test task2 = self.get_task('Task2') self.set_failed(task2, ValueError("Some exception raised")) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) with pytest.raises(FlowError) as flow_error: system_state.update() reported_state = flow_error.value.state assert 'active_nodes' in reported_state.keys() assert 'finished_nodes' in reported_state.keys() assert 'failed_nodes' in reported_state.keys() assert {node['name'] for node in reported_state['active_nodes']} == {'Task3'} assert set(reported_state['finished_nodes'].keys()) == {'Task1'} assert set(reported_state['failed_nodes'].keys()) == {'Task2'}
def test_selective_flow_two_tasks(self): # # flow1: # # Task1 Task2 Task3 Task4 Task5 # \ / \ / | # \ / \ / Task10 # ---------- ------- # | / \ # Task6 Task7 Task8 # | | # | | # Task11 Task9 # # Note: Only Task6 and Task7 should be run # edge_table = { 'flow1': [{ 'from': [], 'to': ['Task1', 'Task2', 'Task3', 'Task4', 'Task5'], 'condition': self.cond_true }, { 'from': ['Task1', 'Task2'], 'to': ['Task6'], 'condition': self.cond_true }, { 'from': ['Task3', 'Task4'], 'to': ['Task7', 'Task8'], 'condition': self.cond_true }, { 'from': ['Task7'], 'to': ['Task9'], 'condition': self.cond_true }, { 'from': ['Task5'], 'to': ['Task10'], 'condition': self.cond_true }], } self.init(edge_table) system_state = self.get_initial_system_state('flow1', None, ['Task6', 'Task7'], follow_subflows=False, run_subsequent=False) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2', 'Task3', 'Task4'} == set(self.instantiated_tasks) # Task1 has finished self.set_finished(self.get_task('Task1'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2', 'Task3', 'Task4'} == set(self.instantiated_tasks) # Task2 has finished self.set_finished(self.get_task('Task2'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6'} == set(self.instantiated_tasks) # Task3 has finished self.set_finished(self.get_task('Task3'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6'} == set(self.instantiated_tasks) # Task4 has finished self.set_finished(self.get_task('Task4'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6', 'Task7'} == set(self.instantiated_tasks) # Task7 has finished self.set_finished(self.get_task('Task7'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2', 'Task3', 'Task4', 'Task6', 'Task7'} == set(self.instantiated_tasks) # Task6 has finished task6 = self.get_task('Task6') self.set_finished(task6, None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() assert retry is None
def test_single_failure_flow_fallback_start(self): # # flow1: # # Task1 X .... Task3 # | | # | | # Task2 Task4 # # Note: # Task1 will fail # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }, { 'from': ['Task3'], 'to': ['Task4'], 'condition': self.cond_true }] } failures = { 'flow1': { 'Task1': { 'next:': {}, 'fallback': [['Task3']], 'conditions': [self.cond_true], 'condition_strs': ['cond_true'] } } } self.init(edge_table, failures=failures) node_args = {'foo': 'bar'} system_state = SystemState(id(self), 'flow1', node_args=node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args == node_args assert 'Task1' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 # Task1 has failed task1 = self.get_task('Task1') self.set_failed(task1) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args == node_args assert 'Task1' in self.instantiated_tasks assert 'Task3' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 task3 = self.get_task('Task3') assert task3.node_args == node_args
def test_selective_flow_one_task(self): # # flow1: # # Task1 # | \ # | \ # Task2 Task4 # | # | # Task3 # # Note: Only Task2 and Task1 should be run # edge_table = { 'flow1': [{ 'from': [], 'to': ['Task1'], 'condition': self.cond_true }, { 'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true }, { 'from': ['Task1'], 'to': ['Task4'], 'condition': self.cond_true }, { 'from': ['Task2'], 'to': ['Task3'], 'condition': self.cond_true }], } node_args = {'foo': 'bar'} self.init(edge_table) system_state = self.get_initial_system_state('flow1', node_args, ['Task2'], follow_subflows=False, run_subsequent=False) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1'} == set(self.instantiated_tasks) self.set_finished(self.get_task('Task1'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() state_dict = system_state.to_dict() selective = system_state.selective assert retry is not None assert {'Task1', 'Task2'} == set(self.instantiated_tasks) self.set_finished(self.get_task('Task2'), None) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args, selective=selective) retry = system_state.update() assert retry is None
def test_singe_failure_flow_fallback(self): # # flow1: # # flow2 X ... Task2 # | # | # Task1 # # Note: # flow2 will fail # edge_table = { 'flow1': [{ 'from': ['flow2'], 'to': ['Task1'], 'condition': self.cond_true }, { 'from': [], 'to': ['flow2'], 'condition': self.cond_true }], 'flow2': [] } failures = { 'flow1': { 'flow2': { 'next:': {}, 'fallback': [['Task2']], 'conditions': [self.cond_true], 'condition_strs': ['cond_true'] } } } self.init(edge_table, failures=failures) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert 'Task1' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 # flow2 has failed flow2 = self.get_flow('flow2') flow_info = { 'finished_nodes': { 'Task1': ['id_task1'] }, 'failed_nodes': { 'Task2': ['id_task21'] } } self.set_failed(flow2, FlowError(flow_info)) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'flow2' in self.instantiated_flows assert 'Task2' in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0
def test_multiple_failured_tasks(self): # # flow1: # Task0 Task1 X Task1 X # # Note: # flow1 will have three tasks, Task1 will be instantiated twice. We will provide fallback for Task1 edge_table = { 'flow1': [{ 'from': [], 'to': ['Task0', 'Task1'], 'condition': self.cond_true }] } failures = { 'flow1': { 'Task1': { 'next': {}, 'fallback': True } }, } self.init(edge_table, failures=failures) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert 'Task0' in self.instantiated_tasks assert 'Task1' in self.instantiated_tasks assert retry is not None assert system_state.node_args is None # Manually append new Task1 instance to simplify this test task1 = self.get_task('Task1') self.set_failed(task1, ValueError("Some exception raised")) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert 'Task1' in state_dict.get('failed_nodes') assert retry is not None state_dict['failed_nodes']['Task1'].append('<injected-task1-id>') system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() assert retry is not None task0 = self.get_task('Task0') self.set_finished(task0) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() assert retry is None
def test_two_failures_no_fallback(self): # # flow1: # # Task1 X Task2 X # | | # | | # Task3 Task4 # # Note: # Task1 will fail, then Task2 will fail # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task3'], 'condition': self.cond_true }, { 'from': ['Task2'], 'to': ['Task4'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1', 'Task2'], 'condition': self.cond_true }], } failures = { 'flow1': { 'Task1': { 'next': { 'Task2': { 'next': {}, 'fallback': [] } }, 'fallback': [] }, 'Task2': { 'next': { 'Task1': { 'next': {}, 'fallback': [] } }, 'fallback': [] } } } self.init(edge_table, failures=failures) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] task1 = self.get_task('Task1') self.set_failed(task1) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] assert 'Task1' in state_dict['failed_nodes'] # No change so far system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert 'Task4' not in self.instantiated_tasks assert 'Task5' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 2 assert 0 in state_dict['waiting_edges'] assert 1 in state_dict['waiting_edges'] assert 'Task1' in state_dict['failed_nodes'] # Task2 has failed task2 = self.get_task('Task2') self.set_failed(task2) with pytest.raises(FlowError): system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) system_state.update()
def test_single_failure_flow_fallback_to_flow(self): # # flow1: # # Task1 X .... flow2 # | # | # Task2 # # Note: # Task1 will fail # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task2'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }], 'flow2': [{ 'from': [], 'to': ['Task3'], 'condition': self.cond_true }] } failures = {'flow1': {'Task1': {'next:': {}, 'fallback': ['flow2']}}} self.init(edge_table, failures=failures) node_args = {'foo': 'bar'} system_state = SystemState(id(self), 'flow1', node_args=node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args == node_args assert 'Task1' in self.instantiated_tasks assert 'flow2' not in self.instantiated_flows assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0 # Task1 has failed task1 = self.get_task('Task1') self.set_failed(task1, KeyError("Some exception raised")) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args == node_args assert 'Task1' in self.instantiated_tasks assert 'flow2' in self.instantiated_flows assert 'Task2' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert state_dict['waiting_edges'][0] == 0
def test_nowait_in_flow(self): # # flow1: # # Task1 # | # ---------- # | | # Task2 Task3 # # Note: # Task3 finishes before Task2 and Task3 is marked as nowait # edge_table = { 'flow1': [{ 'from': ['Task1'], 'to': ['Task2', 'Task3'], 'condition': self.cond_true }, { 'from': [], 'to': ['Task1'], 'condition': self.cond_true }] } nowait_nodes = {'flow1': ['Task3']} self.init(edge_table, nowait_nodes=nowait_nodes) system_state = SystemState(id(self), 'flow1') retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert system_state.node_args is None assert 'Task1' in self.instantiated_tasks assert 'Task2' not in self.instantiated_tasks assert 'Task3' not in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert 0 in state_dict['waiting_edges'] assert len(state_dict.get('finished_nodes')) == 0 assert len(state_dict.get('active_nodes')) == 1 # Task1 has finished task1 = self.get_task('Task1') self.set_finished(task1, 1) system_state = SystemState(id(self), 'flow1', state=state_dict, node_args=system_state.node_args) retry = system_state.update() state_dict = system_state.to_dict() assert retry is not None assert 'Task1' in self.instantiated_tasks assert 'Task2' in self.instantiated_tasks assert 'Task3' in self.instantiated_tasks assert len(state_dict.get('waiting_edges')) == 1 assert 0 in state_dict['waiting_edges'] assert len(state_dict.get('finished_nodes')) == 1 assert len(state_dict.get('active_nodes')) == 1 assert 'Task3' not in state_dict.get('active_nodes') assert 'Task2' in state_dict['active_nodes'][0]['name']