def test_forced_fail__success(self):
node = ServiceActivity(id=self.node_id, service=None)
setattr(node, 'failure_handler', MagicMock())
top_pipeline = PipelineObject(nodes={self.node_id: node})
process = MockPipelineProcess(top_pipeline=top_pipeline)
status = MockStatus()
old_version = status.version
kill = True
ex_data = 'ex_data'
with patch(PIPELINE_STATUS_GET, MagicMock(return_value=status)):
with patch(PIPELINE_PROCESS_GET, MagicMock(return_value=process)):
act_result = api.forced_fail(self.node_id, kill, ex_data)
self.assertTrue(act_result.result)
node.failure_handler.assert_called_once_with(
process.root_pipeline.data)
ScheduleService.objects.delete_schedule.assert_called_once_with(
status.id, old_version)
Data.objects.forced_fail.assert_called_once_with(
self.node_id, ex_data)
ProcessCeleryTask.objects.revoke.assert_called_once_with(
process.id, kill)
process.sleep.assert_called_once_with(adjust_status=True)
self.assertNotEqual(old_version, status.version)
status.save.assert_called_once()
def test_forced_fail__fail_with_invalid_node_type(self):
top_pipeline = PipelineObject(nodes={self.node_id: ServiceActObject()})
process = MockPipelineProcess(top_pipeline=top_pipeline)
with patch(PIPELINE_PROCESS_GET, MagicMock(return_value=process)):
act_result = api.forced_fail(self.node_id, uniqid())
self.assertFalse(act_result.result)
def test_forced_fail__fail_with_process_do_not_exist(self):
act_result = api.forced_fail(self.node_id)
self.assertFalse(act_result.result)
self.assertEqual(
act_result.message,
'invalid operation, this node is finished or pipeline have been revoked'
)
def test_forced_fail__fail_with_transit_fail(self):
top_pipeline = PipelineObject(nodes={self.node_id: ServiceActivity(id=self.node_id,
service=None)})
process = MockPipelineProcess(top_pipeline=top_pipeline)
with patch(PIPELINE_PROCESS_GET, MagicMock(return_value=process)):
act_result = api.forced_fail(self.node_id)
self.assertFalse(act_result.result)
self.assertEqual(act_result.message, 'transit fail')
def spec_nodes_timer_reset(self, node_id, username, inputs):
if not self.has_node(node_id):
return {'result': False, 'message': 'timer which be operated is not in this flow'}
success = pipeline_api.forced_fail(node_id)
if not success:
return {'result': False, 'message': 'timer node not exits or is finished'}
success = pipeline_api.retry_node(node_id, inputs)
if not success:
return {'result': False, 'message': 'reset timer failed, please try again later'}
return {'result': True, 'data': 'success'}
def spec_nodes_timer_reset(self, node_id, username, inputs):
# TODO assert node_id is sub_node of pipeline
success = pipeline_api.forced_fail(node_id)
if not success:
return {
'result': False,
'message': 'timer node not exits or is finished'
}
success = pipeline_api.retry_node(node_id, inputs)
if not success:
return {
'result': False,
'message': 'reset timer failed, please try again later'
}
return {'result': True, 'data': 'success'}
def spec_nodes_timer_reset(self, node_id, username, inputs):
if not self.has_node(node_id):
message = 'node[node_id={node_id}] not found in task[task_id={task_id}]'.format(
node_id=node_id, task_id=self.id)
return {'result': False, 'message': message}
action_result = pipeline_api.forced_fail(node_id)
if not action_result.result:
return {
'result': False,
'message': 'timer node not exits or is finished'
}
action_result = pipeline_api.retry_node(node_id, inputs)
if not action_result.result:
return {
'result': False,
'message': 'reset timer failed, please try again later'
}
return {'result': True, 'data': 'success'}
def node_timeout_check(node_id, version, root_pipeline_id):
NodeCeleryTask.objects.destroy(node_id)
state = Status.objects.state_for(node_id,
version=version,
may_not_exist=True)
if not state or state != states.RUNNING:
logger.info(
'node %s %s timeout kill failed, node not exist or not in running'
% (node_id, version))
return
action_result = api.forced_fail(node_id,
kill=True,
ex_data='node execution timeout')
if action_result.result:
signals.activity_failed.send(sender=Pipeline,
pipeline_id=root_pipeline_id,
pipeline_activity_id=node_id)
else:
logger.error('node %s - %s timeout kill failed' % (node_id, version))
def forced_fail(node_id):
return api.forced_fail(node_id)
def test_forced_fail__fail_with_process_do_not_exist(self):
act_result = api.forced_fail(self.node_id)
self.assertFalse(act_result.result)
