def test_node_started_loop_is_not_latest(self):
username = "******"
component_code = "component_code"
subprocess_stack = ["1"]
loop = 1
pipeline_instance = MagicMock()
kwargs = {"pipeline_instance": pipeline_instance}
runtime = "runtime"
runtime_init = MagicMock(return_value=runtime)
bamboo_api = MagicMock()
get_children_states_return = MagicMock()
get_children_states_return.result = True
get_children_states_return.data = {
"node_id": {"name": "node_name", "loop": 2, "histories": [{"id": "hid", "ex_data": ""}]}
}
bamboo_api.get_children_states = MagicMock(return_value=get_children_states_return)
get_node_histories_return = MagicMock()
get_node_histories_return.result = True
get_node_histories_return.data = [{"outputs": {}, "id": "hid", "version": "hist_version"}]
bamboo_api.get_node_histories = MagicMock(return_value=get_node_histories_return)
dispatcher = NodeCommandDispatcher(engine_ver=2, node_id="node_id")
dispatcher._get_node_info = MagicMock()
dispatcher._assemble_histroy_detail = MagicMock()
format_pipeline_status = MagicMock()
with patch(TASKFLOW_DISPATCHERS_NODE_BAMBOO_RUNTIME, runtime_init):
with patch(TASKFLOW_DISPATCHERS_NODE_BAMBOO_API, bamboo_api):
with patch(TASKFLOW_DISPATCHERS_NODE_FORMAT_PIPELINE_STATUS, format_pipeline_status):
node_detail = dispatcher.get_node_detail(
username=username,
subprocess_stack=subprocess_stack,
component_code=component_code,
loop=loop,
**kwargs
)
bamboo_api.get_children_states.assert_called_once_with(runtime=runtime, node_id=dispatcher.node_id)
bamboo_api.get_node_histories.assert_called_once_with(runtime=runtime, node_id=dispatcher.node_id, loop=1)
dispatcher._get_node_info.assert_not_called()
dispatcher._assemble_histroy_detail.assert_called_once()
self.assertEqual(
node_detail,
{
"result": True,
"data": {
"name": "node_name",
"loop": 2,
"histories": [
{"id": "hid", "history_id": "hid", "state": bamboo_engine_states.FAILED, "ex_data": ""},
],
"version": "hist_version",
"history_id": "hid",
},
"message": "",
"code": err_code.SUCCESS.code,
},
)
def test_non_act_not_started(self):
username = "******"
component_code = "component_code"
subprocess_stack = ["1"]
loop = 1
pipeline_instance = MagicMock()
kwargs = {"pipeline_instance": pipeline_instance}
pipeline_api = MagicMock()
pipeline_api.get_status_tree = MagicMock(side_effect=pipeline_exceptions.InvalidOperationException)
dispatcher = NodeCommandDispatcher(engine_ver=1, node_id="node_id")
dispatcher._get_node_info = MagicMock(return_value={"type": "StartEvent"})
with patch(TASKFLOW_DISPATCHERS_NODE_PIPELINE_API, pipeline_api):
node_data = dispatcher.get_node_data_v1(
username=username, component_code=component_code, subprocess_stack=subprocess_stack, loop=loop, **kwargs
)
pipeline_api.get_status_tree.assert_called_once_with(dispatcher.node_id)
dispatcher._get_node_info.assert_called_once_with(
node_id=dispatcher.node_id, pipeline=pipeline_instance.execution_data, subprocess_stack=subprocess_stack
)
self.assertEqual(
node_data,
{
"result": True,
"data": {"inputs": {}, "outputs": [], "ex_data": ""},
"message": "",
"code": err_code.SUCCESS.code,
},
)
def auto_retry_node(taskflow_id, root_pipeline_id, node_id, retry_times,
engine_ver):
lock_name = "%s-%s-%s" % (root_pipeline_id, node_id, retry_times)
if not settings.redis_inst.set(name=lock_name, value=1, nx=True, ex=5):
metrics.TASKFLOW_NODE_AUTO_RETRY_LOCK_ACCUIRE_FAIL.labels(
hostname=HOST_NAME).inc(1)
logger.warning(
"[auto_retry_node] lock %s accuire failed, operation give up" %
lock_name)
return
# wait process enter a valid state
can_retry = _ensure_node_can_retry(node_id=node_id, engine_ver=engine_ver)
if not can_retry:
settings.redis_inst.delete(lock_name)
logger.warning(
"[auto_retry_node] task(%s) node(%s) ensure_node_can_retry timeout"
% (taskflow_id, node_id))
return
dispatcher = NodeCommandDispatcher(engine_ver=engine_ver,
node_id=node_id,
taskflow_id=taskflow_id)
result = dispatcher.dispatch(command="retry", operator="system", inputs={})
if not result["result"]:
logger.warning(
"[auto_retry_node] task(%s) node(%s) auto retry failed: %s" %
(taskflow_id, node_id, result))
AutoRetryNodeStrategy.objects.filter(
root_pipeline_id=root_pipeline_id,
node_id=node_id).update(retry_times=retry_times + 1)
settings.redis_inst.delete(lock_name)
def test_node_started_loop_is_not_latest(self):
username = "******"
component_code = "component_code"
subprocess_stack = ["1"]
loop = 1
pipeline_instance = MagicMock()
kwargs = {"pipeline_instance": pipeline_instance}
pipeline_api = MagicMock()
detail = {"loop": 2, "histories": [{}]}
pipeline_api.get_status_tree = MagicMock(return_value=detail)
histories = [{"history_id": 3}]
pipeline_api.get_activity_histories = MagicMock(return_value=histories)
dispatcher = NodeCommandDispatcher(engine_ver=1, node_id="node_id")
dispatcher._get_node_info = MagicMock()
dispatcher._assemble_histroy_detail = MagicMock()
format_pipeline_status = MagicMock()
with patch(TASKFLOW_DISPATCHERS_NODE_PIPELINE_API, pipeline_api):
with patch(TASKFLOW_DISPATCHERS_NODE_FORMAT_PIPELINE_STATUS,
format_pipeline_status):
node_detail = dispatcher.get_node_detail_v1(
username=username,
subprocess_stack=subprocess_stack,
component_code=component_code,
loop=loop,
**kwargs,
)
pipeline_api.get_status_tree.assert_called_once_with(
dispatcher.node_id)
pipeline_api.get_status_tree.get_activity_histories(
node_id=dispatcher.node_id, loop=loop)
dispatcher._get_node_info.assert_not_called()
dispatcher._assemble_histroy_detail.assert_called_once_with(
detail=detail, histories=histories)
self.assertEqual(
node_detail,
{
"result": True,
"data": {
"loop": 2,
"history_id": 3,
"histories": [{
"state": pipeline_states.FAILED
}]
},
"message": "",
"code": err_code.SUCCESS.code,
},
)
def test_node_not_stated(self):
username = "******"
component_code = "component_code"
subprocess_stack = ["1"]
loop = 1
pipeline_instance = MagicMock()
kwargs = {"pipeline_instance": pipeline_instance}
runtime = "runtime"
runtime_init = MagicMock(return_value=runtime)
bamboo_api = MagicMock()
get_children_states_return = MagicMock()
get_children_states_return.result = True
get_children_states_return.data = {}
bamboo_api.get_children_states = MagicMock(return_value=get_children_states_return)
dispatcher = NodeCommandDispatcher(engine_ver=2, node_id="node_id")
dispatcher._get_node_info = MagicMock(return_value={"name": "node_name", "error_ignorable": True})
with patch(TASKFLOW_DISPATCHERS_NODE_BAMBOO_RUNTIME, runtime_init):
with patch(TASKFLOW_DISPATCHERS_NODE_BAMBOO_API, bamboo_api):
node_detail = dispatcher.get_node_detail(
username=username,
subprocess_stack=subprocess_stack,
component_code=component_code,
loop=loop,
**kwargs
)
bamboo_api.get_children_states.assert_called_once_with(runtime=runtime, node_id=dispatcher.node_id)
dispatcher._get_node_info.assert_called_once_with(
node_id=dispatcher.node_id, pipeline=pipeline_instance.execution_data, subprocess_stack=subprocess_stack
)
self.assertEqual(
node_detail,
{
"result": True,
"data": {"name": "node_name", "error_ignorable": True, "state": bamboo_engine_states.READY},
"message": "",
"code": err_code.SUCCESS.code,
},
)
def test_node_started_loop_is_not_latest(self):
username = "******"
component_code = "component_code"
subprocess_stack = ["1"]
loop = 1
pipeline_instance = MagicMock()
kwargs = {"pipeline_instance": pipeline_instance}
detail = {"loop": 2}
histories = [{"inputs": "inputs", "outputs": "outputs", "ex_data": "ex_data"}]
pipeline_api = MagicMock()
pipeline_api.get_status_tree = MagicMock(return_value=detail)
pipeline_api.get_activity_histories = MagicMock(return_value=histories)
dispatcher = NodeCommandDispatcher(engine_ver=1, node_id="node_id")
dispatcher._get_node_info = MagicMock()
dispatcher._prerender_node_data = MagicMock()
format_outputs = "format_outputs"
dispatcher._format_outputs = MagicMock(return_value=(True, None, format_outputs))
with patch(TASKFLOW_DISPATCHERS_NODE_PIPELINE_API, pipeline_api):
node_data = dispatcher.get_node_data_v1(
username=username, component_code=component_code, subprocess_stack=subprocess_stack, loop=loop, **kwargs
)
pipeline_api.get_status_tree.assert_called_once_with(dispatcher.node_id)
pipeline_api.get_activity_histories.assert_called_once_with(node_id=dispatcher.node_id, loop=loop)
dispatcher._get_node_info.assert_not_called()
dispatcher._prerender_node_data.assert_not_called()
dispatcher._format_outputs.assert_called_once_with(
outputs={"outputs": "outputs"},
component_code=component_code,
pipeline_instance=pipeline_instance,
subprocess_stack=["1"],
)
self.assertEqual(
node_data,
{
"result": True,
"data": {"inputs": "inputs", "outputs": format_outputs, "ex_data": "ex_data"},
"message": "",
"code": err_code.SUCCESS.code,
},
)