def test__task_operation_throttle_within_times(self):
with patch(TASK_OPERATION_TIMES_CONFIG_GET,
MagicMock(return_value=self.times_config)):
for time_num in range(100):
time_stamp = self.start_time_stamp + time_num * 7
with patch("time.time", MagicMock(return_value=time_stamp)):
result = check_task_operation_throttle(
project_id=1, operation="test_within_times")
self.assertTrue(result)
def test__task_operation_throttle_exceed_times(self):
with patch(TASK_OPERATION_TIMES_CONFIG_GET,
MagicMock(return_value=self.times_config)):
for time_num in range(100):
result = check_task_operation_throttle(
project_id=1, operation="test_exceed_times")
if time_num < 10:
self.assertTrue(result)
else:
self.assertFalse(result)
def operate_task(request, task_id, project_id):
try:
params = json.loads(request.body)
except Exception:
return {
"result": False,
"message": "invalid json format",
"code": err_code.REQUEST_PARAM_INVALID.code
}
action = params.get("action")
username = request.user.username
project = request.project
if env.TASK_OPERATION_THROTTLE and not check_task_operation_throttle(
project.id, action):
return {
"result":
False,
"message":
"project id: {} reach the limit of starting tasks".format(
project.id),
"code":
err_code.INVALID_OPERATION.code,
}
if action == "start":
if TaskFlowInstance.objects.is_task_started(project_id=project.id,
id=task_id):
return {
"result": False,
"code": err_code.INVALID_OPERATION.code,
"message": "task already started"
}
queue, routing_key = PrepareAndStartTaskQueueResolver(
settings.API_TASK_QUEUE_NAME_V2
).resolve_task_queue_and_routing_key()
prepare_and_start_task.apply_async(kwargs=dict(task_id=task_id,
project_id=project.id,
username=username),
queue=queue,
routing_key=routing_key)
return {
"message": "success",
"result": True,
"code": err_code.SUCCESS.code,
}
task = TaskFlowInstance.objects.get(pk=task_id,
project_id=project.id,
is_deleted=False)
ctx = task.task_action(action, username)
return ctx
def start_task(request, task_id, project_id):
username = request.user.username
project = request.project
if env.TASK_OPERATION_THROTTLE and not check_task_operation_throttle(
project.id, "start"):
return {
"result":
False,
"message":
"project id: {} reach the limit of starting tasks".format(
project.id),
"code":
err_code.INVALID_OPERATION.code,
}
if TaskFlowInstance.objects.is_task_started(project_id=project.id,
id=task_id):
return {
"result": False,
"code": err_code.INVALID_OPERATION.code,
"message": "task already started"
}
queue, routing_key = PrepareAndStartTaskQueueResolver(
settings.API_TASK_QUEUE_NAME_V2).resolve_task_queue_and_routing_key()
prepare_and_start_task.apply_async(kwargs=dict(task_id=task_id,
project_id=project.id,
username=username),
queue=queue,
routing_key=routing_key)
task_url = TaskFlowInstance.task_url(project_id=project.id,
task_id=task_id)
return {
"task_url": task_url,
"data": {
"task_url": task_url
},
"result": True,
"code": err_code.SUCCESS.code,
}
def task_action(request, action, project_id):
task_id = json.loads(request.body)["instance_id"]
username = request.user.username
task = TaskFlowInstance.objects.get(pk=task_id, project_id=project_id)
if env.TASK_OPERATION_THROTTLE and not check_task_operation_throttle(
project_id, action):
return JsonResponse({
"result":
False,
"message":
"project id: {} reach the limit of starting tasks".format(
project_id),
"code":
err_code.INVALID_OPERATION.code,
})
ctx = task.task_action(action, username)
return JsonResponse(ctx)
def create_and_start_task(request, template_id, project_id):
params = json.loads(request.body)
project = request.project
template_source = params.get("template_source", BUSINESS)
if env.TASK_OPERATION_THROTTLE and not check_task_operation_throttle(
project.id, "start"):
return {
"result":
False,
"message":
"project id: {} reach the limit of starting tasks".format(
project.id),
"code":
err_code.INVALID_OPERATION.code,
}
logger.info(
"[API] create_and_start_task, template_id: {template_id}, project_id: {project_id}, params: {params}."
.format(template_id=template_id, project_id=project.id, params=params))
# 根据template_id获取template
if template_source == BUSINESS:
try:
tmpl = TaskTemplate.objects.select_related(
"pipeline_template").get(id=template_id,
project_id=project.id,
is_deleted=False)
except TaskTemplate.DoesNotExist:
result = {
"result":
False,
"message":
"template[id={template_id}] of project[project_id={project_id},biz_id={biz_id}] "
"does not exist".format(template_id=template_id,
project_id=project.id,
biz_id=project.bk_biz_id),
"code":
err_code.CONTENT_NOT_EXIST.code,
}
return result
else:
try:
tmpl = CommonTemplate.objects.select_related(
"pipeline_template").get(id=template_id, is_deleted=False)
except CommonTemplate.DoesNotExist:
result = {
"result":
False,
"message":
"common template[id={template_id}] does not exist".format(
template_id=template_id),
"code":
err_code.CONTENT_NOT_EXIST.code,
}
return result
# 检查app_code是否存在
app_code = getattr(request.jwt.app, settings.APIGW_APP_CODE_KEY)
if not app_code:
message = "app_code cannot be empty, make sure api gateway has sent correct params"
return {
"result": False,
"message": message,
"code": err_code.CONTENT_NOT_EXIST.code
}
# 请求参数校验
try:
params.setdefault("flow_type", COMMON)
params.setdefault("template_source", BUSINESS)
params.setdefault("constants", {})
params.setdefault("exclude_task_nodes_id", [])
jsonschema.validate(params, APIGW_CREATE_AND_START_TASK_PARAMS)
except jsonschema.ValidationError as e:
logger.warning("[API] create_and_start_task raise params error: %s" %
e)
message = "task parmas is invalid: %s" % e
return {
"result": False,
"message": message,
"code": err_code.REQUEST_PARAM_INVALID.code
}
# 创建pipeline_instance
pipeline_instance_kwargs = {
"name": params["name"],
"creator": request.user.username,
"description": params.get("description", ""),
}
try:
data = TaskFlowInstance.objects.create_pipeline_instance_exclude_task_nodes(
tmpl, pipeline_instance_kwargs, params["constants"],
params["exclude_task_nodes_id"])
except Exception as e:
return {
"result": False,
"message": str(e),
"code": err_code.UNKNOWN_ERROR.code
}
# 创建task
try:
task = TaskFlowInstance.objects.create(
project=project,
pipeline_instance=data,
category=tmpl.category,
template_id=template_id,
template_source=params["template_source"],
create_method="api",
create_info=app_code,
flow_type=params["flow_type"],
current_flow="execute_task"
if params["flow_type"] == COMMON else "func_claim",
engine_ver=EngineConfig.objects.get_engine_ver(
project_id=project.id,
template_id=template_id,
template_source=template_source),
)
except Exception as e:
return {
"result": False,
"message": str(e),
"code": err_code.UNKNOWN_ERROR.code
}
# 开始执行task
queue, routing_key = PrepareAndStartTaskQueueResolver(
settings.API_TASK_QUEUE_NAME_V2).resolve_task_queue_and_routing_key()
prepare_and_start_task.apply_async(
kwargs=dict(task_id=task.id,
project_id=project.id,
username=request.user.username),
queue=queue,
routing_key=routing_key,
)
return {
"result": True,
"code": err_code.SUCCESS.code,
"data": {
"pipeline_tree": task.pipeline_tree,
"task_id": task.id,
"task_url": task.url
},
}