def _batch_request(
func, params, get_data=lambda x: x["data"]["list"], get_count=lambda x: x["data"]["count"], limit=500,
):
"""
gsekit 并发请求接口
:param func: 请求方法
:param params: 请求参数
:param get_data: 获取数据函数
:param get_count: 获取总数函数
:param limit: 一次请求数量
:return: 请求结果
"""
# 请求第一次获取总数
result = func(page_param={"page": 1, "pagesize": limit}, **params)
if not result["result"]:
message = handle_api_error("[batch_request]", func.__name__, params, result)
logger.error(message)
raise ApiRequestError(message)
count = get_count(result)
data = []
start = 1
# 根据请求总数并发请求
pool = ThreadPool()
params_and_future_list = []
page_count = int(count / 500) + 1
while start <= page_count:
request_params = {"page_param": {"pagesize": limit, "page": start}}
request_params.update(params)
params_and_future_list.append(
{"params": request_params, "future": pool.apply_async(func, kwds=request_params)}
)
start += 1
pool.close()
pool.join()
# 取值
for params_and_future in params_and_future_list:
result = params_and_future["future"].get()
if not result:
message = handle_api_error("[batch_request]", func.__name__, params, result)
logger.error(message)
raise ApiRequestError(message)
data.extend(get_data(result))
return data
def get_biz_internal_module(username, bk_biz_id, bk_supplier_account):
"""
@summary: 根据业务ID获取业务空闲机, 故障机和待回收模块
@param bk_biz_id:
@param bk_supplier_account:
@return:
"""
client = get_client_by_user(username)
params = {
"bk_biz_id": bk_biz_id,
"bk_supplier_account": bk_supplier_account
}
get_biz_internal_module_return = client.cc.get_biz_internal_module(params)
if not get_biz_internal_module_return["result"]:
message = handle_api_error("cc", "cc.get_biz_internal_module", params,
get_biz_internal_module_return)
logger.error(message)
raise ApiRequestError(message)
result = []
for get_biz_internal_module_option in get_biz_internal_module_return[
"data"]["module"]:
result.append({
"id": get_biz_internal_module_option["bk_module_id"],
"name": get_biz_internal_module_option["bk_module_name"],
})
return {"result": True, "data": result, "message": "success"}
def cc_filter_set_variables(operator, bk_biz_id, bk_obj_id, bk_obj_value):
"""
通过集群ID、过滤属性ID、过滤属性值，过滤集群
:param operator: 操作者
:param bk_biz_id: 业务ID
:param bk_obj_id: 过滤属性ID
:param bk_obj_value: 过滤属性值
:return:
"""
client = get_client_by_user(operator)
obj_value_list = bk_obj_value.split(",")
results = []
# 多个过滤属性值时循环请求接口
for obj_value in obj_value_list:
kwargs = {
"bk_biz_id": int(bk_biz_id),
"condition": {
bk_obj_id: obj_value
},
}
result = client.cc.search_set(kwargs)
if not result["result"]:
err_msg = _("[cc_filter_set_variables] 调用 cc.search_set 接口获取集群失败, "
"kwargs={kwargs}, result={result}").format(
kwargs=kwargs, result=result)
logger.error(err_msg)
raise ApiRequestError(err_msg)
results += result["data"]["info"]
if not results:
return [], set()
bk_attributes = reduce(set.intersection,
[set(result.keys()) for result in results])
return results, bk_attributes
def get_set_property(operator):
"""
@summary: 获取集群所有的属性
@return:
"""
client = get_client_by_user(operator)
kwargs = {"bk_obj_id": "set"}
cc_result = client.cc.search_object_attribute(**kwargs)
if not cc_result["result"]:
message = handle_api_error("cc", "search_object_attribute", kwargs,
cc_result)
logger.error(message)
raise ApiRequestError(message)
obj_property = ["bk_set_id"]
for item in cc_result["data"]:
obj_property.append(item["bk_property_id"])
return obj_property
def generate_topo_data(self):
"""根据字符串生成topo数据"""
path_list = cc_parse_path_text(self.raw_data.replace(",", "\n"))
processed_path_list = self._remove_included_topo_path(path_list)
biz_topo_tree = self.gen_kwargs["biz_topo_tree"]
topo_info = {}
self._build_topo_info(biz_topo_tree, topo_info)
generated_topo = []
for path in processed_path_list:
cur_info = {"child": topo_info}
for inst_name in path:
cur_info = cur_info.get("child", {}).get(inst_name)
if cur_info is None:
raise ApiRequestError(
f"path: {'>'.join(path)} not found in topo_tree")
generated_topo.append({
"bk_obj_id": cur_info["bk_obj_id"],
"bk_inst_id": cur_info["bk_inst_id"]
})
return {"result": True, "data": generated_topo, "message": ""}
def get_service_template_list(username, bk_biz_id, bk_supplier_account):
"""
@summary: 批量获取服务模板列表
@param username: 执行接口用户名
@param bk_biz_id: 业务id
@param bk_supplier_account:
@return: [{'id':'', 'name':''}, {'id':'', 'name':''}]
"""
client = get_client_by_user(username)
kwargs = {
"bk_biz_id": int(bk_biz_id),
"bk_supplier_account": bk_supplier_account
}
list_service_template_return = client.cc.list_service_template(kwargs)
if not list_service_template_return["result"]:
message = handle_api_error("cc", "cc.list_service_template", kwargs,
list_service_template_return)
logger.error(message)
raise ApiRequestError(message)
return list_service_template_return["data"]["info"]
def get_module_list(username, bk_biz_id, kwargs=None):
"""
@summary: 查询模块
@param kwargs:
@param username: 执行用户名
@param bk_biz_id: 业务id
@return: [{'bk_module_id':'', 'bk_module_name':''}...]
"""
client = get_client_by_user(username)
params = {
"bk_biz_id": bk_biz_id,
}
if kwargs:
params.update(kwargs)
module_list_result = client.cc.find_module_batch(params)
if not module_list_result["result"]:
message = handle_api_error("cc", "cc.find_module_batch", kwargs,
module_list_result)
logger.error(message)
raise ApiRequestError(message)
return module_list_result["data"]
def cc_search_set_module_name_by_id(operator, bk_biz_id, bk_set_id, bk_module_ids):
"""
通过集群ID和模块ID查询对应的名字
:param operator: 操作者
:param bk_biz_id: 业务ID
:param bk_set_id: 集群ID
:param bk_module_id: 模块ID
:return:
"""
str_module_ids = [str(item) for item in bk_module_ids]
set_module_info = {"set_id": bk_set_id, "module_id": bk_module_ids, "flat__module_id": ",".join(str_module_ids)}
client = get_client_by_user(operator)
set_kwargs = {
"bk_biz_id": bk_biz_id,
"fields": ["bk_set_name"],
"condition": {"bk_set_id": bk_set_id},
"page": {"start": 0, "limit": 1},
}
set_result = client.cc.search_set(set_kwargs)
if set_result["result"] and set_result["data"]["info"]:
set_module_info["set_name"] = set_result["data"]["info"][0]["bk_set_name"]
else:
err_msg = handle_api_error("CC", "search_set", set_kwargs, set_result)
logger.error(err_msg)
raise ApiRequestError(err_msg)
module_kwargs = {"bk_biz_id": bk_biz_id, "bk_set_id": bk_set_id, "fields": ["bk_module_id", "bk_module_name"]}
module_info = batch_request(client.cc.search_module, module_kwargs)
bk_module_names = []
for item in module_info:
if item.get("bk_module_id") in bk_module_ids:
bk_module_names.append(item.get("bk_module_name"))
set_module_info["module_name"] = bk_module_names
set_module_info["flat__module_name"] = ",".join(bk_module_names)
return set_module_info
def batch_request(func,
params,
get_data=lambda x: x["data"]["info"],
get_count=lambda x: x["data"]["count"],
limit=500,
page_param=None,
is_page_merge=False):
"""
并发请求接口
:param page_param: 分页参数，默认使用start/limit分页，例如：{"cur_page_param":"start", "page_size_param":"limit"}
:param is_page_merge: 分页参数是否合并到请求体，默认False，例如：
is_page_merge=True, {"x": "y", "cur_page_param":"start", "page_size_param":"limit"};
is_page_merge=False, {"x": "y", page: {"cur_page_param":"start", "page_size_param":"limit"}}
:param func: 请求方法
:param params: 请求参数
:param get_data: 获取数据函数
:param get_count: 获取总数函数
:param limit: 一次请求数量
:return: 请求结果
"""
# 兼容其他分页参数类型
if page_param:
try:
cur_page_param = page_param["cur_page_param"]
page_size_param = page_param["page_size_param"]
except Exception as e:
message = "[batch_request] please input correct page param, {}".format(
e)
logger.error(message)
raise ApiRequestError(message)
else:
cur_page_param = "start"
page_size_param = "limit"
# 请求第一次获取总数
if is_page_merge:
result = func(**{cur_page_param: 0, page_size_param: 1}, **params)
else:
result = func(page={cur_page_param: 0, page_size_param: 1}, **params)
if not result["result"]:
message = handle_api_error("[batch_request]", func.path, params,
result)
logger.error(message)
raise ApiRequestError(message)
count = get_count(result)
data = []
start = 0
# 根据请求总数并发请求
pool = ThreadPool()
params_and_future_list = []
from bamboo_engine import local as bamboo_local
from pipeline.engine.core import context as pipeline_context
node_info = bamboo_local.get_node_info()
node_id = pipeline_context.get_node_id()
while start < count:
if is_page_merge:
request_params = {page_size_param: limit, cur_page_param: start}
else:
request_params = {
"page": {
page_size_param: limit,
cur_page_param: start
}
}
request_params.update(params)
kwds = {
"target_func": func,
"node_id": node_id,
"node_info": node_info,
"request_params": request_params
}
params_and_future_list.append({
"params":
request_params,
"future":
pool.apply_async(func=local_wrapper, kwds=kwds)
})
start += limit
pool.close()
pool.join()
# 取值
for params_and_future in params_and_future_list:
result = params_and_future["future"].get()
if not result:
message = handle_api_error("[batch_request]", func.path,
params_and_future["params"], result)
logger.error(message)
raise ApiRequestError(message)
data.extend(get_data(result))
return data