def _batch_request( func, params, get_data=lambda x: x["data"]["list"], get_count=lambda x: x["data"]["count"], limit=500, ): """ gsekit 并发请求接口 :param func: 请求方法 :param params: 请求参数 :param get_data: 获取数据函数 :param get_count: 获取总数函数 :param limit: 一次请求数量 :return: 请求结果 """ # 请求第一次获取总数 result = func(page_param={"page": 1, "pagesize": limit}, **params) if not result["result"]: message = handle_api_error("[batch_request]", func.__name__, params, result) logger.error(message) raise ApiRequestError(message) count = get_count(result) data = [] start = 1 # 根据请求总数并发请求 pool = ThreadPool() params_and_future_list = [] page_count = int(count / 500) + 1 while start <= page_count: request_params = {"page_param": {"pagesize": limit, "page": start}} request_params.update(params) params_and_future_list.append( {"params": request_params, "future": pool.apply_async(func, kwds=request_params)} ) start += 1 pool.close() pool.join() # 取值 for params_and_future in params_and_future_list: result = params_and_future["future"].get() if not result: message = handle_api_error("[batch_request]", func.__name__, params, result) logger.error(message) raise ApiRequestError(message) data.extend(get_data(result)) return data
def get_biz_internal_module(username, bk_biz_id, bk_supplier_account): """ @summary: 根据业务ID获取业务空闲机, 故障机和待回收模块 @param bk_biz_id: @param bk_supplier_account: @return: """ client = get_client_by_user(username) params = { "bk_biz_id": bk_biz_id, "bk_supplier_account": bk_supplier_account } get_biz_internal_module_return = client.cc.get_biz_internal_module(params) if not get_biz_internal_module_return["result"]: message = handle_api_error("cc", "cc.get_biz_internal_module", params, get_biz_internal_module_return) logger.error(message) raise ApiRequestError(message) result = [] for get_biz_internal_module_option in get_biz_internal_module_return[ "data"]["module"]: result.append({ "id": get_biz_internal_module_option["bk_module_id"], "name": get_biz_internal_module_option["bk_module_name"], }) return {"result": True, "data": result, "message": "success"}
def cc_filter_set_variables(operator, bk_biz_id, bk_obj_id, bk_obj_value): """ 通过集群ID、过滤属性ID、过滤属性值,过滤集群 :param operator: 操作者 :param bk_biz_id: 业务ID :param bk_obj_id: 过滤属性ID :param bk_obj_value: 过滤属性值 :return: """ client = get_client_by_user(operator) obj_value_list = bk_obj_value.split(",") results = [] # 多个过滤属性值时循环请求接口 for obj_value in obj_value_list: kwargs = { "bk_biz_id": int(bk_biz_id), "condition": { bk_obj_id: obj_value }, } result = client.cc.search_set(kwargs) if not result["result"]: err_msg = _("[cc_filter_set_variables] 调用 cc.search_set 接口获取集群失败, " "kwargs={kwargs}, result={result}").format( kwargs=kwargs, result=result) logger.error(err_msg) raise ApiRequestError(err_msg) results += result["data"]["info"] if not results: return [], set() bk_attributes = reduce(set.intersection, [set(result.keys()) for result in results]) return results, bk_attributes
def get_set_property(operator): """ @summary: 获取集群所有的属性 @return: """ client = get_client_by_user(operator) kwargs = {"bk_obj_id": "set"} cc_result = client.cc.search_object_attribute(**kwargs) if not cc_result["result"]: message = handle_api_error("cc", "search_object_attribute", kwargs, cc_result) logger.error(message) raise ApiRequestError(message) obj_property = ["bk_set_id"] for item in cc_result["data"]: obj_property.append(item["bk_property_id"]) return obj_property
def generate_topo_data(self): """根据字符串生成topo数据""" path_list = cc_parse_path_text(self.raw_data.replace(",", "\n")) processed_path_list = self._remove_included_topo_path(path_list) biz_topo_tree = self.gen_kwargs["biz_topo_tree"] topo_info = {} self._build_topo_info(biz_topo_tree, topo_info) generated_topo = [] for path in processed_path_list: cur_info = {"child": topo_info} for inst_name in path: cur_info = cur_info.get("child", {}).get(inst_name) if cur_info is None: raise ApiRequestError( f"path: {'>'.join(path)} not found in topo_tree") generated_topo.append({ "bk_obj_id": cur_info["bk_obj_id"], "bk_inst_id": cur_info["bk_inst_id"] }) return {"result": True, "data": generated_topo, "message": ""}
def get_service_template_list(username, bk_biz_id, bk_supplier_account): """ @summary: 批量获取服务模板列表 @param username: 执行接口用户名 @param bk_biz_id: 业务id @param bk_supplier_account: @return: [{'id':'', 'name':''}, {'id':'', 'name':''}] """ client = get_client_by_user(username) kwargs = { "bk_biz_id": int(bk_biz_id), "bk_supplier_account": bk_supplier_account } list_service_template_return = client.cc.list_service_template(kwargs) if not list_service_template_return["result"]: message = handle_api_error("cc", "cc.list_service_template", kwargs, list_service_template_return) logger.error(message) raise ApiRequestError(message) return list_service_template_return["data"]["info"]
def get_module_list(username, bk_biz_id, kwargs=None): """ @summary: 查询模块 @param kwargs: @param username: 执行用户名 @param bk_biz_id: 业务id @return: [{'bk_module_id':'', 'bk_module_name':''}...] """ client = get_client_by_user(username) params = { "bk_biz_id": bk_biz_id, } if kwargs: params.update(kwargs) module_list_result = client.cc.find_module_batch(params) if not module_list_result["result"]: message = handle_api_error("cc", "cc.find_module_batch", kwargs, module_list_result) logger.error(message) raise ApiRequestError(message) return module_list_result["data"]
def cc_search_set_module_name_by_id(operator, bk_biz_id, bk_set_id, bk_module_ids): """ 通过集群ID和模块ID查询对应的名字 :param operator: 操作者 :param bk_biz_id: 业务ID :param bk_set_id: 集群ID :param bk_module_id: 模块ID :return: """ str_module_ids = [str(item) for item in bk_module_ids] set_module_info = {"set_id": bk_set_id, "module_id": bk_module_ids, "flat__module_id": ",".join(str_module_ids)} client = get_client_by_user(operator) set_kwargs = { "bk_biz_id": bk_biz_id, "fields": ["bk_set_name"], "condition": {"bk_set_id": bk_set_id}, "page": {"start": 0, "limit": 1}, } set_result = client.cc.search_set(set_kwargs) if set_result["result"] and set_result["data"]["info"]: set_module_info["set_name"] = set_result["data"]["info"][0]["bk_set_name"] else: err_msg = handle_api_error("CC", "search_set", set_kwargs, set_result) logger.error(err_msg) raise ApiRequestError(err_msg) module_kwargs = {"bk_biz_id": bk_biz_id, "bk_set_id": bk_set_id, "fields": ["bk_module_id", "bk_module_name"]} module_info = batch_request(client.cc.search_module, module_kwargs) bk_module_names = [] for item in module_info: if item.get("bk_module_id") in bk_module_ids: bk_module_names.append(item.get("bk_module_name")) set_module_info["module_name"] = bk_module_names set_module_info["flat__module_name"] = ",".join(bk_module_names) return set_module_info
def batch_request(func, params, get_data=lambda x: x["data"]["info"], get_count=lambda x: x["data"]["count"], limit=500, page_param=None, is_page_merge=False): """ 并发请求接口 :param page_param: 分页参数,默认使用start/limit分页,例如:{"cur_page_param":"start", "page_size_param":"limit"} :param is_page_merge: 分页参数是否合并到请求体,默认False,例如: is_page_merge=True, {"x": "y", "cur_page_param":"start", "page_size_param":"limit"}; is_page_merge=False, {"x": "y", page: {"cur_page_param":"start", "page_size_param":"limit"}} :param func: 请求方法 :param params: 请求参数 :param get_data: 获取数据函数 :param get_count: 获取总数函数 :param limit: 一次请求数量 :return: 请求结果 """ # 兼容其他分页参数类型 if page_param: try: cur_page_param = page_param["cur_page_param"] page_size_param = page_param["page_size_param"] except Exception as e: message = "[batch_request] please input correct page param, {}".format( e) logger.error(message) raise ApiRequestError(message) else: cur_page_param = "start" page_size_param = "limit" # 请求第一次获取总数 if is_page_merge: result = func(**{cur_page_param: 0, page_size_param: 1}, **params) else: result = func(page={cur_page_param: 0, page_size_param: 1}, **params) if not result["result"]: message = handle_api_error("[batch_request]", func.path, params, result) logger.error(message) raise ApiRequestError(message) count = get_count(result) data = [] start = 0 # 根据请求总数并发请求 pool = ThreadPool() params_and_future_list = [] from bamboo_engine import local as bamboo_local from pipeline.engine.core import context as pipeline_context node_info = bamboo_local.get_node_info() node_id = pipeline_context.get_node_id() while start < count: if is_page_merge: request_params = {page_size_param: limit, cur_page_param: start} else: request_params = { "page": { page_size_param: limit, cur_page_param: start } } request_params.update(params) kwds = { "target_func": func, "node_id": node_id, "node_info": node_info, "request_params": request_params } params_and_future_list.append({ "params": request_params, "future": pool.apply_async(func=local_wrapper, kwds=kwds) }) start += limit pool.close() pool.join() # 取值 for params_and_future in params_and_future_list: result = params_and_future["future"].get() if not result: message = handle_api_error("[batch_request]", func.path, params_and_future["params"], result) logger.error(message) raise ApiRequestError(message) data.extend(get_data(result)) return data