def handler_update_k8sstatefulset(self, ns, cluster_id, spec): client = K8SClient(self.access_token, self.project_id, cluster_id, env=None) deployment_name = spec['metadata']['name'] result = client.update_statefulset(ns, deployment_name, spec) if result.get('code') != 0: if result.get('code') == 4001: raise ConfigError( _("配置文件格式错误:{}").format(result.get('message'))) raise ComponentError( _("更新K8sStatefulSet失败,{}").format(result.get('message')))
def handler_update_k8sconfigmap(self, ns, cluster_id, spec): client = K8SClient(self.access_token, self.project_id, cluster_id, env=None) name = spec['metadata']['name'] result = client.update_configmap(ns, name, spec) if result.get('code') != 0: if result.get('code') == 4001: raise ConfigError( _("配置文件格式错误:{}").format(result.get('message'))) raise ComponentError( _("更新K8sConfigMap失败,{}").format(result.get('message')))
def init_namespace_by_bcs(self, access_token, project_id, project_code, data): """ k8s 的集群需要创建 Namespace 和 jfrog Sercret """ client = K8SClient(access_token, project_id, data['cluster_id'], env=None) name = data['name'] # 创建 ns self.create_ns_by_bcs(client, name, data) # 创建 jfrog Sercret self.create_jfrog_secret(client, access_token, project_id, project_code, data)
def rescheduler_taskgroup(self, request, project_id, cluster_id, ns, instance_name, taskgroup_name, kind=2): """重启更新""" if kind == 2: client = MesosClient(request.user.token.access_token, project_id, cluster_id, None) resp = client.rescheduler_mesos_taskgroup(ns, instance_name, taskgroup_name) else: client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) resp = client.delete_pod(ns, taskgroup_name) if resp.get("code") != ErrorCode.NoError: return APIResponse( {"code": resp.get("code", DEFAULT_ERROR_CODE), "message": resp.get("message", _("请求出现异常!"))} ) return APIResponse({"message": _("重新调度成功!")})
def handler_k8shpa(self, ns, cluster_id, spec): """下发HPA配置 """ client = K8SClient(self.access_token, self.project_id, cluster_id, env=None) spec['apiVersion'] = 'autoscaling/v2beta2' try: result = client.apply_hpa(ns, spec) except Exception as error: logger.exception('deploy hpa error, %s', error) raise Rollback({}) return result
def get_k8s_category_info(self, request, project_id, resource_name, inst_name, cluster_id, ns_name): """获取分类的上报信息 {'BCS-K8S-15007': {'K8sDeployment': {'inst_list': ['bellke-test-deploy-1'], 'ns_list': ['abc1']}}} """ ret_data = {} client = K8SClient( request.user.token.access_token, project_id, cluster_id, None ) curr_func = FUNC_MAP[resource_name] % 'get' resp = getattr(client, curr_func)({ 'name': inst_name, 'namespace': ns_name, 'field': ','.join(app_constants.RESOURCE_STATUS_FIELD_LIST) }) if resp.get('code') != ErrorCode.NoError: raise error_codes.APIError.f(resp.get('message')) data = resp.get('data') or [] # 添加HPA绑定信息 data = get_deployment_hpa(request, project_id, cluster_id, ns_name, data) for info in data: spec = getitems(info, ['data', 'spec'], default={}) # 针对不同的模板获取不同的值 replicas, available = utils.get_k8s_desired_ready_instance_count(info, resource_name) curr_key = (info['namespace'], info['resourceName']) labels = getitems(info, ['data', 'metadata', 'labels'], default={}) source_type = labels.get('io.tencent.paas.source_type') or 'other' annotations = getitems(info, ['data', 'metadata', 'annotations'], default={}) ret_data[curr_key] = { 'backend_status': 'BackendNormal', 'backend_status_message': '请求失败,已通知管理员!', 'category': resource_name, 'pod_count': f'{available}/{replicas}', 'build_instance': available, 'instance': replicas, 'status': utils.get_k8s_resource_status(resource_name, info, replicas, available), 'name': info['resourceName'], 'namespace': info['namespace'], 'create_at': info['createTime'], 'update_at': info['updateTime'], 'source_type': SOURCE_TYPE_MAP.get(source_type), 'version': get_instance_version_name(annotations, labels), # 标识应用的线上版本 'hpa': info['hpa'] # 是否绑定了HPA } if spec.get('paused'): ret_data[curr_key]['status'] = 'Paused' return ret_data
def delete_instance( self, request, project_id, cluster_id, ns, instance_name, category="application", kind=2, inst_id_list=None, enforce=0, ): """删除instance""" client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) # deployment 需要级联删除 res\pod; daemonset/job/statefulset 需要级联删除 pod if FUNC_MAP[category] in [ '%s_deployment', '%s_daemonset', '%s_job', '%s_statefulset' ]: fun_prefix = 'deep_delete' else: fun_prefix = 'delete' curr_func = getattr(client, FUNC_MAP[category] % fun_prefix) resp = curr_func(ns, instance_name) # 级联删除,会返回空 if resp is None: # 启动后台任务,轮训任务状态 if inst_id_list: delete_instance_task.delay(request.user.token.access_token, inst_id_list, kind) return APIResponse({ "code": ErrorCode.NoError, "message": _("删除成功") }) # response msg = resp.get("message") # message中有not found或者node does not exist时,认为已经删除成功 # 状态码为正常或者满足不存在条件时,认为已经删除成功 if (resp.get("code") in [ ErrorCode.NoError ]) or ("not found" in msg) or ("node does not exist" in msg): return APIResponse({ "code": ErrorCode.NoError, "message": _("删除成功") }) return APIResponse({"code": resp.get("code"), "message": msg})
def resume_update_deployment(self, request, project_id, cluster_id, ns, deployment_name, kind=2, category=None): """重启更新""" if kind == 2: client = MesosClient(request.user.token.access_token, project_id, cluster_id, None) resp = client.resume_update_deployment(ns, deployment_name) else: client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) curr_func = getattr(client, FUNC_MAP[category] % "patch") params = {"spec": {"paused": False}} resp = curr_func(ns, deployment_name, params) if resp.get("code") != ErrorCode.NoError: return APIResponse( {"code": resp.get("code", DEFAULT_ERROR_CODE), "message": resp.get("message", _("请求出现异常!"))} ) return APIResponse({"message": _("重启更新成功!")})
def update_deployment(self, request, project_id, cluster_id, ns, data, kind=2, category=None, app_name=None): """滚动升级""" if kind == 2: client = MesosClient(request.user.token.access_token, project_id, cluster_id, None) resp = client.update_deployment(ns, data) else: client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) curr_func = getattr(client, FUNC_MAP[category] % "update") resp = curr_func(ns, app_name, data) if resp.get("code") != ErrorCode.NoError: return APIResponse( {"code": resp.get("code", DEFAULT_ERROR_CODE), "message": resp.get("message", _("请求出现异常!"))} ) return APIResponse({"message": _("更新成功!")})
def get_namespace(access_token, project_id, cluster_id): client = K8SClient(access_token, project_id, cluster_id, env=None) resp = client.get_namespace() if resp.get('code') != ErrorCode.NoError: raise error_codes.APIError(f'get namespace error, {resp.get("message")}') data = resp.get('data') or [] namespace_list = [] # 过滤掉 系统命名空间和平台占用命名空间 # TODO: 命名空间是否有状态不为Active的场景 for info in data: resource_name = info['resourceName'] if resource_name in K8S_SYS_NAMESPACE or resource_name in K8S_PLAT_NAMESPACE: continue namespace_list.append(resource_name) return namespace_list
def create_instance(self, request, project_id, cluster_id, ns, data, category="application", kind=2): """创建实例""" if kind == 2: client = MesosClient(request.user.token.access_token, project_id, cluster_id, None) if category == "application": resp = client.create_application(ns, data) else: resp = client.create_deployment(ns, data) else: client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) curr_func = getattr(client, FUNC_MAP[category] % "create") resp = curr_func(ns, data) if resp.get("code") != ErrorCode.NoError: return APIResponse({"code": resp.get("code") or DEFAULT_ERROR_CODE, "message": resp.get("message")}) return APIResponse({"code": resp.get("code", DEFAULT_ERROR_CODE), "message": resp.get("message")})
def init_namespace_by_bcs(self, access_token, project_id, project_code, data): """k8s 的集群需要创建 Namespace""" client = K8SClient(access_token, project_id, data['cluster_id'], env=None) name = data['name'] # 创建 ns self.create_ns_by_bcs(client, name, data, project_code) # 如果需要使用资源配额,创建配额 if data.get("quota"): client = ns_resource.NamespaceQuota(access_token, project_id, data["cluster_id"]) client.create_namespace_quota(name, data["quota"])
def destroy(self, request, project_id, cluster_id, namespace, name): """ 删除 ServiceMonitor """ self._validate_namespace_use_perm(project_id, cluster_id, namespace) client = K8SClient(request.user.token.access_token, project_id, cluster_id, env=None) result = self._single_service_monitor_operate_handler( client.delete_service_monitor, _('删除'), project_id, BaseActivityType.Delete, namespace, name, manifest=None, log_success=True, ) return Response(result)
def create_imagepullsecret(access_token, project_id, project_code, cluster_id, namespace): """先和先前逻辑保持一致""" dept_auth = {'auths': create_dept_account(access_token, project_id, project_code, cluster_id)} auth_bytes = bytes(json.dumps(dept_auth), 'utf-8') secret_config = { "apiVersion": "v1", "kind": "Secret", "metadata": {"name": f"{K8S_IMAGE_SECRET_PRFIX}{namespace}", "namespace": namespace}, "data": {".dockerconfigjson": base64.b64encode(auth_bytes).decode()}, "type": "kubernetes.io/dockerconfigjson", } # client = K8SClient(access_token, project_id, cluster_id, env=None) resp = client.create_secret(namespace, secret_config) if (resp.get('code') != ErrorCode.NoError) and ('already exist' not in resp.get('message', '')): raise error_codes.APIError(f'create secret error, {resp.get("message")}')
def update_instance( self, request, project_id, cluster_id, ns, instance_num, conf, kind=2, category=None, name=None ): # noqa if kind == 2: client = MesosClient(request.user.token.access_token, project_id, cluster_id, None) resp = client.update_mesos_app_instance(ns, instance_num, conf) else: client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) curr_func = getattr(client, FUNC_MAP[category] % "update") resp = curr_func(ns, name, conf) if resp.get("code") != ErrorCode.NoError: return False, APIResponse( {"code": resp.get("code", DEFAULT_ERROR_CODE), "message": resp.get("message", _("请求出现异常!"))} ) return True, APIResponse({"message": _("更新成功!")})
def list(self, request, project_id, cluster_id): """ 获取 ServiceMonitor 列表 """ cluster_map = self._get_cluster_map(project_id) namespace_map = self._get_namespace_map(project_id) if cluster_id not in cluster_map: raise error_codes.APIError(_('集群 ID {} 不合法').format(cluster_id)) client = K8SClient(request.user.token.access_token, project_id, cluster_id, env=None) manifest = client.list_service_monitor() response_data = self._handle_items(cluster_id, cluster_map, namespace_map, manifest) perm = bcs_perm.Namespace(request, project_id, bcs_perm.NO_RES) response_data = perm.hook_perms(response_data, ns_id_flag='namespace_id') response_data = self._update_service_monitor_perm(response_data) return Response(response_data)
def get_k8s_category_info(self, request, project_id, cluster_ns_inst, category): """获取分类的上报信息 添加类型,只是为了mesos和k8s进行适配 {'BCS-K8S-15007': {'K8sDeployment': {'inst_list': ['bellke-test-deploy-1'], 'ns_list': ['abc1'], 'inst_ns_map': {'test1': 'deployment-232132132'}}}} """ resource_name = CATEGORY_MAP[category] ret_data = {} for cluster_id, info in cluster_ns_inst.items(): client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) if not info.get(resource_name): continue curr_func = FUNC_MAP[category] % 'get' resp = getattr(client, curr_func)({ 'name': ','.join(info[resource_name]['inst_list']), 'namespace': ','.join(info[resource_name]['ns_list']), 'field': ','.join(app_constants.RESOURCE_STATUS_FIELD_LIST) }) if resp.get('code') != ErrorCode.NoError: raise error_codes.APIError(resp.get('message')) data = resp.get('data') or [] inst_ns_map = info[resource_name]['inst_ns_map'] exist_item = [] for info_item in data: exist_item.append(info_item['resourceName']) replicas, available = utils.get_k8s_desired_ready_instance_count( info_item, category) if available != replicas or available == 0: curr_key = f'{resource_name}:{info_item["resourceName"]}' if curr_key in ret_data: ret_data[curr_key] += 1 else: ret_data[curr_key] = 1 for key, val in inst_ns_map.items(): if val in exist_item: continue curr_key = f'{resource_name}:{val}' if curr_key in ret_data: ret_data[curr_key] += 1 else: ret_data[curr_key] = 1 return ret_data
def create(self, request, project_id, cluster_id): """ 创建 ServiceMonitor """ params = self.params_validate(ServiceMonitorCreateSLZ) name, namespace = params['name'], params['namespace'] endpoints = [{ 'path': params['path'], 'interval': params['interval'], 'port': params['port'], 'params': params.get('params') or {}, }] manifest = { 'apiVersion': 'monitoring.coreos.com/v1', 'kind': 'ServiceMonitor', 'metadata': { 'labels': { 'release': 'po', 'io.tencent.paas.source_type': 'bcs', 'io.tencent.bcs.service_name': params['service_name'], }, 'name': name, 'namespace': namespace, }, 'spec': { 'endpoints': endpoints, 'selector': { 'matchLabels': params['selector'] }, 'sampleLimit': params['sample_limit'], }, } client = K8SClient(request.user.token.access_token, project_id, cluster_id, env=None) result = self._single_service_monitor_operate_handler( client.create_service_monitor, _('创建'), project_id, ActivityType.Add, namespace, name, manifest, log_success=True, ) return Response(result)
def get_k8s_app_status(access_token, project_id, cluster_id, instance_name, namespace, category): field = [ "resourceName", "namespace", ] client = K8SClient(access_token, project_id, cluster_id, None) curr_func = getattr(client, "%s_with_post" % (FUNC_MAP[category] % "get")) params = { "name": instance_name, "namespace": namespace, "field": ",".join(field) } resp = curr_func(params) if resp.get("code") != ErrorCode.NoError: return [] return resp.get("data") or []
def retrieve(self, request, project_id, cluster_id, namespace, name): """ 获取单个 ServiceMonitor """ client = K8SClient(request.user.token.access_token, project_id, cluster_id, env=None) result = client.get_service_monitor(namespace, name) if result.get('status') == 'Failure': raise error_codes.APIError(result.get('message', '')) labels = getitems(result, 'metadata.labels', {}) result['metadata'] = { k: v for k, v in result['metadata'].items() if k not in constants.INNER_USE_SERVICE_METADATA_FIELDS } result['metadata']['service_name'] = labels.get(constants.SM_SERVICE_NAME_LABEL) if isinstance(getitems(result, 'spec.endpoints'), list): result['spec']['endpoints'] = self._handle_endpoints(result['spec']['endpoints']) return Response(result)
def scale_instance( self, request, project_id, cluster_id, ns, app_name, instance_num, kind=2, category=None, data=None ): # noqa """扩缩容""" if kind == 2: client = MesosClient(request.user.token.access_token, project_id, cluster_id, None) resp = client.scale_mesos_app_instance(ns, app_name, instance_num) else: client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) data["spec"]["replicas"] = int(instance_num) curr_func = getattr(client, FUNC_MAP[category] % "update") resp = curr_func(ns, app_name, data) if resp.get("code") != ErrorCode.NoError: return APIResponse( {"code": resp.get("code", DEFAULT_ERROR_CODE), "message": resp.get("message", _("请求出现异常!"))} ) return APIResponse({"message": _("更新成功!")})
def delete(self, request, project_id, namespace_id, is_validate_perm=True): """删除集群 """ access_token = request.user.token.access_token # 检查权限 perm = bcs_perm.Namespace(request, project_id, namespace_id) perm.can_delete(raise_exception=is_validate_perm) access_token = request.user.token.access_token ns_result = paas_cc.get_namespace(access_token, project_id, namespace_id) if ns_result.get('code') != 0: raise error_codes.APIError.f(ns_result.get('message', '')) ns_data = ns_result.get('data') cluster_id = ns_data.get('cluster_id') name = ns_data.get('name') project_kind = request.project.kind if project_kind == K8S_PROJECT_KIND: client = K8SClient(access_token, project_id, cluster_id, env=None) elif project_kind == MESOS_PROJECT_KIND: client = MesosClient(access_token, project_id, cluster_id, env=None) else: raise ValidationError("项目编排类型不正确") # 删除 namespace result = client.delete_namespace(name) if result.get('code') != 0: raise error_codes.ComponentError.f("删除Namespace失败,%s, 请联系管理员解决" % result.get('message')) result = paas_cc.delete_namespace(access_token, project_id, namespace_id) if result.get('code') != 0: raise error_codes.APIError.f(result.get('message')) # 删除资源 perm.delete() return response.Response(result)
def list(self, request, project_id, cluster_id): """获取可选 Service 列表""" client = K8SClient(request.user.token.access_token, project_id, cluster_id, env=None) resp = client.get_service({'env': 'k8s'}) services = self._slim_down_service(resp.get('data') or []) # 共享集群需要再过滤下属于当前项目的命名空间 if get_cluster_type(cluster_id) == ClusterType.SHARED: project_namespaces = get_shared_cluster_proj_namespaces( request.ctx_cluster, request.project.english_name) services = [ svc for svc in services if svc['namespace'] in project_namespaces ] return Response(services)
def get_k8s_category_info(self, request, project_id, cluster_ns_inst, category): """获取分类的上报信息 {'BCS-K8S-15007': {'K8sDeployment': {'inst_list': ['bellke-test-deploy-1'], 'ns_list': ['abc1']}}} """ ret_data = {} for cluster_id, app_info in cluster_ns_inst.items(): client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) curr_func = FUNC_MAP[category] % 'get' resp = getattr(client, curr_func)({ 'field': ','.join(app_constants.RESOURCE_STATUS_FIELD_LIST) }) if resp.get('code') != ErrorCode.NoError: raise error_codes.APIError.f(resp.get('message')) data = resp.get('data') or [] # TODO: 关于状态匹配可以再根据实际情况进行调整 for info in data: curr_app_id = f'{info["namespace"]}:{info["resourceName"]}' if curr_app_id not in app_info: continue spec = (info.get('data') or {}).get('spec') or {} # 针对不同的模板获取不同的值 replicas, available = utils.get_k8s_desired_ready_instance_count( info, category) curr_key = (cluster_id, info['namespace'], info['resourceName']) ret_data[curr_key] = { 'pod_count': f'{available}/{replicas}', 'build_instance': available, 'instance': replicas, 'status': utils.get_k8s_resource_status(category, info, replicas, available), } if spec.get('paused'): ret_data[curr_key]['status'] = 'Paused' return ret_data
def get_k8s_rs_info(self, request, project_id, cluster_id, ns_name, resource_name): """获取k8s deployment副本信息""" ret_data = {} client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) extra = {"data.metadata.ownerReferences.name": resource_name} extra_encode = base64_encode_params(extra) resp = client.get_rs({"extra": extra_encode, "namespace": ns_name, "field": "resourceName,data.status"}) if resp.get("code") != 0: raise error_codes.APIError.f(resp.get("message") or _("查询出现异常"), replace=True) data = resp.get("data") or [] if not data: return ret_data # NOTE: 因为线上存在revision history,需要忽略掉replica为0的rs rs_name_list = [ info["resourceName"] for info in data if info.get("resourceName") and getitems(info, ["data", "status", "replicas"], 0) > 0 ] return rs_name_list
def create_node_label_via_bcs(self, request, project_id, cluster_id, node_id_labels={}): """调用BCS服务创建节点标签""" nodes = cluster_utils.get_cluster_nodes(request.user.token.access_token, project_id, cluster_id) # compose id: ip node_id_ip = {info["id"]: info["inner_ip"] for info in nodes if str(info["id"]) in node_id_labels.keys()} client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) for node_id, ip in node_id_ip.items(): k8s_resp = client.get_node_detail(ip) if k8s_resp.get("code") != 0: raise error_codes.APIError.f(k8s_resp.get("message"), replace=True) exist_metadata = (k8s_resp.get("data") or {}).get("metadata") or {} exist_labels = exist_metadata.get("labels") or {} if node_id_labels[str(node_id)] == "del": exist_labels.pop("nodetype", None) else: exist_labels.update(K8S_LB_LABEL) exist_labels["$patch"] = "replace" resp = client.create_node_labels(ip, exist_labels) if resp.get("code") != 0: raise error_codes.APIError(_("节点打标签异常,请联系管理员处理!"))
def exec_command(access_token: str, project_id: str, cluster_id: str, container_id: str, command: str) -> str: """在k8s容器中执行命令 """ context = {} client = K8SClient(access_token, project_id, cluster_id, None) _context = utils.get_k8s_context(client, container_id) if not _context: raise ValueError(_("container_id不正确或者容器不是运行状态")) context.update(_context) try: bcs_context = utils.get_k8s_cluster_context(client, project_id, cluster_id) except Exception as error: raise ValueError(_('获取集群信息失败,{}').format(error)) bcs_context = utils.get_k8s_admin_context(client, bcs_context, WebConsoleMode.INTERNEL.value) bcs_context['user_pod_name'] = context['pod_name'] context.update(bcs_context) client = _k8s_client(context) result = client.exec_command(command) return result
def online_app_conf(self, request, project_id, project_kind, cluster_id, name, namespace, category): """针对非模板创建的应用,获取线上的配置 """ conf = {} if project_kind == 1: client = K8SClient(request.user.token.access_token, project_id, cluster_id, None) curr_func = FUNC_MAP[category] % "get" resp = getattr(client, curr_func)({ "name": name, "namespace": namespace }) if resp.get("code") != 0: raise error_codes.APIError.f( resp.get("message", _("获取应用线上配置异常,请联系管理员处理!"))) data = resp.get("data") or [] if not data: return {} data = data[0] # 组装数据 conf["kind"] = data["resourceType"] conf["metadata"] = {} conf["spec"] = data["data"]["spec"] conf["metadata"]["name"] = data["data"]["metadata"]["name"] conf["metadata"]["namespace"] = data["data"]["metadata"][ "namespace"] conf["metadata"]["labels"] = data["data"]["metadata"]["labels"] conf["metadata"]["annotations"] = data["data"]["metadata"][ "annotations"] else: resp = self.get_mesos_application_deployment_config( request.user.token.access_token, project_id, cluster_id, name, namespace, category) if resp.get("code") != ErrorCode.NoError: raise error_codes.APIError.f( resp.get("message", _("获取应用线上配置异常,请联系管理员处理!"))) conf = resp.get('data') or {} return conf
def batch_delete(self, request, project_id, cluster_id): """ 批量删除 ServiceMonitor """ params = self.params_validate(ServiceMonitorBatchDeleteSLZ) svc_monitors = params['service_monitors'] self._validate_namespace_use_perm(project_id, cluster_id, [sm['namespace'] for sm in svc_monitors]) client = K8SClient(request.user.token.access_token, project_id, cluster_id, env=None) for m in svc_monitors: self._single_service_monitor_operate_handler( client.delete_service_monitor, _('删除'), project_id, BaseActivityType.Delete, m['namespace'], m['name'] ) metrics_names = ','.join([f"{sm['namespace']}/{sm['name']}" for sm in svc_monitors]) message = _('删除 Metrics: {} 成功').format(metrics_names) self._activity_log( project_id, request.user.username, metrics_names, message, BaseActivityType.Delete, BaseActivityStatus.Succeed, ) return Response({'successes': svc_monitors})
def get_k8s_app_deploy_with_post( self, request, project_id, cluster_id, instance_name=None, category="application", namespace=None, field=None): """获取k8s下application和deployment状态 """ client = K8SClient( request.user.token.access_token, project_id, cluster_id, None ) curr_func = getattr(client, "%s_with_post" % (FUNC_MAP[category] % "get")) params = { "name": instance_name, "namespace": namespace, "field": ",".join(field) } resp = curr_func(params) if resp.get("code") != ErrorCode.NoError: return False, APIResponse({ "code": resp.get("code") or DEFAULT_ERROR_CODE, "message": resp.get("message") }) return True, resp