def read_resource_quota(api_instance: client, name, namespace): try: tenant_quota = api_instance.read_namespaced_resource_quota( name=name, namespace=namespace) except ApiException as apiException: raise KubernetesGetException('resource quota', apiException) return tenant_quota
def list_models(namespace: str, id_token): if not tenant_exists(namespace, id_token): raise TenantDoesNotExistException(namespace) apps_api_client = get_k8s_apps_api_client(id_token) try: deployments = apps_api_client.list_namespaced_deployment(namespace) except ApiException as apiException: raise KubernetesGetException('deployment', apiException) try: bucket = minio_resource.Bucket(name=namespace) except ClientError as clientError: raise MinioCallException( f'An error occurred during bucket reading: {clientError}') models = [] for object in bucket.objects.all(): if object.size > 0: model_path = object.key.split('/')[0].rsplit('-', 1) model_name = model_path[0] model_version = model_path[-1] model_size = object.size deployment_count = len( endpoints_using_model(deployments, model_path)) models.append( (model_name, model_version, model_size, deployment_count)) if not models: return f"There are no models present in {namespace} tenant\n" else: return f'Models in {namespace} tenant ' \ f'(model name, model version, model size, deployed count): {models}\n'
def delete_model(parameters: dict, namespace: str, id_token): if not tenant_exists(namespace, id_token): raise TenantDoesNotExistException(namespace) model_path = f"{parameters['modelName']}-{parameters['modelVersion']}" bucket = minio_resource.Bucket(name=namespace) model_in_bucket = bucket.objects.filter(Prefix=model_path) if not model_exists(model_in_bucket): raise ModelDoesNotExistException(model_path) apps_api_client = get_k8s_apps_api_client(id_token) try: deployments = apps_api_client.list_namespaced_deployment(namespace) except ApiException as apiException: raise KubernetesGetException('deployment', apiException) endpoint_names = endpoints_using_model(deployments, model_path) if endpoint_names: raise ModelDeleteException( f'model is used by endpoints: {endpoint_names}') for key in model_in_bucket: key.delete() return model_path
def update_endpoint(parameters: dict, namespace: str, endpoint_name: str, id_token: str): custom_obj_api_instance = get_k8s_api_custom_client(id_token) try: endpoint_object = custom_obj_api_instance. \ get_namespaced_custom_object(CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL, endpoint_name) except ApiException as apiException: raise KubernetesGetException('endpoint', apiException) if 'modelName' in parameters: endpoint_object['spec']['modelName'] = parameters['modelName'] if 'modelVersionPolicy' in parameters: endpoint_object['spec']['modelVersionPolicy'] = \ normalize_version_policy(parameters['modelVersionPolicy']) if 'resources' in parameters: endpoint_object['spec']['resources'] = transform_quota( parameters['resources']) if 'subjectName' in parameters: endpoint_object['spec']['subjectName'] = parameters['subjectName'] try: custom_obj_api_instance.patch_namespaced_custom_object( CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL, endpoint_name, endpoint_object) except ApiException as apiException: raise KubernetesUpdateException('endpoint', apiException) endpoint_url = create_url_to_service(endpoint_name, namespace) return endpoint_url
def get_endpoint_number(apps_api_instance, namespace): try: endpoints = apps_api_instance.list_namespaced_deployment(namespace) except ApiException as apiException: raise KubernetesGetException('endpoint', apiException) endpoint_number = len(endpoints.to_dict()['items']) return endpoint_number
def get_replicas(apps_api_instance, namespace, endpoint_name): try: deployment_status = apps_api_instance.read_namespaced_deployment_status( endpoint_name, namespace) except ApiException as apiException: raise KubernetesGetException('deployment', apiException) available_replicas = deployment_status.to_dict()['status']['available_replicas'] unavailable_replicas = deployment_status.to_dict()['status']['unavailable_replicas'] return {'Available': available_replicas, 'Unavailable': unavailable_replicas}
def get_ing_host_path(api_instance: client, ing_name: str, namespace: str): try: api_response = api_instance.read_namespaced_ingress(ing_name, namespace) logger.info(f"get ingress api response : {api_response}") host_path = api_response.spec.rules[0].host host_ip = api_response.spec.rules[0].http.paths[0].backend.service_port except ApiException as e: raise KubernetesGetException('Get ingress', e) return host_path, host_ip
def endpoint_exists(endpoint_name, namespace, id_token: str): custom_api_instance = get_k8s_api_custom_client(id_token) try: custom_api_instance.get_namespaced_custom_object(CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL, endpoint_name) except ApiException as apiException: if apiException.status == RESOURCE_DOES_NOT_EXIST: return False raise KubernetesGetException('endpoint', apiException) return True
def verify_endpoint_amount(api_instance, apps_api_instance, namespace): try: namespace_spec = api_instance.read_namespace(namespace) except ApiException as apiException: raise KubernetesGetException('namespace', apiException) namespace_annotations = namespace_spec.to_dict()['metadata']['annotations'] if namespace_annotations and 'maxEndpoints' in namespace_annotations: endpoint_number = get_endpoint_number(apps_api_instance, namespace) if endpoint_number >= int(namespace_annotations['maxEndpoints']): raise EndpointsReachedMaximumException()
def list_endpoints(namespace: str, id_token: str): if not tenant_exists(namespace, id_token=id_token): raise TenantDoesNotExistException(tenant_name=namespace) apps_api_instance = get_k8s_apps_api_client(id_token) try: deployments = apps_api_instance.list_namespaced_deployment(namespace) except ApiException as apiException: raise KubernetesGetException('endpoint', apiException) endpoints_name_status = get_endpoints_name_status(deployments, namespace) logger.info(endpoints_name_status) return endpoints_name_status
def get_crd_subject_name_and_resources(custom_api_instance, namespace, endpoint_name): try: crd = custom_api_instance.get_namespaced_custom_object(CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL, endpoint_name) except ApiException as apiException: raise KubernetesGetException('endpoint', apiException) subject_name = crd['spec']['subjectName'] resources = "Not specified" if 'resources' in crd['spec']: resources = crd['spec']['resources'] return subject_name, resources
def list_tenants(id_token): tenants = [] api_instance = get_k8s_api_client(id_token) label = f'created_by = {PLATFORM_ADMIN_LABEL}' namespaces = {} try: namespaces = api_instance.list_namespace(label_selector=label) except ApiException as apiException: KubernetesGetException('namespaces', apiException) for item in namespaces.to_dict()['items']: if item['status']['phase'] != TERMINATION_IN_PROGRESS: tenants.append(item['metadata']['name']) return tenants
def is_namespace_available(namespace, id_token): api_instance = get_k8s_api_client(id_token) try: response = api_instance.read_namespace_status(namespace) except ApiException as apiException: if apiException.status == RESOURCE_DOES_NOT_EXIST: return False if apiException.status == K8S_FORBIDDEN: raise KubernetesForbiddenException('forbidden', apiException) raise KubernetesGetException('namespace status', apiException) if response and response.status.phase == TERMINATION_IN_PROGRESS: return False return True
def get_svc_external_ip_port(api_instance: client, label_selector: str, namespace: str): try: api_response = api_instance.list_namespaced_service(namespace, label_selector=label_selector) logger.info(f"list services api response : {api_response}") except ApiException as e: raise KubernetesGetException('List services', e) ports = api_response.items[0].spec.ports logger.info(f"Ports : {ports}") https = next(item for item in ports if item.name == "https") logger.info(f"Load balancer: {api_response.items[0].status.load_balancer}") ip = api_response.items[0].status.load_balancer.ingress[0].ip ipaddress.ip_address(ip) port = int(https.port) return ip, port
def get_endpoint_status(api_instance, namespace, endpoint_name): try: pods = api_instance.list_namespaced_pod(namespace) except ApiException as apiException: raise KubernetesGetException('pods', apiException) pod_phases = [] for pod in pods.to_dict()['items']: if pod['metadata']['labels']['endpoint'] == endpoint_name: pod_phase = pod['status']['phase'] pod_phases.append(pod_phase) running = sum(pod_phase == 'Running' for pod_phase in pod_phases) pending = sum(pod_phase == 'Pending' for pod_phase in pod_phases) failed = sum(pod_phase == 'Failed' for pod_phase in pod_phases) status = {'Running pods': running, 'Pending pods': pending, 'Failed pods': failed} return status
def list_servings(id_token): if not is_namespace_available(CRD_NAMESPACE, id_token): raise ResourceIsNotAvailableException('namespace', CRD_NAMESPACE) api_client = get_k8s_api_client(id_token) try: config_maps = api_client.list_namespaced_config_map(CRD_NAMESPACE, pretty='true') except ApiException as apiException: raise KubernetesGetException('config map', apiException) crd_config_maps = [] for item in config_maps.to_dict()['items']: crd_config_maps.append(item['metadata']['name']) return crd_config_maps
def propagate_secret(source_secret_path, target_namespace, id_token): source_secret_namespace, source_secret_name = source_secret_path.split('/') api_instance = get_k8s_api_client(id_token) try: source_secret = api_instance.read_namespaced_secret( source_secret_name, source_secret_namespace) except ApiException as apiException: raise KubernetesGetException('secret', apiException) source_secret.metadata.namespace = target_namespace source_secret.metadata.resource_version = None try: api_instance.create_namespaced_secret(namespace=target_namespace, body=source_secret) except ApiException as apiException: raise KubernetesCreateException('secret', apiException)
def get_serving(id_token, serving_name): if not is_namespace_available(CRD_NAMESPACE, id_token): raise ResourceIsNotAvailableException('namespace', CRD_NAMESPACE) api_client = get_k8s_api_client(id_token) try: config_map = api_client.read_namespaced_config_map(serving_name, CRD_NAMESPACE, pretty='true') except ApiException as apiException: raise KubernetesGetException('config map', apiException) crd_config_map = dict() try: crd_config_map = config_map.to_dict()['data'] except KeyError: raise ResourceIsNotAvailableException('serving template configuration', serving_name) return crd_config_map
def list_tenants(id_token): tenants = [] api_instance = get_k8s_api_client(id_token) label = f'created_by = {PLATFORM_ADMIN}' namespaces = {} try: namespaces = api_instance.list_namespace(label_selector=label) except ApiException as apiException: KubernetesGetException('namespaces', apiException) for item in namespaces.to_dict()['items']: if item['status']['phase'] != TERMINATION_IN_PROGRESS: tenants.append(item['metadata']['name']) if not tenants: message = "There's no tenants present on platform\n" logger.info(message) return message message = f"Tenants present on platform: {tenants}\n" logger.info(message) return message
def scale_endpoint(parameters: dict, namespace: str, endpoint_name: str, id_token: str): custom_obj_api_instance = get_k8s_api_custom_client(id_token) try: endpoint_object = custom_obj_api_instance. \ get_namespaced_custom_object(CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL, endpoint_name) except ApiException as apiException: raise KubernetesGetException('endpoint', apiException) endpoint_object['spec']['replicas'] = parameters['replicas'] try: custom_obj_api_instance.patch_namespaced_custom_object( CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL, endpoint_name, endpoint_object) except ApiException as apiException: raise KubernetesUpdateException('endpoint', apiException) endpoint_url = create_url_to_service(endpoint_name, namespace) return endpoint_url