def read_resource_quota(api_instance: client, name, namespace):
try:
tenant_quota = api_instance.read_namespaced_resource_quota(
name=name, namespace=namespace)
except ApiException as apiException:
raise KubernetesGetException('resource quota', apiException)
return tenant_quota
def list_models(namespace: str, id_token):
if not tenant_exists(namespace, id_token):
raise TenantDoesNotExistException(namespace)
apps_api_client = get_k8s_apps_api_client(id_token)
try:
deployments = apps_api_client.list_namespaced_deployment(namespace)
except ApiException as apiException:
raise KubernetesGetException('deployment', apiException)
try:
bucket = minio_resource.Bucket(name=namespace)
except ClientError as clientError:
raise MinioCallException(
f'An error occurred during bucket reading: {clientError}')
models = []
for object in bucket.objects.all():
if object.size > 0:
model_path = object.key.split('/')[0].rsplit('-', 1)
model_name = model_path[0]
model_version = model_path[-1]
model_size = object.size
deployment_count = len(
endpoints_using_model(deployments, model_path))
models.append(
(model_name, model_version, model_size, deployment_count))
if not models:
return f"There are no models present in {namespace} tenant\n"
else:
return f'Models in {namespace} tenant ' \
f'(model name, model version, model size, deployed count): {models}\n'
def delete_model(parameters: dict, namespace: str, id_token):
if not tenant_exists(namespace, id_token):
raise TenantDoesNotExistException(namespace)
model_path = f"{parameters['modelName']}-{parameters['modelVersion']}"
bucket = minio_resource.Bucket(name=namespace)
model_in_bucket = bucket.objects.filter(Prefix=model_path)
if not model_exists(model_in_bucket):
raise ModelDoesNotExistException(model_path)
apps_api_client = get_k8s_apps_api_client(id_token)
try:
deployments = apps_api_client.list_namespaced_deployment(namespace)
except ApiException as apiException:
raise KubernetesGetException('deployment', apiException)
endpoint_names = endpoints_using_model(deployments, model_path)
if endpoint_names:
raise ModelDeleteException(
f'model is used by endpoints: {endpoint_names}')
for key in model_in_bucket:
key.delete()
return model_path
def update_endpoint(parameters: dict, namespace: str, endpoint_name: str,
id_token: str):
custom_obj_api_instance = get_k8s_api_custom_client(id_token)
try:
endpoint_object = custom_obj_api_instance. \
get_namespaced_custom_object(CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL,
endpoint_name)
except ApiException as apiException:
raise KubernetesGetException('endpoint', apiException)
if 'modelName' in parameters:
endpoint_object['spec']['modelName'] = parameters['modelName']
if 'modelVersionPolicy' in parameters:
endpoint_object['spec']['modelVersionPolicy'] = \
normalize_version_policy(parameters['modelVersionPolicy'])
if 'resources' in parameters:
endpoint_object['spec']['resources'] = transform_quota(
parameters['resources'])
if 'subjectName' in parameters:
endpoint_object['spec']['subjectName'] = parameters['subjectName']
try:
custom_obj_api_instance.patch_namespaced_custom_object(
CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL, endpoint_name,
endpoint_object)
except ApiException as apiException:
raise KubernetesUpdateException('endpoint', apiException)
endpoint_url = create_url_to_service(endpoint_name, namespace)
return endpoint_url
def get_endpoint_number(apps_api_instance, namespace):
try:
endpoints = apps_api_instance.list_namespaced_deployment(namespace)
except ApiException as apiException:
raise KubernetesGetException('endpoint', apiException)
endpoint_number = len(endpoints.to_dict()['items'])
return endpoint_number
def get_replicas(apps_api_instance, namespace, endpoint_name):
try:
deployment_status = apps_api_instance.read_namespaced_deployment_status(
endpoint_name, namespace)
except ApiException as apiException:
raise KubernetesGetException('deployment', apiException)
available_replicas = deployment_status.to_dict()['status']['available_replicas']
unavailable_replicas = deployment_status.to_dict()['status']['unavailable_replicas']
return {'Available': available_replicas, 'Unavailable': unavailable_replicas}
def get_ing_host_path(api_instance: client, ing_name: str, namespace: str):
try:
api_response = api_instance.read_namespaced_ingress(ing_name, namespace)
logger.info(f"get ingress api response : {api_response}")
host_path = api_response.spec.rules[0].host
host_ip = api_response.spec.rules[0].http.paths[0].backend.service_port
except ApiException as e:
raise KubernetesGetException('Get ingress', e)
return host_path, host_ip
def endpoint_exists(endpoint_name, namespace, id_token: str):
custom_api_instance = get_k8s_api_custom_client(id_token)
try:
custom_api_instance.get_namespaced_custom_object(CRD_GROUP, CRD_VERSION, namespace,
CRD_PLURAL, endpoint_name)
except ApiException as apiException:
if apiException.status == RESOURCE_DOES_NOT_EXIST:
return False
raise KubernetesGetException('endpoint', apiException)
return True
def verify_endpoint_amount(api_instance, apps_api_instance, namespace):
try:
namespace_spec = api_instance.read_namespace(namespace)
except ApiException as apiException:
raise KubernetesGetException('namespace', apiException)
namespace_annotations = namespace_spec.to_dict()['metadata']['annotations']
if namespace_annotations and 'maxEndpoints' in namespace_annotations:
endpoint_number = get_endpoint_number(apps_api_instance, namespace)
if endpoint_number >= int(namespace_annotations['maxEndpoints']):
raise EndpointsReachedMaximumException()
def list_endpoints(namespace: str, id_token: str):
if not tenant_exists(namespace, id_token=id_token):
raise TenantDoesNotExistException(tenant_name=namespace)
apps_api_instance = get_k8s_apps_api_client(id_token)
try:
deployments = apps_api_instance.list_namespaced_deployment(namespace)
except ApiException as apiException:
raise KubernetesGetException('endpoint', apiException)
endpoints_name_status = get_endpoints_name_status(deployments, namespace)
logger.info(endpoints_name_status)
return endpoints_name_status
def get_crd_subject_name_and_resources(custom_api_instance, namespace, endpoint_name):
try:
crd = custom_api_instance.get_namespaced_custom_object(CRD_GROUP, CRD_VERSION, namespace,
CRD_PLURAL, endpoint_name)
except ApiException as apiException:
raise KubernetesGetException('endpoint', apiException)
subject_name = crd['spec']['subjectName']
resources = "Not specified"
if 'resources' in crd['spec']:
resources = crd['spec']['resources']
return subject_name, resources
def list_tenants(id_token):
tenants = []
api_instance = get_k8s_api_client(id_token)
label = f'created_by = {PLATFORM_ADMIN_LABEL}'
namespaces = {}
try:
namespaces = api_instance.list_namespace(label_selector=label)
except ApiException as apiException:
KubernetesGetException('namespaces', apiException)
for item in namespaces.to_dict()['items']:
if item['status']['phase'] != TERMINATION_IN_PROGRESS:
tenants.append(item['metadata']['name'])
return tenants
def is_namespace_available(namespace, id_token):
api_instance = get_k8s_api_client(id_token)
try:
response = api_instance.read_namespace_status(namespace)
except ApiException as apiException:
if apiException.status == RESOURCE_DOES_NOT_EXIST:
return False
if apiException.status == K8S_FORBIDDEN:
raise KubernetesForbiddenException('forbidden', apiException)
raise KubernetesGetException('namespace status', apiException)
if response and response.status.phase == TERMINATION_IN_PROGRESS:
return False
return True
def get_svc_external_ip_port(api_instance: client, label_selector: str, namespace: str):
try:
api_response = api_instance.list_namespaced_service(namespace,
label_selector=label_selector)
logger.info(f"list services api response : {api_response}")
except ApiException as e:
raise KubernetesGetException('List services', e)
ports = api_response.items[0].spec.ports
logger.info(f"Ports : {ports}")
https = next(item for item in ports if item.name == "https")
logger.info(f"Load balancer: {api_response.items[0].status.load_balancer}")
ip = api_response.items[0].status.load_balancer.ingress[0].ip
ipaddress.ip_address(ip)
port = int(https.port)
return ip, port
def get_endpoint_status(api_instance, namespace, endpoint_name):
try:
pods = api_instance.list_namespaced_pod(namespace)
except ApiException as apiException:
raise KubernetesGetException('pods', apiException)
pod_phases = []
for pod in pods.to_dict()['items']:
if pod['metadata']['labels']['endpoint'] == endpoint_name:
pod_phase = pod['status']['phase']
pod_phases.append(pod_phase)
running = sum(pod_phase == 'Running' for pod_phase in pod_phases)
pending = sum(pod_phase == 'Pending' for pod_phase in pod_phases)
failed = sum(pod_phase == 'Failed' for pod_phase in pod_phases)
status = {'Running pods': running, 'Pending pods': pending, 'Failed pods': failed}
return status
def list_servings(id_token):
if not is_namespace_available(CRD_NAMESPACE, id_token):
raise ResourceIsNotAvailableException('namespace', CRD_NAMESPACE)
api_client = get_k8s_api_client(id_token)
try:
config_maps = api_client.list_namespaced_config_map(CRD_NAMESPACE, pretty='true')
except ApiException as apiException:
raise KubernetesGetException('config map', apiException)
crd_config_maps = []
for item in config_maps.to_dict()['items']:
crd_config_maps.append(item['metadata']['name'])
return crd_config_maps
def propagate_secret(source_secret_path, target_namespace, id_token):
source_secret_namespace, source_secret_name = source_secret_path.split('/')
api_instance = get_k8s_api_client(id_token)
try:
source_secret = api_instance.read_namespaced_secret(
source_secret_name, source_secret_namespace)
except ApiException as apiException:
raise KubernetesGetException('secret', apiException)
source_secret.metadata.namespace = target_namespace
source_secret.metadata.resource_version = None
try:
api_instance.create_namespaced_secret(namespace=target_namespace,
body=source_secret)
except ApiException as apiException:
raise KubernetesCreateException('secret', apiException)
def get_serving(id_token, serving_name):
if not is_namespace_available(CRD_NAMESPACE, id_token):
raise ResourceIsNotAvailableException('namespace', CRD_NAMESPACE)
api_client = get_k8s_api_client(id_token)
try:
config_map = api_client.read_namespaced_config_map(serving_name, CRD_NAMESPACE,
pretty='true')
except ApiException as apiException:
raise KubernetesGetException('config map', apiException)
crd_config_map = dict()
try:
crd_config_map = config_map.to_dict()['data']
except KeyError:
raise ResourceIsNotAvailableException('serving template configuration', serving_name)
return crd_config_map
def list_tenants(id_token):
tenants = []
api_instance = get_k8s_api_client(id_token)
label = f'created_by = {PLATFORM_ADMIN}'
namespaces = {}
try:
namespaces = api_instance.list_namespace(label_selector=label)
except ApiException as apiException:
KubernetesGetException('namespaces', apiException)
for item in namespaces.to_dict()['items']:
if item['status']['phase'] != TERMINATION_IN_PROGRESS:
tenants.append(item['metadata']['name'])
if not tenants:
message = "There's no tenants present on platform\n"
logger.info(message)
return message
message = f"Tenants present on platform: {tenants}\n"
logger.info(message)
return message
def scale_endpoint(parameters: dict, namespace: str, endpoint_name: str, id_token: str):
custom_obj_api_instance = get_k8s_api_custom_client(id_token)
try:
endpoint_object = custom_obj_api_instance. \
get_namespaced_custom_object(CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL,
endpoint_name)
except ApiException as apiException:
raise KubernetesGetException('endpoint', apiException)
endpoint_object['spec']['replicas'] = parameters['replicas']
try:
custom_obj_api_instance.patch_namespaced_custom_object(
CRD_GROUP, CRD_VERSION, namespace, CRD_PLURAL, endpoint_name, endpoint_object)
except ApiException as apiException:
raise KubernetesUpdateException('endpoint', apiException)
endpoint_url = create_url_to_service(endpoint_name, namespace)
return endpoint_url