def deploy(self, model_details: ModelDetails): model_spec = self._get_spec(model_details) logger.debug(model_spec) api_instance = client.CustomObjectsApi() try: existing = api_instance.get_namespaced_custom_object( "serving.kubeflow.org", "v1alpha2", self.k8s_options.namespace, "inferenceservices", model_details.name, ) model_spec["metadata"]["resourceVersion"] = existing["metadata"][ "resourceVersion"] api_instance.replace_namespaced_custom_object( "serving.kubeflow.org", "v1alpha2", self.k8s_options.namespace, "inferenceservices", model_details.name, model_spec, ) except ApiException as e: if e.status == 404: api_instance.create_namespaced_custom_object( "serving.kubeflow.org", "v1alpha2", self.k8s_options.namespace, "inferenceservices", model_spec, ) else: raise e
def deploy(self, model_details: ModelDetails): model_spec = self._get_spec(model_details) logger.debug(model_spec) api_instance = client.CustomObjectsApi() try: existing = api_instance.get_namespaced_custom_object( "machinelearning.seldon.io", "v1", self.k8s_options.namespace, "seldondeployments", model_details.name, ) model_spec["metadata"]["resourceVersion"] = existing["metadata"][ "resourceVersion"] api_instance.replace_namespaced_custom_object( "machinelearning.seldon.io", "v1", self.k8s_options.namespace, "seldondeployments", model_details.name, model_spec, ) except ApiException as e: if e.status == 404: api_instance.create_namespaced_custom_object( "machinelearning.seldon.io", "v1", self.k8s_options.namespace, "seldondeployments", model_spec, ) else: raise e
def deploy_spec(self, model_spec: ModelSpec): spec = self._get_spec(model_spec) logger.debug(model_spec) self.create_k8s_client() api_instance = client.CustomObjectsApi() try: existing = api_instance.get_namespaced_custom_object( "serving.kubeflow.org", "v1beta1", model_spec.runtime_options.k8s_options.namespace, "inferenceservices", model_spec.model_details.name, ) spec["metadata"]["resourceVersion"] = existing["metadata"]["resourceVersion"] api_instance.replace_namespaced_custom_object( "serving.kubeflow.org", "v1beta1", model_spec.runtime_options.k8s_options.namespace, "inferenceservices", model_spec.model_details.name, spec, ) except ApiException as e: if e.status == 404: api_instance.create_namespaced_custom_object( "serving.kubeflow.org", "v1beta1", model_spec.runtime_options.k8s_options.namespace, "inferenceservices", spec, ) else: raise e
def create_k8s_client(): inside_cluster = os.getenv(ENV_K8S_SERVICE_HOST) if inside_cluster: logger.debug("Loading cluster local config") config.load_incluster_config() else: logger.debug("Loading external kubernetes config") config.load_kube_config()
def create_k8s_client(self): if self._inside_cluster(): logger.debug("Loading cluster local config") config.load_incluster_config() return True else: logger.debug("Loading external kubernetes config") config.load_kube_config() return False
def __init__(self): self.inside_cluster = os.getenv(ENV_K8S_SERVICE_HOST) try: if self.inside_cluster: logger.debug("Loading cluster local config") config.load_incluster_config() else: logger.debug("Loading external kubernetes config") config.load_kube_config() except Exception: logger.warning( "Failed to load kubeconfig. Only local mode is possible.")
def remote(self, model_spec: ModelSpec, *args, **kwargs) -> Any: req = model_spec.protocol.to_protocol_request(*args, **kwargs) endpoint = self.get_endpoint_spec(model_spec) logger.debug("Endpoint is", endpoint) headers = self.get_headers(model_spec) logger.debug("Headers are", headers) response_raw = requests.post( endpoint, json=req, headers=headers, verify=model_spec.runtime_options.ingress_options.verify_ssl) if response_raw.status_code == 200: return model_spec.protocol.from_protocol_response( response_raw.json(), model_spec.model_details.outputs) else: raise ValueError("Bad return code", response_raw.status_code, response_raw.text)
def __init__(self, model_details: ModelDetails, namespace, protocol: Protocol, gateway=ISTIO_GATEWAY): self.inside_cluster = os.getenv(ENV_K8S_SERVICE_HOST) try: if self.inside_cluster: logger.debug("Loading cluster local config") config.load_incluster_config() else: logger.debug("Loading external kubernetes config") config.load_kube_config() except: logger.warning( "Failed to load kubeconfig. Only local mode is possible.") self.gateway = gateway self.model_details = model_details self.namespace = namespace self.protocol = protocol
def deploy_spec(self, model_spec: ModelSpec): create_k8s_client() k8s_specer = KubernetesSpec(model_spec, self.seldon_core_options) k8s_spec = k8s_specer.spec logger.debug(k8s_spec) api_instance = client.CustomObjectsApi() try: existing = api_instance.get_namespaced_custom_object( "machinelearning.seldon.io", "v1", model_spec.runtime_options.k8s_options.namespace, "seldondeployments", model_spec.model_details.name, ) k8s_spec["metadata"]["resourceVersion"] = existing["metadata"]["resourceVersion"] api_instance.replace_namespaced_custom_object( "machinelearning.seldon.io", "v1", model_spec.runtime_options.k8s_options.namespace, "seldondeployments", model_spec.model_details.name, k8s_spec, ) except ApiException as e: if e.status == 404: api_instance.create_namespaced_custom_object( "machinelearning.seldon.io", "v1", model_spec.runtime_options.k8s_options.namespace, "seldondeployments", k8s_spec, ) else: raise e
def deploy_insights_message_dumper(): create_k8s_client() api_instance = client.CoreV1Api() try: api_instance.read_namespaced_pod( DefaultInsightsServiceName, DefaultSeldonSystemNamespace, ) logger.debug( f"Pod with name {DefaultInsightsServiceName} in namespace {DefaultSeldonSystemNamespace} already exists" ) except ApiException as e: if e.status != 404: raise e k8s_pod_spec = { "apiVersion": "v1", "kind": "Pod", "metadata": { "name": DefaultInsightsServiceName, "labels": { "app": DefaultInsightsServiceName, }, }, "spec": { "containers": [{ "name": "default", "image": DefaultInsightsImage, "ports": [{ "containerPort": DefaultInsightsPort }], }] }, } logger.debug( f"Creating kubernetes insights manager pod: {k8s_pod_spec}") api_instance.create_namespaced_pod(DefaultSeldonSystemNamespace, k8s_pod_spec) try: api_instance.read_namespaced_service( DefaultInsightsServiceName, DefaultSeldonSystemNamespace, ) logger.debug( f"Service with name {DefaultInsightsServiceName} in namespace {DefaultSeldonSystemNamespace} already exists" ) except ApiException as e: if e.status != 404: raise e k8s_svc_spec = { "apiVersion": "v1", "kind": "Service", "metadata": { "name": DefaultInsightsServiceName, }, "spec": { "selector": { "app": DefaultInsightsServiceName }, "ports": [{ "port": DefaultInsightsPort, "targetPort": DefaultInsightsPort, }], }, } logger.debug( f"Creating kubernetes insights manager pod: {k8s_svc_spec}") api_instance.create_namespaced_service(DefaultSeldonSystemNamespace, k8s_svc_spec)
def remote(self, model_spec: ModelSpec, *args, **kwargs) -> Any: req = model_spec.protocol.to_protocol_request(*args, **kwargs) endpoint = self.get_endpoint_spec(model_spec) logger.debug("Endpoint is ", endpoint) response_raw = requests.post(endpoint, json=req, verify=model_spec.runtime_options.ingress_options.verify_ssl) return model_spec.protocol.from_protocol_response(response_raw.json(), model_spec.model_details.outputs)