def deploy(self, model_details: ModelDetails):
model_spec = self._get_spec(model_details)
logger.debug(model_spec)
api_instance = client.CustomObjectsApi()
try:
existing = api_instance.get_namespaced_custom_object(
"serving.kubeflow.org",
"v1alpha2",
self.k8s_options.namespace,
"inferenceservices",
model_details.name,
)
model_spec["metadata"]["resourceVersion"] = existing["metadata"][
"resourceVersion"]
api_instance.replace_namespaced_custom_object(
"serving.kubeflow.org",
"v1alpha2",
self.k8s_options.namespace,
"inferenceservices",
model_details.name,
model_spec,
)
except ApiException as e:
if e.status == 404:
api_instance.create_namespaced_custom_object(
"serving.kubeflow.org",
"v1alpha2",
self.k8s_options.namespace,
"inferenceservices",
model_spec,
)
else:
raise e
def deploy(self, model_details: ModelDetails):
model_spec = self._get_spec(model_details)
logger.debug(model_spec)
api_instance = client.CustomObjectsApi()
try:
existing = api_instance.get_namespaced_custom_object(
"machinelearning.seldon.io",
"v1",
self.k8s_options.namespace,
"seldondeployments",
model_details.name,
)
model_spec["metadata"]["resourceVersion"] = existing["metadata"][
"resourceVersion"]
api_instance.replace_namespaced_custom_object(
"machinelearning.seldon.io",
"v1",
self.k8s_options.namespace,
"seldondeployments",
model_details.name,
model_spec,
)
except ApiException as e:
if e.status == 404:
api_instance.create_namespaced_custom_object(
"machinelearning.seldon.io",
"v1",
self.k8s_options.namespace,
"seldondeployments",
model_spec,
)
else:
raise e
def deploy_spec(self, model_spec: ModelSpec):
spec = self._get_spec(model_spec)
logger.debug(model_spec)
self.create_k8s_client()
api_instance = client.CustomObjectsApi()
try:
existing = api_instance.get_namespaced_custom_object(
"serving.kubeflow.org",
"v1beta1",
model_spec.runtime_options.k8s_options.namespace,
"inferenceservices",
model_spec.model_details.name,
)
spec["metadata"]["resourceVersion"] = existing["metadata"]["resourceVersion"]
api_instance.replace_namespaced_custom_object(
"serving.kubeflow.org",
"v1beta1",
model_spec.runtime_options.k8s_options.namespace,
"inferenceservices",
model_spec.model_details.name,
spec,
)
except ApiException as e:
if e.status == 404:
api_instance.create_namespaced_custom_object(
"serving.kubeflow.org",
"v1beta1",
model_spec.runtime_options.k8s_options.namespace,
"inferenceservices",
spec,
)
else:
raise e
def create_k8s_client():
inside_cluster = os.getenv(ENV_K8S_SERVICE_HOST)
if inside_cluster:
logger.debug("Loading cluster local config")
config.load_incluster_config()
else:
logger.debug("Loading external kubernetes config")
config.load_kube_config()
def create_k8s_client(self):
if self._inside_cluster():
logger.debug("Loading cluster local config")
config.load_incluster_config()
return True
else:
logger.debug("Loading external kubernetes config")
config.load_kube_config()
return False
def __init__(self):
self.inside_cluster = os.getenv(ENV_K8S_SERVICE_HOST)
try:
if self.inside_cluster:
logger.debug("Loading cluster local config")
config.load_incluster_config()
else:
logger.debug("Loading external kubernetes config")
config.load_kube_config()
except Exception:
logger.warning(
"Failed to load kubeconfig. Only local mode is possible.")
def remote(self, model_spec: ModelSpec, *args, **kwargs) -> Any:
req = model_spec.protocol.to_protocol_request(*args, **kwargs)
endpoint = self.get_endpoint_spec(model_spec)
logger.debug("Endpoint is", endpoint)
headers = self.get_headers(model_spec)
logger.debug("Headers are", headers)
response_raw = requests.post(
endpoint,
json=req,
headers=headers,
verify=model_spec.runtime_options.ingress_options.verify_ssl)
if response_raw.status_code == 200:
return model_spec.protocol.from_protocol_response(
response_raw.json(), model_spec.model_details.outputs)
else:
raise ValueError("Bad return code", response_raw.status_code,
response_raw.text)
def __init__(self,
model_details: ModelDetails,
namespace,
protocol: Protocol,
gateway=ISTIO_GATEWAY):
self.inside_cluster = os.getenv(ENV_K8S_SERVICE_HOST)
try:
if self.inside_cluster:
logger.debug("Loading cluster local config")
config.load_incluster_config()
else:
logger.debug("Loading external kubernetes config")
config.load_kube_config()
except:
logger.warning(
"Failed to load kubeconfig. Only local mode is possible.")
self.gateway = gateway
self.model_details = model_details
self.namespace = namespace
self.protocol = protocol
def deploy_spec(self, model_spec: ModelSpec):
create_k8s_client()
k8s_specer = KubernetesSpec(model_spec, self.seldon_core_options)
k8s_spec = k8s_specer.spec
logger.debug(k8s_spec)
api_instance = client.CustomObjectsApi()
try:
existing = api_instance.get_namespaced_custom_object(
"machinelearning.seldon.io",
"v1",
model_spec.runtime_options.k8s_options.namespace,
"seldondeployments",
model_spec.model_details.name,
)
k8s_spec["metadata"]["resourceVersion"] = existing["metadata"]["resourceVersion"]
api_instance.replace_namespaced_custom_object(
"machinelearning.seldon.io",
"v1",
model_spec.runtime_options.k8s_options.namespace,
"seldondeployments",
model_spec.model_details.name,
k8s_spec,
)
except ApiException as e:
if e.status == 404:
api_instance.create_namespaced_custom_object(
"machinelearning.seldon.io",
"v1",
model_spec.runtime_options.k8s_options.namespace,
"seldondeployments",
k8s_spec,
)
else:
raise e
def deploy_insights_message_dumper():
create_k8s_client()
api_instance = client.CoreV1Api()
try:
api_instance.read_namespaced_pod(
DefaultInsightsServiceName,
DefaultSeldonSystemNamespace,
)
logger.debug(
f"Pod with name {DefaultInsightsServiceName} in namespace {DefaultSeldonSystemNamespace} already exists"
)
except ApiException as e:
if e.status != 404:
raise e
k8s_pod_spec = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": DefaultInsightsServiceName,
"labels": {
"app": DefaultInsightsServiceName,
},
},
"spec": {
"containers": [{
"name":
"default",
"image":
DefaultInsightsImage,
"ports": [{
"containerPort": DefaultInsightsPort
}],
}]
},
}
logger.debug(
f"Creating kubernetes insights manager pod: {k8s_pod_spec}")
api_instance.create_namespaced_pod(DefaultSeldonSystemNamespace,
k8s_pod_spec)
try:
api_instance.read_namespaced_service(
DefaultInsightsServiceName,
DefaultSeldonSystemNamespace,
)
logger.debug(
f"Service with name {DefaultInsightsServiceName} in namespace {DefaultSeldonSystemNamespace} already exists"
)
except ApiException as e:
if e.status != 404:
raise e
k8s_svc_spec = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": DefaultInsightsServiceName,
},
"spec": {
"selector": {
"app": DefaultInsightsServiceName
},
"ports": [{
"port": DefaultInsightsPort,
"targetPort": DefaultInsightsPort,
}],
},
}
logger.debug(
f"Creating kubernetes insights manager pod: {k8s_svc_spec}")
api_instance.create_namespaced_service(DefaultSeldonSystemNamespace,
k8s_svc_spec)
def remote(self, model_spec: ModelSpec, *args, **kwargs) -> Any:
req = model_spec.protocol.to_protocol_request(*args, **kwargs)
endpoint = self.get_endpoint_spec(model_spec)
logger.debug("Endpoint is ", endpoint)
response_raw = requests.post(endpoint, json=req, verify=model_spec.runtime_options.ingress_options.verify_ssl)
return model_spec.protocol.from_protocol_response(response_raw.json(), model_spec.model_details.outputs)
