def create_inference_service(namespace: str, name: str, storage_url: str,
runtime_version: str, service_account_name: str):
api_version = os.path.join(constants.KFSERVING_GROUP,
constants.KFSERVING_VERSION)
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(
min_replicas=1,
service_account_name=service_account_name,
triton=V1alpha2TritonSpec(runtime_version=runtime_version,
storage_uri=storage_url,
resources=V1ResourceRequirements(
requests={
'cpu': '100m',
'memeory': '1Gi'
},
limits={
'cpu': '100m',
'memory': '1Gi'
}))))
isvc = V1alpha2InferenceService(
api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=client.V1ObjectMeta(name=name, namespace=namespace),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec),
)
KFServing = KFServingClient()
KFServing.create(isvc)
KFServing.get(name, namespace=namespace, watch=True, timeout_seconds=300)
def run(self):
parser = argparse.ArgumentParser()
parser.add_argument('--namespace', required=False, default='kubeflow')
# pvc://${PVCNAME}/dir
parser.add_argument('--storage_uri', required=False, default='/mnt/export')
parser.add_argument('--name', required=False, default='kfserving-sample')
args = parser.parse_args()
namespace = args.namespace
serving_name = args.name
api_version = constants.KFSERVING_GROUP + '/' + constants.KFSERVING_VERSION
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(
tensorflow=V1alpha2TensorflowSpec(
storage_uri=args.storage_uri,
resources=V1ResourceRequirements(
requests={'cpu':'100m','memory':'1Gi'},
limits={'cpu':'100m', 'memory':'1Gi'}))))
isvc = V1alpha2InferenceService(api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=client.V1ObjectMeta(
name=serving_name, namespace=namespace),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec))
KFServing = KFServingClient()
KFServing.create(isvc)
print('waiting 5 sec for Creating InferenceService')
time.sleep(5)
KFServing.get(serving_name, namespace=namespace, watch=True, timeout_seconds=300)
def deploy_model(action, model_name, default_model_uri, canary_model_uri, canary_model_traffic, namespace, framework, default_custom_model_spec, canary_custom_model_spec, autoscaling_target=0):
if int(autoscaling_target) != 0:
annotations = {"autoscaling.knative.dev/target": str(autoscaling_target)}
else:
annotations = None
metadata = client.V1ObjectMeta(name=model_name, namespace=namespace, annotations=annotations)
if framework != 'custom':
default_model_spec = ModelSpec(framework, default_model_uri)
else:
default_model_spec = customModelSpec(default_custom_model_spec)
# Create Canary deployment if canary model uri is provided.
if framework != 'custom' and canary_model_uri:
canary_model_spec = ModelSpec(framework, canary_model_uri)
kfsvc = kfserving_deployment(metadata, default_model_spec, canary_model_spec, canary_model_traffic)
elif framework == 'custom' and canary_custom_model_spec:
canary_model_spec = customModelSpec(canary_custom_model_spec)
kfsvc = kfserving_deployment(metadata, default_model_spec, canary_model_spec, canary_model_traffic)
else:
kfsvc = kfserving_deployment(metadata, default_model_spec)
KFServing = KFServingClient()
if action == 'create':
KFServing.create(kfsvc)
elif action == 'update':
KFServing.patch(model_name, kfsvc)
elif action == 'delete':
KFServing.delete(model_name, namespace=namespace)
else:
raise("Error: No matching action: " + action)
model_status = KFServing.get(model_name, namespace=namespace)
return model_status
def deploy_model(namespace: str, trained_model_path: InputPath(str)):
from kubernetes import client
from kfserving import KFServingClient
from kfserving import constants
from kfserving import V1alpha2EndpointSpec
from kfserving import V1alpha2PredictorSpec
from kfserving import V1alpha2TensorflowSpec
from kfserving import V1alpha2InferenceServiceSpec
from kfserving import V1alpha2InferenceService
from kubernetes.client import V1ResourceRequirements
api_version = constants.KFSERVING_GROUP + '/' + constants.KFSERVING_VERSION
inference_service_name = 'inference112cbk'
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(tensorflow=V1alpha2TensorflowSpec(
storage_uri=trained_model_path,
resources=V1ResourceRequirements(requests={
'cpu': '100m',
'memory': '1Gi'
},
limits={
'cpu': '100m',
'memory': '1Gi'
}))))
isvc = V1alpha2InferenceService(
api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=client.V1ObjectMeta(name=inference_service_name,
namespace=namespace),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec))
KFServing = KFServingClient()
KFServing.create(isvc)
print('Inference service ' + inference_service_name + " created ...")
KFServing.get(inference_service_name,
namespace=namespace,
watch=True,
timeout_seconds=120)
print('Model deployed')
def deploy_model(namespace,trained_model_path):
logging.basicConfig(level=logging.INFO)
logging.info('Starting deploy model step ..')
logging.info('Input data ..')
logging.info('namespace:{}'.format(namespace))
logging.info('trained_model_path:{}'.format(trained_model_path))
logging.info('STEP: DEPLOY MODEL (1/2) Generating definition..')
api_version = constants.KFSERVING_GROUP + '/' + constants.KFSERVING_VERSION
now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
inference_service_name = 'simple-model'+now
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(
tensorflow=V1alpha2TensorflowSpec(
storage_uri=trained_model_path,
resources=V1ResourceRequirements(
requests={'cpu': '100m', 'memory': '1Gi'},
limits={'cpu': '100m', 'memory': '1Gi'}))))
isvc = V1alpha2InferenceService(api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=client.V1ObjectMeta(
name=inference_service_name,
annotations=
{
'sidecar.istio.io/inject': 'false',
'autoscaling.knative.dev/target': '1'
},
namespace=namespace),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec))
#velascoluis: sidecar is disabled by https://github.com/knative/serving/issues/6829
#Note: make sure trained model path starts with file:// or gs://
KFServing = KFServingClient()
logging.info('STEP: DEPLOY MODEL (2/2) Creating inference service..')
KFServing.create(isvc)
logging.info('Inference service ' + inference_service_name + " created ...")
KFServing.get(inference_service_name, namespace=namespace, watch=True, timeout_seconds=120)
logging.info('Deploy model step finished')
def deploy_model(namespace, model_file_name, gcp_bucket):
api_version = constants.KFSERVING_GROUP + '/' + constants.KFSERVING_VERSION
now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
inference_service_name = 'xgboost-r' + now
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(
min_replicas=1,
xgboost=V1alpha2XGBoostSpec(
#storage_uri='gs://'+gcp_bucket+'/rmodel/'+model_file_name,
storage_uri='gs://' + gcp_bucket + '/rmodel',
resources=V1ResourceRequirements(requests={
'cpu': '100m',
'memory': '1Gi'
},
limits={
'cpu': '100m',
'memory': '1Gi'
}))))
isvc = V1alpha2InferenceService(
api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=client.V1ObjectMeta(
name=inference_service_name,
namespace=namespace,
annotations={'sidecar.istio.io/inject': 'false'}),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec))
#@velascoluis - annotation The sidecar.istio.io/inject: "false", otherwise the ingress does not work
KFServing = KFServingClient()
KFServing.create(isvc)
KFServing.get(inference_service_name,
namespace=namespace,
watch=True,
timeout_seconds=120)
def create_isvc(self, namespace, isvc):
"""Create the provided InferenceService in the specified namespace.
:param namespace: The custom resource
:param InferenceService: The InferenceService body
:returns: object: Created InferenceService.
"""
KFServing = KFServingClient()
try:
created_isvc = KFServing.create(isvc, namespace=namespace)
isvc_name = created_isvc['metadata']['name']
isvc_namespace = created_isvc['metadata']['namespace']
KFServing.get(isvc_name, isvc_namespace, watch=True)
return created_isvc
except client.rest.ApiException:
raise RuntimeError("Failed to create InferenceService. Perhaps the CRD "
"InferenceService version {} is not installed? "\
.format(constants.KFSERVING_VERSION))
def deploy_model(action,
model_name,
default_model_uri,
canary_model_uri,
canary_model_traffic,
namespace,
framework,
default_custom_model_spec,
canary_custom_model_spec,
autoscaling_target=0):
if int(autoscaling_target) != 0:
annotations = {
"autoscaling.knative.dev/target": str(autoscaling_target)
}
else:
annotations = None
metadata = client.V1ObjectMeta(name=model_name,
namespace=namespace,
annotations=annotations)
# Create Default deployment if default model uri is provided.
if framework != 'custom' and default_model_uri:
default_model_spec = EndpointSpec(framework, default_model_uri)
elif framework == 'custom' and default_custom_model_spec:
default_model_spec = customEndpointSpec(default_custom_model_spec)
# Create Canary deployment if canary model uri is provided.
if framework != 'custom' and canary_model_uri:
canary_model_spec = EndpointSpec(framework, canary_model_uri)
kfsvc = InferenceService(metadata, default_model_spec,
canary_model_spec, canary_model_traffic)
elif framework == 'custom' and canary_custom_model_spec:
canary_model_spec = customEndpointSpec(canary_custom_model_spec)
kfsvc = InferenceService(metadata, default_model_spec,
canary_model_spec, canary_model_traffic)
else:
kfsvc = InferenceService(metadata, default_model_spec)
KFServing = KFServingClient()
if action == 'create':
KFServing.create(kfsvc, watch=True, timeout_seconds=120)
elif action == 'update':
KFServing.patch(model_name, kfsvc)
elif action == 'rollout':
KFServing.rollout_canary(model_name,
canary=canary_model_spec,
percent=canary_model_traffic,
namespace=namespace,
watch=True,
timeout_seconds=120)
elif action == 'promote':
KFServing.promote(model_name,
namespace=namespace,
watch=True,
timeout_seconds=120)
elif action == 'delete':
KFServing.delete(model_name, namespace=namespace)
else:
raise ("Error: No matching action: " + action)
model_status = KFServing.get(model_name, namespace=namespace)
return model_status
def deploy(self, kfservice_id):
mykfservice = db.session.query(KfService).filter_by(
id=kfservice_id).first()
namespace = conf.get('KFSERVING_NAMESPACE')
crd_info = conf.get('CRD_INFO')['inferenceservice']
# 根据service生成container
def make_container(service, mykfservice):
from myapp.utils.py.py_k8s import K8s
k8s = K8s() # 不部署，不需要配置集群信息
container = k8s.make_container(
name=mykfservice.name + "-" + service.name,
command=["sh", "-c", service.command]
if service.command else None,
args=None,
volume_mount=None,
image_pull_policy=conf.get('IMAGE_PULL_POLICY', 'Always'),
image=service.images,
working_dir=service.working_dir
if service.working_dir else None,
env=service.env,
resource_memory=service.resource_memory,
resource_cpu=service.resource_cpu,
resource_gpu=service.resource_gpu,
username=service.created_by.username)
return container
api_version = crd_info['group'] + '/' + crd_info['version']
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(
min_replicas=mykfservice.default_service.min_replicas,
max_replicas=mykfservice.default_service.max_replicas,
custom=V1alpha2CustomSpec(container=make_container(
mykfservice.default_service,
mykfservice)))) if mykfservice.default_service else None
canary_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(
min_replicas=mykfservice.canary_service.min_replicas,
max_replicas=mykfservice.canary_service.max_replicas,
custom=V1alpha2CustomSpec(container=make_container(
mykfservice.canary_service,
mykfservice)))) if mykfservice.canary_service else None
metadata = kubernetes.client.V1ObjectMeta(
name=mykfservice.name,
labels={
"app": mykfservice.name,
"rtx-user": mykfservice.created_by.username
},
namespace=namespace)
isvc = V1alpha2InferenceService(
api_version=api_version,
kind=crd_info['kind'],
metadata=metadata,
spec=V1alpha2InferenceServiceSpec(
default=default_endpoint_spec,
canary=canary_endpoint_spec,
canary_traffic_percent=mykfservice.canary_traffic_percent))
KFServing = KFServingClient()
try:
KFServing.delete(mykfservice.name,
namespace=namespace,
version=crd_info['version'])
except Exception as e:
print(e)
KFServing.create(isvc,
namespace=namespace,
version=crd_info['version'])
flash(category='warning', message='部署启动，一分钟后部署完成')
return redirect('/kfservice_modelview/list/')
# Create Canary deployment if canary model uri is provided.
if framework != 'custom' and canary_model_uri:
canary_model_spec = ModelSpec(framework, canary_model_uri)
kfsvc = kfserving_deployment(metadata, default_model_spec,
canary_model_spec, canary_model_traffic)
elif framework == 'custom' and canary_custom_model_spec:
canary_model_spec = customModelSpec(canary_custom_model_spec)
kfsvc = kfserving_deployment(metadata, default_model_spec,
canary_model_spec, canary_model_traffic)
else:
kfsvc = kfserving_deployment(metadata, default_model_spec)
KFServing = KFServingClient()
if action == 'create':
KFServing.create(kfsvc)
elif action == 'update':
KFServing.patch(model_name, kfsvc)
elif action == 'delete':
KFServing.delete(model_name, namespace=namespace)
else:
raise ("Error: No matching action: " + action)
model_status = KFServing.get(model_name, namespace=namespace)
print(model_status)
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
with open(output_path, "w") as report:
report.write(json.dumps(model_status))
default_endpoint_spec = V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec(
tensorflow=V1alpha2TensorflowSpec(
storage_uri="s3://anonymous-model-result/result/saved_model",
resources=V1ResourceRequirements(requests={
"cpu": "100m",
"memory": "1Gi"
},
limits={
"cpu": "100m",
"memory": "1Gi"
}))))
isvc = V1alpha2InferenceService(
api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=client.V1ObjectMeta(name="mnist-kfserving", namespace=namespace),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec))
KFServing = KFServingClient()
KFServing.set_credentials(
storage_type="S3",
namespace='anonymous',
credentials_file='credentials',
s3_profile="default",
s3_endpoint="minio-service.kubeflow.svc.cluster.local:9000",
s3_region="us-west-1",
s3_use_https="0",
s3_verify_ssl="0")
KFServing.create(isvc)
def main():
api_version = constants.KFSERVING_GROUP + '/' + constants.KFSERVING_VERSION
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(custom=V1alpha2CustomSpec(
container=V1Container(
name="kfserving-container",
image=FLAGS.image,
env=[{
"name": "STORAGE_URI",
"value": "%s" % FLAGS.storage_uri
}],
resources=V1ResourceRequirements(
limits={"nvidia.com/gpu": FLAGS.gpus_to_inference}),
command=["python"],
args=[
"model.py",
"--model-name",
"%s" % FLAGS.inference_name,
"--out_dir",
"%s" % FLAGS.model_path,
"--classes_file",
"%s" % FLAGS.classes_file,
]))))
isvc = V1alpha2InferenceService(
api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=k8s_client.V1ObjectMeta(name=FLAGS.inference_name,
namespace=FLAGS.namespace),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec))
# Create inference service
KFServing = KFServingClient()
KFServing.create(isvc)
time.sleep(2)
# Check inference service
KFServing.get(FLAGS.inference_name,
namespace=FLAGS.namespace,
watch=True,
timeout_seconds=180)
model_status = KFServing.get(FLAGS.inference_name,
namespace=FLAGS.namespace)
for condition in model_status["status"]["conditions"]:
if condition['type'] == 'Ready':
if condition['status'] == 'True':
print('Model is ready')
break
else:
print(
'Model is timed out, please check the inferenceservice events for more details.'
)
exit(1)
try:
print(
model_status["status"]["url"] +
" is the knative domain header. $ISTIO_INGRESS_ENDPOINT are defined in the below commands"
)
print("Sample test commands: ")
print(
"# Note: If Istio Ingress gateway is not served with LoadBalancer, use $CLUSTER_NODE_IP:31380 as the ISTIO_INGRESS_ENDPOINT"
)
print(
"ISTIO_INGRESS_ENDPOINT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
)
# model_status['status']['url'] is like http://flowers-sample.kubeflow.example.com/v1/models/flowers-sample
url = re.compile(r"https?://")
host, path = url.sub("", model_status["status"]["url"]).split("/", 1)
print('curl -X GET -H "Host: ' + host +
'" http://$ISTIO_INGRESS_ENDPOINT/' + path)
except:
print("Model is not ready, check the logs for the Knative URL status.")
exit(1)