def test_transformer():
service_name = 'isvc-transformer'
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(
min_replicas=1,
pytorch=V1alpha2PyTorchSpec(
storage_uri='gs://kfserving-samples/models/pytorch/cifar10',
model_class_name="Net",
resources=V1ResourceRequirements(
requests={'cpu': '100m', 'memory': '256Mi'},
limits={'cpu': '100m', 'memory': '256Mi'}))),
transformer=V1alpha2TransformerSpec(
min_replicas=1,
custom=V1alpha2CustomSpec(
container=V1Container(
image='809251082950.dkr.ecr.us-west-2.amazonaws.com/kfserving/image-transformer:latest',
name='kfserving-container',
resources=V1ResourceRequirements(
requests={'cpu': '100m', 'memory': '256Mi'},
limits={'cpu': '100m', 'memory': '256Mi'})))))
isvc = V1alpha2InferenceService(api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=client.V1ObjectMeta(
name=service_name, namespace=KFSERVING_TEST_NAMESPACE),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec))
KFServing.create(isvc)
try:
KFServing.wait_isvc_ready(service_name, namespace=KFSERVING_TEST_NAMESPACE)
except RuntimeError as e:
print(KFServing.api_instance.get_namespaced_custom_object("serving.knative.dev", "v1", KFSERVING_TEST_NAMESPACE,
"services", service_name + "-predictor"))
pods = KFServing.core_api.list_namespaced_pod(KFSERVING_TEST_NAMESPACE,
label_selector='serving.kubeflow.org/inferenceservice={}'
.format(service_name))
for pod in pods.items:
print(pod)
raise e
res = predict(service_name, './data/transformer.json')
assert(np.argmax(res["predictions"]) == 3)
KFServing.delete(service_name, KFSERVING_TEST_NAMESPACE)
def EndpointSpec(framework, storage_uri):
if framework == 'tensorflow':
return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec(
tensorflow=V1alpha2TensorflowSpec(storage_uri=storage_uri)))
elif framework == 'pytorch':
return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec(
pytorch=V1alpha2PyTorchSpec(storage_uri=storage_uri)))
elif framework == 'sklearn':
return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec(
sklearn=V1alpha2SKLearnSpec(storage_uri=storage_uri)))
elif framework == 'xgboost':
return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec(
xgboost=V1alpha2XGBoostSpec(storage_uri=storage_uri)))
elif framework == 'onnx':
return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec(
onnx=V1alpha2ONNXSpec(storage_uri=storage_uri)))
elif framework == 'tensorrt':
return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec(
tensorrt=V1alpha2TensorRTSpec(storage_uri=storage_uri)))
else:
raise ("Error: No matching framework: " + framework)
def test_batcher():
service_name = 'isvc-pytorch-batcher'
default_endpoint_spec = V1alpha2EndpointSpec(
predictor=V1alpha2PredictorSpec(
batcher=V1alpha2Batcher(
max_batchsize=32,
max_latency=5000,
timeout=60
),
min_replicas=1,
pytorch=V1alpha2PyTorchSpec(
storage_uri='gs://kfserving-samples/models/pytorch/cifar10',
model_class_name='Net',
resources=V1ResourceRequirements(
requests={'cpu': '2000m', 'memory': '2Gi'},
limits={'cpu': '2000m', 'memory': '2Gi'}))))
isvc = V1alpha2InferenceService(api_version=api_version,
kind=constants.KFSERVING_KIND,
metadata=client.V1ObjectMeta(
name=service_name,
namespace=KFSERVING_TEST_NAMESPACE
),
spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec))
KFServing.create(isvc)
try:
KFServing.wait_isvc_ready(service_name, namespace=KFSERVING_TEST_NAMESPACE)
except RuntimeError as e:
print(KFServing.api_instance.get_namespaced_custom_object("serving.knative.dev", "v1", KFSERVING_TEST_NAMESPACE,
"services", service_name + "-predictor-default"))
pods = KFServing.core_api.list_namespaced_pod(KFSERVING_TEST_NAMESPACE,
label_selector='serving.kubeflow.org/inferenceservice={}'.
format(service_name))
for pod in pods.items:
print(pod)
raise e
time.sleep(30)
probs = predict(service_name, './data/cifar_input.json')
assert(np.argmax(probs) == 3)
KFServing.delete(service_name, KFSERVING_TEST_NAMESPACE)
def EndpointSpec(framework, storage_uri, service_account, min_replicas,
max_replicas):
endpointSpec = V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec(
service_account_name=service_account,
min_replicas=(min_replicas if min_replicas >= 0 else None),
max_replicas=(max_replicas if max_replicas > 0
and max_replicas >= min_replicas else None)))
if framework == "tensorflow":
endpointSpec.predictor.tensorflow = V1alpha2TensorflowSpec(
storage_uri=storage_uri)
return endpointSpec
elif framework == "pytorch":
endpointSpec.predictor.pytorch = V1alpha2PyTorchSpec(
storage_uri=storage_uri)
return endpointSpec
elif framework == "sklearn":
endpointSpec.predictor.sklearn = V1alpha2SKLearnSpec(
storage_uri=storage_uri)
return endpointSpec
elif framework == "xgboost":
endpointSpec.predictor.xgboost = V1alpha2XGBoostSpec(
storage_uri=storage_uri)
return endpointSpec
elif framework == "onnx":
endpointSpec.predictor.onnx = V1alpha2ONNXSpec(storage_uri=storage_uri)
return endpointSpec
elif framework == "tensorrt":
endpointSpec.predictor.tensorrt = V1alpha2TensorRTSpec(
storage_uri=storage_uri)
return endpointSpec
else:
raise ("Error: No matching framework: " + framework)