def test_transformer(): service_name = 'isvc-transformer' default_endpoint_spec = V1alpha2EndpointSpec( predictor=V1alpha2PredictorSpec( min_replicas=1, pytorch=V1alpha2PyTorchSpec( storage_uri='gs://kfserving-samples/models/pytorch/cifar10', model_class_name="Net", resources=V1ResourceRequirements( requests={'cpu': '100m', 'memory': '256Mi'}, limits={'cpu': '100m', 'memory': '256Mi'}))), transformer=V1alpha2TransformerSpec( min_replicas=1, custom=V1alpha2CustomSpec( container=V1Container( image='809251082950.dkr.ecr.us-west-2.amazonaws.com/kfserving/image-transformer:latest', name='kfserving-container', resources=V1ResourceRequirements( requests={'cpu': '100m', 'memory': '256Mi'}, limits={'cpu': '100m', 'memory': '256Mi'}))))) isvc = V1alpha2InferenceService(api_version=api_version, kind=constants.KFSERVING_KIND, metadata=client.V1ObjectMeta( name=service_name, namespace=KFSERVING_TEST_NAMESPACE), spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec)) KFServing.create(isvc) try: KFServing.wait_isvc_ready(service_name, namespace=KFSERVING_TEST_NAMESPACE) except RuntimeError as e: print(KFServing.api_instance.get_namespaced_custom_object("serving.knative.dev", "v1", KFSERVING_TEST_NAMESPACE, "services", service_name + "-predictor")) pods = KFServing.core_api.list_namespaced_pod(KFSERVING_TEST_NAMESPACE, label_selector='serving.kubeflow.org/inferenceservice={}' .format(service_name)) for pod in pods.items: print(pod) raise e res = predict(service_name, './data/transformer.json') assert(np.argmax(res["predictions"]) == 3) KFServing.delete(service_name, KFSERVING_TEST_NAMESPACE)
def EndpointSpec(framework, storage_uri): if framework == 'tensorflow': return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec( tensorflow=V1alpha2TensorflowSpec(storage_uri=storage_uri))) elif framework == 'pytorch': return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec( pytorch=V1alpha2PyTorchSpec(storage_uri=storage_uri))) elif framework == 'sklearn': return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec( sklearn=V1alpha2SKLearnSpec(storage_uri=storage_uri))) elif framework == 'xgboost': return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec( xgboost=V1alpha2XGBoostSpec(storage_uri=storage_uri))) elif framework == 'onnx': return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec( onnx=V1alpha2ONNXSpec(storage_uri=storage_uri))) elif framework == 'tensorrt': return V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec( tensorrt=V1alpha2TensorRTSpec(storage_uri=storage_uri))) else: raise ("Error: No matching framework: " + framework)
def test_batcher(): service_name = 'isvc-pytorch-batcher' default_endpoint_spec = V1alpha2EndpointSpec( predictor=V1alpha2PredictorSpec( batcher=V1alpha2Batcher( max_batchsize=32, max_latency=5000, timeout=60 ), min_replicas=1, pytorch=V1alpha2PyTorchSpec( storage_uri='gs://kfserving-samples/models/pytorch/cifar10', model_class_name='Net', resources=V1ResourceRequirements( requests={'cpu': '2000m', 'memory': '2Gi'}, limits={'cpu': '2000m', 'memory': '2Gi'})))) isvc = V1alpha2InferenceService(api_version=api_version, kind=constants.KFSERVING_KIND, metadata=client.V1ObjectMeta( name=service_name, namespace=KFSERVING_TEST_NAMESPACE ), spec=V1alpha2InferenceServiceSpec(default=default_endpoint_spec)) KFServing.create(isvc) try: KFServing.wait_isvc_ready(service_name, namespace=KFSERVING_TEST_NAMESPACE) except RuntimeError as e: print(KFServing.api_instance.get_namespaced_custom_object("serving.knative.dev", "v1", KFSERVING_TEST_NAMESPACE, "services", service_name + "-predictor-default")) pods = KFServing.core_api.list_namespaced_pod(KFSERVING_TEST_NAMESPACE, label_selector='serving.kubeflow.org/inferenceservice={}'. format(service_name)) for pod in pods.items: print(pod) raise e time.sleep(30) probs = predict(service_name, './data/cifar_input.json') assert(np.argmax(probs) == 3) KFServing.delete(service_name, KFSERVING_TEST_NAMESPACE)
def EndpointSpec(framework, storage_uri, service_account, min_replicas, max_replicas): endpointSpec = V1alpha2EndpointSpec(predictor=V1alpha2PredictorSpec( service_account_name=service_account, min_replicas=(min_replicas if min_replicas >= 0 else None), max_replicas=(max_replicas if max_replicas > 0 and max_replicas >= min_replicas else None))) if framework == "tensorflow": endpointSpec.predictor.tensorflow = V1alpha2TensorflowSpec( storage_uri=storage_uri) return endpointSpec elif framework == "pytorch": endpointSpec.predictor.pytorch = V1alpha2PyTorchSpec( storage_uri=storage_uri) return endpointSpec elif framework == "sklearn": endpointSpec.predictor.sklearn = V1alpha2SKLearnSpec( storage_uri=storage_uri) return endpointSpec elif framework == "xgboost": endpointSpec.predictor.xgboost = V1alpha2XGBoostSpec( storage_uri=storage_uri) return endpointSpec elif framework == "onnx": endpointSpec.predictor.onnx = V1alpha2ONNXSpec(storage_uri=storage_uri) return endpointSpec elif framework == "tensorrt": endpointSpec.predictor.tensorrt = V1alpha2TensorRTSpec( storage_uri=storage_uri) return endpointSpec else: raise ("Error: No matching framework: " + framework)