def test_ovms_serving_status(ovms_endpoint):
ovms_endpoint_response, namespace = ovms_endpoint
url = json.loads(ovms_endpoint_response.text)['data']['url']
start_time = time.time()
tick = start_time
running = False
while tick - start_time < 100:
tick = time.time()
try:
all_pods = get_all_pods_in_namespace(namespace)
pod_name = all_pods.items[0].metadata.name
logging.info("Pod name :", pod_name)
logs = get_logs_of_pod("ovms", pod_name)
logging.info(logs)
if "Server listens on port 9000 and will be serving models" in logs:
running = True
break
except Exception as e:
logging.info(e)
time.sleep(10)
assert running is True
trusted_cert, trusted_key, trusted_ca = prepare_certs(
CERT_SERVER,
CERT_CLIENT_KEY,
CERT_CLIENT)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert, private_key=trusted_key,
certificate_chain=trusted_ca)
stub, _ = prepare_stub_and_request(url, "", creds=creds)
request = get_model_metadata_request("ovms_resnet")
response = stub.GetModelMetadata(request, 10)
assert "ovms_resnet" == response.model_spec.name
def test_create_endpoint():
endpoint_response = create_endpoint()
assert "created" in endpoint_response.text
assert endpoint_response.status_code == 200
start_time = time.time()
tick = start_time
running = False
pod_name = None
while tick - start_time < 100:
tick = time.time()
try:
all_pods = get_all_pods_in_namespace(TENANT_NAME)
pod_name = all_pods.items[0].metadata.name
logging.info("Pod name :", pod_name)
logs = get_logs_of_pod(TENANT_NAME, pod_name)
logs = filter_serving_logs(logs)
logging.info(logs)
if "Running gRPC ModelServer at 0.0.0.0:9000" in logs:
running = True
break
except Exception as e:
logging.info(e)
time.sleep(10)
assert running is True
test_create_endpoint.endpoint_info = get_url_from_response(endpoint_response)
test_create_endpoint.pod_name = pod_name
return endpoint_response
def test_prediction_with_certificates():
url = test_create_endpoint.endpoint_info
trusted_cert, trusted_key, trusted_ca = prepare_certs(
CERT_SERVER,
CERT_CLIENT_KEY,
CERT_CLIENT)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert,
private_key=trusted_key, certificate_chain=trusted_ca)
stub, request = prepare_stub_and_request(url, MODEL_NAME, creds)
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(image, shape=image.shape))
prediction_response = "Failed"
try:
prediction_response = stub.Predict(request, 10.0)
except: # noqa
logging.info("Prediction failed")
pass
logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name)
logging.info(filter_serving_logs(logs))
assert not prediction_response == "Failed"
response = numpy.array(prediction_response.outputs[model_output].float_val)
max_output = numpy.argmax(response) - 1
num_label = classes.imagenet_classes[max_output]
test_label = classes.imagenet_classes[first_label]
assert max_output == first_label
assert num_label == test_label
def test_version_not_served():
stub, request = prepare_stub_and_request(endpoint_info.url,
MODEL_NAME,
model_version=1,
creds=endpoint_info.credentials)
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(image, shape=image.shape))
with pytest.raises(grpc.RpcError) as context:
stub.Predict(request, RPC_TIMEOUT)
logs = get_logs_of_pod(TENANT_NAME, endpoint_info.pod_name)
logging.info(filter_serving_logs(logs))
assert "Servable not found" in context.value.details()
def perform_inference(rpc_timeout=RPC_TIMEOUT, image=image):
stub, request = prepare_stub_and_request(endpoint_info.url,
MODEL_NAME,
creds=endpoint_info.credentials)
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(image, shape=image.shape))
prediction_response = "Failed"
try:
prediction_response = stub.Predict(request, rpc_timeout)
except Exception as e: # noqa
logging.info("Prediction failed: " + str(e))
print(str(e))
pass
logs = get_logs_of_pod(TENANT_NAME, endpoint_info.pod_name)
logging.info(filter_serving_logs(logs))
return prediction_response
def test_no_certificates():
url = test_create_endpoint.endpoint_info
trusted_cert, _, _ = prepare_certs(CERT_SERVER)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert)
stub, request = prepare_stub_and_request(url, MODEL_NAME, creds)
numpy_input = numpy.zeros((1, 224, 224, 3), numpy.dtype('<f'))
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(numpy_input, shape=[1, 224, 224, 3]))
with pytest.raises(grpc.RpcError) as context:
stub.Predict(request, 10.0)
logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name)
logging.info(filter_serving_logs(logs))
assert context.value.details() == 'Received http2 header with status: 400'
def test_prediction_batch_with_certificates():
url = test_create_endpoint.endpoint_info
trusted_cert, trusted_key, trusted_ca = prepare_certs(
CERT_SERVER,
CERT_CLIENT_KEY,
CERT_CLIENT)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert,
private_key=trusted_key, certificate_chain=trusted_ca)
stub, request = prepare_stub_and_request(url, MODEL_NAME, creds)
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(images, shape=images.shape))
prediction_response = "Failed"
try:
prediction_response = stub.Predict(request, 10.0)
except: # noqa
logging.info("Prediction failed")
logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name)
logging.info(filter_serving_logs(logs))
response = numpy.array(prediction_response.outputs[model_output].float_val)
offset = 1001
max_outputs = []
for i in range(0, len(response), offset):
one_output = response[i:i + offset]
max_output = numpy.argmax(one_output) - 1
max_outputs.append(max_output)
for i in range(len(max_outputs)):
label = classes.imagenet_classes[max_outputs[i]]
test_label = classes.imagenet_classes[labels[i]]
assert max_outputs[i] == labels[i]
assert label == test_label
def wait_endpoint_setup():
start_time = time.time()
tick = start_time
running = False
pod_name = None
while tick - start_time < 100:
tick = time.time()
try:
all_pods = get_all_pods_in_namespace(TENANT_NAME)
all_pods.items.sort(key=lambda pod: pod.status.start_time,
reverse=True)
pod_name = all_pods.items[0].metadata.name
logging.info("Pod name :", pod_name)
logs = get_logs_of_pod(TENANT_NAME, pod_name)
logs = filter_serving_logs(logs)
logging.info(logs)
if "Running gRPC ModelServer at 0.0.0.0:9000" in logs:
running = True
break
except Exception as e:
logging.info(e)
time.sleep(10)
return running, pod_name