def test_ovms_serving_status(ovms_endpoint):
ovms_endpoint_response, namespace = ovms_endpoint
url = json.loads(ovms_endpoint_response.text)['data']['url']
start_time = time.time()
tick = start_time
running = False
while tick - start_time < 100:
tick = time.time()
try:
all_pods = get_all_pods_in_namespace(namespace)
pod_name = all_pods.items[0].metadata.name
logging.info("Pod name :", pod_name)
logs = get_logs_of_pod("ovms", pod_name)
logging.info(logs)
if "Server listens on port 9000 and will be serving models" in logs:
running = True
break
except Exception as e:
logging.info(e)
time.sleep(10)
assert running is True
trusted_cert, trusted_key, trusted_ca = prepare_certs(
CERT_SERVER,
CERT_CLIENT_KEY,
CERT_CLIENT)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert, private_key=trusted_key,
certificate_chain=trusted_ca)
stub, _ = prepare_stub_and_request(url, "", creds=creds)
request = get_model_metadata_request("ovms_resnet")
response = stub.GetModelMetadata(request, 10)
assert "ovms_resnet" == response.model_spec.name
def test_prediction_with_certificates():
url = test_create_endpoint.endpoint_info
trusted_cert, trusted_key, trusted_ca = prepare_certs(
CERT_SERVER, CERT_CLIENT_KEY, CERT_CLIENT)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert,
private_key=trusted_key,
certificate_chain=trusted_ca)
stub, request = prepare_stub_and_request(url, MODEL_NAME, creds)
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(image, shape=image.shape))
prediction_response = "Failed"
try:
prediction_response = stub.Predict(request, 10.0)
except: # noqa
logging.info("Prediction failed")
pass
logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name)
logging.info(filter_serving_logs(logs))
assert not prediction_response == "Failed"
response = numpy.array(prediction_response.outputs[model_output].float_val)
max_output = numpy.argmax(response) - 1
num_label = classes.imagenet_classes[max_output]
test_label = classes.imagenet_classes[first_label]
assert max_output == first_label
assert num_label == test_label
def get_stub_and_request(endpoint_address, model_name, certs, ssl,
target_name):
if ssl:
server_ca_cert, client_key, client_cert = prepare_certs(
server_cert=certs['server_cert'],
client_key=certs['client_key'],
client_ca=certs['client_cert'])
creds = grpc.ssl_channel_credentials(root_certificates=server_ca_cert,
private_key=client_key,
certificate_chain=client_cert)
stub, request = prepare_stub_and_request(address=endpoint_address,
model_name=model_name,
creds=creds,
opts=target_name)
else:
stub, request = prepare_stub_and_request(address=endpoint_address,
model_name=model_name,
creds=None,
opts=target_name)
return stub, request
def test_no_certificates():
url = endpoint_info.info
trusted_cert, _, _ = prepare_certs(CERT_SERVER)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert)
stub, request = prepare_stub_and_request(url, MODEL_NAME, creds=creds)
numpy_input = numpy.zeros((1, 224, 224, 3), numpy.dtype('<f'))
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(numpy_input, shape=[1, 224, 224, 3]))
with pytest.raises(grpc.RpcError) as context:
stub.Predict(request, RPC_TIMEOUT)
assert context.value.details() == 'Received http2 header with status: 400'
def test_version_not_served():
stub, request = prepare_stub_and_request(endpoint_info.url,
MODEL_NAME,
model_version=1,
creds=endpoint_info.credentials)
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(image, shape=image.shape))
with pytest.raises(grpc.RpcError) as context:
stub.Predict(request, RPC_TIMEOUT)
logs = get_logs_of_pod(TENANT_NAME, endpoint_info.pod_name)
logging.info(filter_serving_logs(logs))
assert "Servable not found" in context.value.details()
def perform_inference(rpc_timeout=RPC_TIMEOUT, image=image):
stub, request = prepare_stub_and_request(endpoint_info.url,
MODEL_NAME,
creds=endpoint_info.credentials)
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(image, shape=image.shape))
prediction_response = "Failed"
try:
prediction_response = stub.Predict(request, rpc_timeout)
except Exception as e: # noqa
logging.info("Prediction failed: " + str(e))
print(str(e))
pass
logs = get_logs_of_pod(TENANT_NAME, endpoint_info.pod_name)
logging.info(filter_serving_logs(logs))
return prediction_response
def test_no_certificates():
url = test_create_endpoint.endpoint_info
trusted_cert, _, _ = prepare_certs(CERT_SERVER)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert)
stub, request = prepare_stub_and_request(url, MODEL_NAME, creds)
numpy_input = numpy.zeros((1, 224, 224, 3), numpy.dtype('<f'))
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(numpy_input, shape=[1, 224, 224, 3]))
with pytest.raises(grpc.RpcError) as context:
stub.Predict(request, 10.0)
logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name)
logging.info(filter_serving_logs(logs))
assert context.value.details() == 'Received http2 header with status: 400'
def test_prediction_batch_with_certificates():
url = test_create_endpoint.endpoint_info
trusted_cert, trusted_key, trusted_ca = prepare_certs(
CERT_SERVER, CERT_CLIENT_KEY, CERT_CLIENT)
creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert,
private_key=trusted_key,
certificate_chain=trusted_ca)
stub, request = prepare_stub_and_request(url, MODEL_NAME, creds)
request.inputs[model_input].CopyFrom(
tf.contrib.util.make_tensor_proto(images, shape=images.shape))
prediction_response = "Failed"
try:
prediction_response = stub.Predict(request, 30.0)
except: # noqa
logging.info("Prediction failed")
logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name)
logging.info(filter_serving_logs(logs))
response = numpy.array(prediction_response.outputs[model_output].float_val)
offset = 1001
max_outputs = []
for i in range(0, len(response), offset):
one_output = response[i:i + offset]
max_output = numpy.argmax(one_output) - 1
max_outputs.append(max_output)
for i in range(len(max_outputs)):
label = classes.imagenet_classes[max_outputs[i]]
test_label = classes.imagenet_classes[labels[i]]
assert max_outputs[i] == labels[i]
assert label == test_label