def test_ovms_serving_status(ovms_endpoint): ovms_endpoint_response, namespace = ovms_endpoint url = json.loads(ovms_endpoint_response.text)['data']['url'] start_time = time.time() tick = start_time running = False while tick - start_time < 100: tick = time.time() try: all_pods = get_all_pods_in_namespace(namespace) pod_name = all_pods.items[0].metadata.name logging.info("Pod name :", pod_name) logs = get_logs_of_pod("ovms", pod_name) logging.info(logs) if "Server listens on port 9000 and will be serving models" in logs: running = True break except Exception as e: logging.info(e) time.sleep(10) assert running is True trusted_cert, trusted_key, trusted_ca = prepare_certs( CERT_SERVER, CERT_CLIENT_KEY, CERT_CLIENT) creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert, private_key=trusted_key, certificate_chain=trusted_ca) stub, _ = prepare_stub_and_request(url, "", creds=creds) request = get_model_metadata_request("ovms_resnet") response = stub.GetModelMetadata(request, 10) assert "ovms_resnet" == response.model_spec.name
def test_prediction_with_certificates(): url = test_create_endpoint.endpoint_info trusted_cert, trusted_key, trusted_ca = prepare_certs( CERT_SERVER, CERT_CLIENT_KEY, CERT_CLIENT) creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert, private_key=trusted_key, certificate_chain=trusted_ca) stub, request = prepare_stub_and_request(url, MODEL_NAME, creds) request.inputs[model_input].CopyFrom( tf.contrib.util.make_tensor_proto(image, shape=image.shape)) prediction_response = "Failed" try: prediction_response = stub.Predict(request, 10.0) except: # noqa logging.info("Prediction failed") pass logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name) logging.info(filter_serving_logs(logs)) assert not prediction_response == "Failed" response = numpy.array(prediction_response.outputs[model_output].float_val) max_output = numpy.argmax(response) - 1 num_label = classes.imagenet_classes[max_output] test_label = classes.imagenet_classes[first_label] assert max_output == first_label assert num_label == test_label
def test_no_certificates(): url = endpoint_info.info trusted_cert, _, _ = prepare_certs(CERT_SERVER) creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert) stub, request = prepare_stub_and_request(url, MODEL_NAME, creds=creds) numpy_input = numpy.zeros((1, 224, 224, 3), numpy.dtype('<f')) request.inputs[model_input].CopyFrom( tf.contrib.util.make_tensor_proto(numpy_input, shape=[1, 224, 224, 3])) with pytest.raises(grpc.RpcError) as context: stub.Predict(request, RPC_TIMEOUT) assert context.value.details() == 'Received http2 header with status: 400'
def test_no_certificates(): url = test_create_endpoint.endpoint_info trusted_cert, _, _ = prepare_certs(CERT_SERVER) creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert) stub, request = prepare_stub_and_request(url, MODEL_NAME, creds) numpy_input = numpy.zeros((1, 224, 224, 3), numpy.dtype('<f')) request.inputs[model_input].CopyFrom( tf.contrib.util.make_tensor_proto(numpy_input, shape=[1, 224, 224, 3])) with pytest.raises(grpc.RpcError) as context: stub.Predict(request, 10.0) logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name) logging.info(filter_serving_logs(logs)) assert context.value.details() == 'Received http2 header with status: 400'
def get_stub_and_request(endpoint_address, model_name, certs, ssl, target_name): if ssl: server_ca_cert, client_key, client_cert = prepare_certs( server_cert=certs['server_cert'], client_key=certs['client_key'], client_ca=certs['client_cert']) creds = grpc.ssl_channel_credentials(root_certificates=server_ca_cert, private_key=client_key, certificate_chain=client_cert) stub, request = prepare_stub_and_request(address=endpoint_address, model_name=model_name, creds=creds, opts=target_name) else: stub, request = prepare_stub_and_request(address=endpoint_address, model_name=model_name, creds=None, opts=target_name) return stub, request
def test_prediction_with_certificates(): time.sleep(30) endpoint_info.url = endpoint_info.info trusted_cert, trusted_key, trusted_ca = prepare_certs( CERT_SERVER, CERT_CLIENT_KEY, CERT_CLIENT) endpoint_info.credentials = grpc.ssl_channel_credentials( root_certificates=trusted_cert, private_key=trusted_key, certificate_chain=trusted_ca) # resnet_v1 test prediction_response = perform_inference() assert not prediction_response == "Failed" response = numpy.array(prediction_response.outputs[model_output].float_val) max_output = numpy.argmax(response) - 1 num_label = classes.imagenet_classes[max_output] test_label = classes.imagenet_classes[first_label] assert max_output == first_label assert num_label == test_label assert response.size == 1000
def test_prediction_batch_with_certificates(): url = test_create_endpoint.endpoint_info trusted_cert, trusted_key, trusted_ca = prepare_certs( CERT_SERVER, CERT_CLIENT_KEY, CERT_CLIENT) creds = grpc.ssl_channel_credentials(root_certificates=trusted_cert, private_key=trusted_key, certificate_chain=trusted_ca) stub, request = prepare_stub_and_request(url, MODEL_NAME, creds) request.inputs[model_input].CopyFrom( tf.contrib.util.make_tensor_proto(images, shape=images.shape)) prediction_response = "Failed" try: prediction_response = stub.Predict(request, 30.0) except: # noqa logging.info("Prediction failed") logs = get_logs_of_pod(TENANT_NAME, test_create_endpoint.pod_name) logging.info(filter_serving_logs(logs)) response = numpy.array(prediction_response.outputs[model_output].float_val) offset = 1001 max_outputs = [] for i in range(0, len(response), offset): one_output = response[i:i + offset] max_output = numpy.argmax(one_output) - 1 max_outputs.append(max_output) for i in range(len(max_outputs)): label = classes.imagenet_classes[max_outputs[i]] test_label = classes.imagenet_classes[labels[i]] assert max_outputs[i] == labels[i] assert label == test_label