def test_correct_signature(self):
runtime = TensorflowRuntime("models/tf_summator")
runtime.start(port="9090")
try:
time.sleep(1)
channel = grpc.insecure_channel('localhost:9090')
client = hs.PredictionServiceStub(channel=channel)
a = hs.TensorProto()
a.ParseFromString(
tf.contrib.util.make_tensor_proto(
3, dtype=tf.int8, shape=[]).SerializeToString())
b = hs.TensorProto()
b.ParseFromString(
tf.contrib.util.make_tensor_proto(
2, dtype=tf.int8, shape=[]).SerializeToString())
request = hs.PredictRequest(
model_spec=hs.ModelSpec(signature_name="add"),
inputs={
"a": a,
"b": b
})
result = client.Predict(request)
expected = hs.PredictResponse(
outputs={
"sum":
hs.TensorProto(dtype=hs.DT_INT8,
tensor_shape=hs.TensorShapeProto(),
int_val=[5])
})
self.assertEqual(result, expected)
finally:
runtime.stop()
def test_incorrect_signature(self):
runtime = TensorflowRuntime("models/tf_summator")
runtime.start(port="9090")
try:
time.sleep(1)
channel = grpc.insecure_channel('localhost:9090')
client = hs.PredictionServiceStub(channel=channel)
a = hs.TensorProto()
a.ParseFromString(
tf.contrib.util.make_tensor_proto(
3, dtype=tf.int8).SerializeToString())
b = hs.TensorProto()
b.ParseFromString(
tf.contrib.util.make_tensor_proto(
2, dtype=tf.int8).SerializeToString())
request = hs.PredictRequest(
model_spec=hs.ModelSpec(signature_name="missing_sig"),
inputs={
"a": a,
"b": b
})
client.Predict(request)
except grpc.RpcError as ex:
self.assertEqual(ex.code(), grpc.StatusCode.INVALID_ARGUMENT)
assert ("missing_sig signature is not present in the model"
in ex.details())
except Exception as ex:
self.fail("Unexpected exception: {}".format(ex))
finally:
runtime.stop(0)
def test_lstm(self):
runtime = TensorflowRuntime("models/lstm")
runtime.start(port="9090")
results = []
try:
time.sleep(1)
channel = grpc.insecure_channel('localhost:9090')
client = hs.PredictionServiceStub(channel=channel)
shape = hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=-1),
hs.TensorShapeProto.Dim(size=1),
hs.TensorShapeProto.Dim(size=24)
])
data_tensor = hs.TensorProto(
dtype=hs.DT_FLOAT,
tensor_shape=shape,
float_val=[x for x in np.random.random(24).tolist()])
for x in range(1, 5):
request = hs.PredictRequest(
model_spec=hs.ModelSpec(signature_name="infer"),
inputs={"data": data_tensor})
result = client.Predict(request)
results.append(result)
finally:
runtime.stop()
print(results)
def simulate_production_traffic(path="./data",
application_name="mnist_app",
host=None,
request_delay=1,
request_amount=10000,
imgs_file="imgs.npz",
labels_file="labels.npz",
shuffle=False):
# conn = psycopg2.connect(f"postgresql://{POSTGRES_USER}:{POSTGRES_PASS}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}")
# cur = conn.cursor()
# cur.execute('''
# CREATE TABLE IF NOT EXISTS
# requests (hex_uid varchar(256), ground_truth integer);
# ''')
# conn.commit()
if not host:
host = f"dev.k8s.hydrosphere.io"
creds = grpc.ssl_channel_credentials()
channel = grpc.secure_channel(host, creds)
# channel = grpc.insecure_channel(host)
stub = hs.PredictionServiceStub(channel)
# an application, that will be invoked
model_spec = hs.ModelSpec(name=application_name)
# basic shape for images
tensor_shape = hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=1),
hs.TensorShapeProto.Dim(size=28),
hs.TensorShapeProto.Dim(size=28),
hs.TensorShapeProto.Dim(size=1),
])
images, labels = generate_data(path,
imgs_file,
labels_file,
shuffle=shuffle)
for index, (image, label) in tqdm(enumerate(zip(images, labels)),
total=request_amount):
if index == request_amount: break
if not image.flags['C_CONTIGUOUS']:
image = np.ascontiguousarray(image)
# form a request and get a prediction
tensor = hs.TensorProto(dtype=hs.DT_FLOAT,
tensor_shape=tensor_shape,
float_val=image.flatten().tolist())
request = hs.PredictRequest(model_spec=model_spec,
inputs={"imgs": tensor})
stub.Predict(request)
# insert uid and ground_truth labels into database
# cur.execute("INSERT INTO requests VALUES (%s, %s)",
# (hashlib.sha1(image).hexdigest(), int(label)))
# conn.commit()
time.sleep(request_delay)
def simulate_production_traffic(host=None,
request_delay=2,
request_amount=10000,
file="combined.npz",
shuffle=False):
conn = psycopg2.connect(
f"postgresql://{USER}:{PASS}@{ADDRESS}:{PORT}/{DATABASE}")
cur = conn.cursor()
cur.execute('''
CREATE TABLE IF NOT EXISTS
requests (timestamp bigint, uid integer, ground_truth integer);
''')
if not host:
namespace = os.environ["NAMESPACE"]
host = f"hydro-serving-ui-{namespace}:9091"
channel = grpc.insecure_channel(host)
stub = hs.PredictionServiceStub(channel)
# an application, that will be invoked
model_spec = hs.ModelSpec(name="mnist-app")
# basic shape for images
tensor_shape = hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=1),
hs.TensorShapeProto.Dim(size=28),
hs.TensorShapeProto.Dim(size=28),
hs.TensorShapeProto.Dim(size=1),
])
images, labels = generate_data('/data', file, shuffle=shuffle)
for index, (image, label) in tqdm(enumerate(zip(images, labels)),
total=request_amount):
if index == request_amount: break
# form a request
tensor = hs.TensorProto(dtype=hs.DT_FLOAT,
tensor_shape=tensor_shape,
float_val=image.flatten().tolist())
request = hs.PredictRequest(model_spec=model_spec,
inputs={"imgs": tensor})
# get prediction
result = stub.Predict(request)
# insert trace_id and ground_truth labels into database
cur.execute("INSERT INTO requests VALUES (%s, %s, %s)",
(result.trace_data.ts, result.trace_data.uid, int(label)))
conn.commit()
time.sleep(request_delay)
def test_runtime(self):
channel = grpc.insecure_channel('localhost:6969')
client = hs.PredictionServiceStub(channel=channel)
with open("test_pic.jpg", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
image_tensor = hs.TensorProto(
dtype=hs.DT_STRING,
string_val=[encoded_string]
)
request = hs.PredictRequest(
model_spec=hs.ModelSpec(signature_name="detect"),
inputs={
"image_b64": image_tensor,
}
)
result = client.Predict(request)
print(result)
def tensorflow_case(self, tf_version):
docker_client = docker.from_env()
container = docker_client.containers.run(
"hydrosphere/serving-runtime-tensorflow:{}-latest".format(tf_version),
remove=True, detach=True,
ports={'9090/tcp': 9090},
volumes={os.path.abspath('models/tf_summator'): {'bind': '/model', 'mode': 'ro'}}
)
time.sleep(15)
try:
channel = grpc.insecure_channel('localhost:9090')
client = hs.PredictionServiceStub(channel=channel)
a = hs.TensorProto()
a.ParseFromString(tf.contrib.util.make_tensor_proto(3, dtype=tf.int8).SerializeToString())
b = hs.TensorProto()
b.ParseFromString(tf.contrib.util.make_tensor_proto(2, dtype=tf.int8).SerializeToString())
request = hs.PredictRequest(
model_spec=hs.ModelSpec(signature_name="add"),
inputs={
"a": a,
"b": b
}
)
result = client.Predict(request)
expected = hs.PredictResponse(
outputs={
"sum": hs.TensorProto(
dtype=hs.DT_INT8,
tensor_shape=hs.TensorShapeProto(),
int_val=[5]
)
}
)
self.assertEqual(result, expected)
finally:
print("Container logs:")
print(container.logs().decode("utf-8"))
container.stop()
time.sleep(15)
import grpc
import hydro_serving_grpc as hs # pip install hydro-serving-grpc
# connect to your ML Lamba instance
channel = grpc.insecure_channel("localhost")
stub = hs.PredictionServiceStub(channel)
# 1. define a model, that you'll use
model_spec = hs.ModelSpec(name="linear_regression", signature_name="infer")
# 2. define tensor_shape for Tensor instance
tensor_shape = hs.TensorShapeProto(
dim=[
hs.TensorShapeProto.Dim(size=-1),
hs.TensorShapeProto.Dim(size=2)
]
)
# 3. define tensor with needed data
tensor = hs.TensorProto(dtype=hs.DT_DOUBLE, tensor_shape=tensor_shape, double_val=[1,1,1,1])
# 4. create PredictRequest instance
request = hs.PredictRequest(model_spec=model_spec, inputs={"x": tensor})
# call Predict method
result = stub.Predict(request)
import hydro_serving_grpc as hs
import numpy as np
import grpc
# connect to your ML Lamba instance
channel = grpc.insecure_channel("localhost:8080")
stub = hs.PredictionServiceStub(channel)
# 1. define a model, that you'll use
model_spec = hs.ModelSpec(name="seq2seq-mono",
signature_name="serving_default")
# 2. define tensor_shape for Tensor instance
tensor_shape = hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=-1),
hs.TensorShapeProto.Dim(size=1)])
# 3. define tensor with needed data
tensor = hs.TensorProto(dtype=hs.DT_STRING,
tensor_shape=tensor_shape,
string_val=np.array([b"hello , there , sugar"]))
# 4. create PredictRequest instance
request = hs.PredictRequest(model_spec=model_spec,
inputs={"input_data": tensor})
# call Predict method
result = stub.Predict(request)
print(result)
# given request.
trace_id = str(uuid.uuid4()) # uuid used as an example
execution_metadata_proto = hs.ExecutionMetadata(
model_name="external-model-example",
modelVersion_id=2,
model_version=3,
signature_name="predict",
request_id=trace_id,
latency=0.014,
)
# 2. Create a PredictRequest message. PredictRequest is used to define the data
# passed to the model for inference.
predict_request_proto = hs.PredictRequest(
model_spec=hs.ModelSpec(
name="external-model-example",
signature_name="predict",
),
inputs={
"in": hs.TensorProto(
dtype=hs.DT_DOUBLE,
double_val=[random.random()],
tensor_shape=hs.TensorShapeProto()
),
},
)
# 3. Create a PredictResponse message. PredictResponse is used to define the
# outputs of the model inference.
predict_response_proto = hs.PredictResponse(
outputs={
"out": hs.TensorProto(
config_file = open("config.json", "r");
configuration = json.loads(config_file.read());
config_file.close();
if (len(sys.argv) < 2):
path_to_the_image = "im0162.ppm";
#print("Please specify the path to the image in command line, e.g. :\n python client.py image.png");
else:
path_to_the_image = sys.argv[1];
creds = grpc.ssl_channel_credentials();
channel = grpc.secure_channel('dev.k8s.hydrosphere.io:443', creds);
stub = hs.PredictionServiceStub(channel);
model_spec = hs.ModelSpec(name="retina_screening");
image = Image.open(path_to_the_image);
resized_image = image.resize((600, 600));
image_array = tf.keras.preprocessing.image.img_to_array(resized_image);
image_shaped = image_array.reshape((1,) + image_array.shape);
image_tensor_shape = hs.TensorShapeProto\
(
dim = \
[
import grpc
import hydro_serving_grpc as hs # pip install hydro-serving-grpc
# connect to your ML Lamba instance
channel = grpc.insecure_channel("<host>")
stub = hs.PredictionServiceStub(channel)
# 1. define a model, that you'll use
model_spec = hs.ModelSpec(name="model")
# 2. define tensor_shape for Tensor instance
tensor_shape = hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=-1),
hs.TensorShapeProto.Dim(size=2)])
# 3. define tensor with needed data
tensor = hs.TensorProto(dtype=hs.DT_DOUBLE,
tensor_shape=tensor_shape,
double_val=[1, 1, 1, 1])
# 4. create PredictRequest instance
request = hs.PredictRequest(model_spec=model_spec, inputs={"x": tensor})
# call Predict method
result = stub.Predict(request)
