def test_correct_signature(self): runtime = TensorflowRuntime("models/tf_summator") runtime.start(port="9090") try: time.sleep(1) channel = grpc.insecure_channel('localhost:9090') client = hs.PredictionServiceStub(channel=channel) a = hs.TensorProto() a.ParseFromString( tf.contrib.util.make_tensor_proto( 3, dtype=tf.int8, shape=[]).SerializeToString()) b = hs.TensorProto() b.ParseFromString( tf.contrib.util.make_tensor_proto( 2, dtype=tf.int8, shape=[]).SerializeToString()) request = hs.PredictRequest( model_spec=hs.ModelSpec(signature_name="add"), inputs={ "a": a, "b": b }) result = client.Predict(request) expected = hs.PredictResponse( outputs={ "sum": hs.TensorProto(dtype=hs.DT_INT8, tensor_shape=hs.TensorShapeProto(), int_val=[5]) }) self.assertEqual(result, expected) finally: runtime.stop()
def test_lstm(self): runtime = TensorflowRuntime("models/lstm") runtime.start(port="9090") results = [] try: time.sleep(1) channel = grpc.insecure_channel('localhost:9090') client = hs.PredictionServiceStub(channel=channel) shape = hs.TensorShapeProto(dim=[ hs.TensorShapeProto.Dim(size=-1), hs.TensorShapeProto.Dim(size=1), hs.TensorShapeProto.Dim(size=24) ]) data_tensor = hs.TensorProto( dtype=hs.DT_FLOAT, tensor_shape=shape, float_val=[x for x in np.random.random(24).tolist()]) for x in range(1, 5): request = hs.PredictRequest( model_spec=hs.ModelSpec(signature_name="infer"), inputs={"data": data_tensor}) result = client.Predict(request) results.append(result) finally: runtime.stop() print(results)
def test_incorrect_signature(self): runtime = TensorflowRuntime("models/tf_summator") runtime.start(port="9090") try: time.sleep(1) channel = grpc.insecure_channel('localhost:9090') client = hs.PredictionServiceStub(channel=channel) a = hs.TensorProto() a.ParseFromString( tf.contrib.util.make_tensor_proto( 3, dtype=tf.int8).SerializeToString()) b = hs.TensorProto() b.ParseFromString( tf.contrib.util.make_tensor_proto( 2, dtype=tf.int8).SerializeToString()) request = hs.PredictRequest( model_spec=hs.ModelSpec(signature_name="missing_sig"), inputs={ "a": a, "b": b }) client.Predict(request) except grpc.RpcError as ex: self.assertEqual(ex.code(), grpc.StatusCode.INVALID_ARGUMENT) assert ("missing_sig signature is not present in the model" in ex.details()) except Exception as ex: self.fail("Unexpected exception: {}".format(ex)) finally: runtime.stop(0)
def simulate_production_traffic(path="./data", application_name="mnist_app", host=None, request_delay=1, request_amount=10000, imgs_file="imgs.npz", labels_file="labels.npz", shuffle=False): # conn = psycopg2.connect(f"postgresql://{POSTGRES_USER}:{POSTGRES_PASS}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}") # cur = conn.cursor() # cur.execute(''' # CREATE TABLE IF NOT EXISTS # requests (hex_uid varchar(256), ground_truth integer); # ''') # conn.commit() if not host: host = f"dev.k8s.hydrosphere.io" creds = grpc.ssl_channel_credentials() channel = grpc.secure_channel(host, creds) # channel = grpc.insecure_channel(host) stub = hs.PredictionServiceStub(channel) # an application, that will be invoked model_spec = hs.ModelSpec(name=application_name) # basic shape for images tensor_shape = hs.TensorShapeProto(dim=[ hs.TensorShapeProto.Dim(size=1), hs.TensorShapeProto.Dim(size=28), hs.TensorShapeProto.Dim(size=28), hs.TensorShapeProto.Dim(size=1), ]) images, labels = generate_data(path, imgs_file, labels_file, shuffle=shuffle) for index, (image, label) in tqdm(enumerate(zip(images, labels)), total=request_amount): if index == request_amount: break if not image.flags['C_CONTIGUOUS']: image = np.ascontiguousarray(image) # form a request and get a prediction tensor = hs.TensorProto(dtype=hs.DT_FLOAT, tensor_shape=tensor_shape, float_val=image.flatten().tolist()) request = hs.PredictRequest(model_spec=model_spec, inputs={"imgs": tensor}) stub.Predict(request) # insert uid and ground_truth labels into database # cur.execute("INSERT INTO requests VALUES (%s, %s)", # (hashlib.sha1(image).hexdigest(), int(label))) # conn.commit() time.sleep(request_delay)
def simulate_production_traffic(host=None, request_delay=2, request_amount=10000, file="combined.npz", shuffle=False): conn = psycopg2.connect( f"postgresql://{USER}:{PASS}@{ADDRESS}:{PORT}/{DATABASE}") cur = conn.cursor() cur.execute(''' CREATE TABLE IF NOT EXISTS requests (timestamp bigint, uid integer, ground_truth integer); ''') if not host: namespace = os.environ["NAMESPACE"] host = f"hydro-serving-ui-{namespace}:9091" channel = grpc.insecure_channel(host) stub = hs.PredictionServiceStub(channel) # an application, that will be invoked model_spec = hs.ModelSpec(name="mnist-app") # basic shape for images tensor_shape = hs.TensorShapeProto(dim=[ hs.TensorShapeProto.Dim(size=1), hs.TensorShapeProto.Dim(size=28), hs.TensorShapeProto.Dim(size=28), hs.TensorShapeProto.Dim(size=1), ]) images, labels = generate_data('/data', file, shuffle=shuffle) for index, (image, label) in tqdm(enumerate(zip(images, labels)), total=request_amount): if index == request_amount: break # form a request tensor = hs.TensorProto(dtype=hs.DT_FLOAT, tensor_shape=tensor_shape, float_val=image.flatten().tolist()) request = hs.PredictRequest(model_spec=model_spec, inputs={"imgs": tensor}) # get prediction result = stub.Predict(request) # insert trace_id and ground_truth labels into database cur.execute("INSERT INTO requests VALUES (%s, %s, %s)", (result.trace_data.ts, result.trace_data.uid, int(label))) conn.commit() time.sleep(request_delay)
def test_runtime(self): channel = grpc.insecure_channel('localhost:6969') client = hs.PredictionServiceStub(channel=channel) with open("test_pic.jpg", "rb") as image_file: encoded_string = base64.b64encode(image_file.read()) image_tensor = hs.TensorProto( dtype=hs.DT_STRING, string_val=[encoded_string] ) request = hs.PredictRequest( model_spec=hs.ModelSpec(signature_name="detect"), inputs={ "image_b64": image_tensor, } ) result = client.Predict(request) print(result)
def tensorflow_case(self, tf_version): docker_client = docker.from_env() container = docker_client.containers.run( "hydrosphere/serving-runtime-tensorflow:{}-latest".format(tf_version), remove=True, detach=True, ports={'9090/tcp': 9090}, volumes={os.path.abspath('models/tf_summator'): {'bind': '/model', 'mode': 'ro'}} ) time.sleep(15) try: channel = grpc.insecure_channel('localhost:9090') client = hs.PredictionServiceStub(channel=channel) a = hs.TensorProto() a.ParseFromString(tf.contrib.util.make_tensor_proto(3, dtype=tf.int8).SerializeToString()) b = hs.TensorProto() b.ParseFromString(tf.contrib.util.make_tensor_proto(2, dtype=tf.int8).SerializeToString()) request = hs.PredictRequest( model_spec=hs.ModelSpec(signature_name="add"), inputs={ "a": a, "b": b } ) result = client.Predict(request) expected = hs.PredictResponse( outputs={ "sum": hs.TensorProto( dtype=hs.DT_INT8, tensor_shape=hs.TensorShapeProto(), int_val=[5] ) } ) self.assertEqual(result, expected) finally: print("Container logs:") print(container.logs().decode("utf-8")) container.stop() time.sleep(15)
import grpc import hydro_serving_grpc as hs # pip install hydro-serving-grpc # connect to your ML Lamba instance channel = grpc.insecure_channel("localhost") stub = hs.PredictionServiceStub(channel) # 1. define a model, that you'll use model_spec = hs.ModelSpec(name="linear_regression", signature_name="infer") # 2. define tensor_shape for Tensor instance tensor_shape = hs.TensorShapeProto( dim=[ hs.TensorShapeProto.Dim(size=-1), hs.TensorShapeProto.Dim(size=2) ] ) # 3. define tensor with needed data tensor = hs.TensorProto(dtype=hs.DT_DOUBLE, tensor_shape=tensor_shape, double_val=[1,1,1,1]) # 4. create PredictRequest instance request = hs.PredictRequest(model_spec=model_spec, inputs={"x": tensor}) # call Predict method result = stub.Predict(request)
modelVersion_id=2, model_version=3, signature_name="predict", request_id=trace_id, latency=0.014, ) # 2. Create a PredictRequest message. PredictRequest is used to define the data # passed to the model for inference. predict_request_proto = hs.PredictRequest( model_spec=hs.ModelSpec( name="external-model-example", signature_name="predict", ), inputs={ "in": hs.TensorProto( dtype=hs.DT_DOUBLE, double_val=[random.random()], tensor_shape=hs.TensorShapeProto() ), }, ) # 3. Create a PredictResponse message. PredictResponse is used to define the # outputs of the model inference. predict_response_proto = hs.PredictResponse( outputs={ "out": hs.TensorProto( dtype=hs.DT_DOUBLE, double_val=[random.random()], tensor_shape=hs.TensorShapeProto()
image = Image.open(path_to_the_image); resized_image = image.resize((600, 600)); image_array = tf.keras.preprocessing.image.img_to_array(resized_image); image_shaped = image_array.reshape((1,) + image_array.shape); image_tensor_shape = hs.TensorShapeProto\ ( dim = \ [ hs.TensorShapeProto.Dim(size = dim)\ for dim in image_shaped.shape ] ); image_tensor_proto = hs.TensorProto\ ( dtype = hs.DT_DOUBLE, double_val = image_shaped.flatten(), tensor_shape = image_tensor_shape ); request = hs.PredictRequest(model_spec = model_spec, inputs = {"shaped_image": image_tensor_proto}) result = stub.Predict(request); print(result);