def Predict(self,request,context): features = get_data_from_proto(request) datadef = request.data data_type = request.WhichOneof("data_oneof") predictions = predict(self.user_model,features,datadef.names) # Construct meta data meta = prediction_pb2.Meta() metaJson = {} tags = get_custom_tags(self.user_model) if tags: metaJson["tags"] = tags metrics = get_custom_metrics(self.user_model) if metrics: metaJson["metrics"] = metrics json_format.ParseDict(metaJson,meta) if isinstance(predictions, np.ndarray) or data_type == "data": predictions = np.array(predictions) if len(predictions.shape)>1: class_names = get_class_names(self.user_model, predictions.shape[1]) else: class_names = [] if data_type == "data": default_data_type = request.data.WhichOneof("data_oneof") else: default_data_type = "tensor" data = array_to_grpc_datadef(predictions, class_names, default_data_type) return prediction_pb2.SeldonMessage(data=data,meta=meta) else: return prediction_pb2.SeldonMessage(binData=predictions,meta=meta)
def TransformOutput(self, request, context): features = get_data_from_proto(request) datadef = request.data data_type = request.WhichOneof("data_oneof") # Construct meta data meta = prediction_pb2.Meta() metaJson = {} tags = get_custom_tags(self.user_model) if tags: metaJson["tags"] = tags metrics = get_custom_metrics(self.user_model) if metrics: metaJson["metrics"] = metrics json_format.ParseDict(metaJson, meta) transformed = transform_output(self.user_model, features, datadef.names) if isinstance(transformed, np.ndarray) or data_type == "data": transformed = np.array(transformed) class_names = get_class_names(self.user_model, datadef.names) if data_type == "data": default_data_type = request.data.WhichOneof("data_oneof") else: default_data_type = "tensor" data = array_to_grpc_datadef(transformed, class_names, default_data_type) return prediction_pb2.SeldonMessage(data=data, meta=meta) else: return prediction_pb2.SeldonMessage(binData=transformed, meta=meta)
def test_transform_output_proto_bin_data(): user_object = UserObject() app = SeldonTransformerGRPC(user_object) binData = b"\0\1" request = prediction_pb2.SeldonMessage(binData = binData) resp = app.TransformOutput(request,None) assert resp.binData == binData
def get_prediction(self): datadef = prediction_pb2.DefaultData( names=["a", "b"], tensor=prediction_pb2.Tensor( shape=[3, 2], values=[1.0, 1.0, 2.0, 3.0, 4.0, 5.0])) request = prediction_pb2.SeldonMessage(data=datadef) start_time = time.time() try: if self.oauth_enabled: metadata = [('oauth_token', self.access_token)] response = self.stub.Predict(request=request, metadata=metadata) else: response = self.stub.Predict(request=request) except Exception as e: total_time = int((time.time() - start_time) * 1000) print(e) events.request_failure.fire(request_type="grpc", name=HOST, response_time=total_time, exception=e) else: total_time = int((time.time() - start_time) * 1000) events.request_success.fire(request_type="grpc", name=HOST, response_time=total_time, response_length=0)
def Predict(self, request, context): datadef = request.data features = grpc_datadef_to_array(datadef) predictions = np.array( predict(self.user_model, features, datadef.names)) if len(predictions.shape) > 1: class_names = get_class_names(self.user_model, predictions.shape[1]) else: class_names = [] data = array_to_grpc_datadef(predictions, class_names, request.data.WhichOneof("data_oneof")) # Construct meta data meta = prediction_pb2.Meta() metaJson = {} tags = get_custom_tags(self.user_model) if tags: metaJson["tags"] = tags metrics = get_custom_metrics(self.user_model) if metrics: metaJson["metrics"] = metrics json_format.ParseDict(metaJson, meta) return prediction_pb2.SeldonMessage(data=data, meta=meta)
def get_prediction(image, server_host='127.0.0.1', server_port=8080, deployment_name="server"): """ Retrieve a prediction from a TensorFlow model server :param image: a MNIST image represented as a 1x784 array :param server_host: the address of the Seldon server :param server_port: the port used by the server :param deployment_name: the name of the deployment :return 0: the integer predicted in the MNIST image """ try: # build request chosen = 0 data = image[chosen].reshape(784) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=data.shape, values=data)) # retrieve results request = prediction_pb2.SeldonMessage(data=datadef) logging.info("connecting to:%s:%i", server_host, server_port) channel = grpc.insecure_channel(server_host + ":" + str(server_port)) stub = prediction_pb2_grpc.SeldonStub(channel) metadata = [('seldon', deployment_name)] response = stub.Predict(request=request, metadata=metadata) except IOError as e: # server connection failed logging.error("Could Not Connect to Server: %s", str(e)) return response.data.tensor.values
def test_proto_bin_data(): user_object = UserObject() app = SeldonModelGRPC(user_object) bdata = b"123" bdata_base64 = base64.b64encode(bdata) request = prediction_pb2.SeldonMessage(binData=bdata_base64) resp = app.Predict(request, None) assert resp.binData == bdata_base64
def test_transform_output_proto_bin_data_nparray(): user_object = UserObject(ret_nparray=True) app = SeldonTransformerGRPC(user_object) binData = b"\0\1" request = prediction_pb2.SeldonMessage(binData = binData) resp = app.TransformOutput(request,None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) print(j) assert j["data"]["tensor"]["values"] == list(user_object.nparray.flatten())
def SendFeedback(self,feedback,context): datadef_request = feedback.request.data features = grpc_datadef_to_array(datadef_request) truth = grpc_datadef_to_array(feedback.truth) reward = feedback.reward send_feedback(self.user_model,features,datadef_request.names,truth,reward) return prediction_pb2.SeldonMessage()
def Route(self,request,context): datadef = request.data features = grpc_datadef_to_array(datadef) routing = np.array([[route(self.user_model,features,datadef.names)]]) #TODO: check that predictions is 2 dimensional class_names = [] data = array_to_grpc_datadef(routing, class_names, request.data.WhichOneof("data_oneof")) return prediction_pb2.SeldonMessage(data=data)
def grpc_request(deploymentName, data): datadef = prediction_pb2.DefaultData(names=["a", "b"], tensor=prediction_pb2.Tensor( shape=[1, 784], values=data)) request = prediction_pb2.SeldonMessage(data=datadef) channel = grpc.insecure_channel(AMBASSADOR_API_IP) stub = prediction_pb2_grpc.SeldonStub(channel) metadata = [('seldon', deploymentName)] response = stub.Predict(request=request, metadata=metadata) return response
def Predict(self,request,context): datadef = request.data features = grpc_datadef_to_array(datadef) predictions = np.array(predict(self.user_model,features,datadef.names)) #TODO: check that predictions is 2 dimensional class_names = get_class_names(self.user_model, predictions.shape[1]) data = array_to_grpc_datadef(predictions, class_names, request.data.WhichOneof("data_oneof")) return prediction_pb2.SeldonMessage(data=data)
def TransformOutput(self,request,context): datadef = request.data features = grpc_datadef_to_array(datadef) transformed = np.array(transform_output(self.user_model,features,datadef.names)) #TODO: check that predictions is 2 dimensional class_names = get_class_names(self.user_model, datadef.names) data = array_to_grpc_datadef(transformed, class_names, request.data.WhichOneof("data_oneof")) return prediction_pb2.SeldonMessage(data=data)
def gen_GRPC_request(batch, features, tensor=True): if tensor: datadef = prediction_pb2.DefaultData( names=features, tensor=prediction_pb2.Tensor(shape=batch.shape, values=batch.ravel().tolist())) else: datadef = prediction_pb2.DefaultData( names=features, ndarray=array_to_list_value(batch)) request = prediction_pb2.SeldonMessage(data=datadef) return request
def SendFeedback(self,feedback,context): datadef_request = feedback.request.data features = grpc_datadef_to_array(datadef_request) truth = grpc_datadef_to_array(feedback.truth) reward = feedback.reward routing = feedback.response.meta.routing.get(PRED_UNIT_ID) send_feedback(self.user_model,features,datadef_request.names,routing,reward,truth) return prediction_pb2.SeldonMessage()
def gen_GRPC_request(batch, features, tensor=True): if tensor: datadef = prediction_pb2.DefaultData( names=features, tensor=prediction_pb2.Tensor(shape=batch.shape, values=batch.ravel().tolist())) else: v = '{"names":' + json.dumps(features) + ',"ndarray":' + json.dumps( batch.tolist()) + '}' datadef = json_format.Parse(v, prediction_pb2.DefaultData()) request = prediction_pb2.SeldonMessage(data=datadef) return request
def grpc_request_ambassador(deploymentName, endpoint="localhost:8004", data_size=5, rows=1): shape, arr = create_random_data(data_size, rows) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=shape, values=arr)) request = prediction_pb2.SeldonMessage(data=datadef) channel = grpc.insecure_channel(endpoint) stub = prediction_pb2_grpc.SeldonStub(channel) metadata = [('seldon', deploymentName)] response = stub.Predict(request=request, metadata=metadata) print(response)
def Predict(self, request, context): datadef = request.data features = grpc_datadef_to_array(datadef) predictions = np.array( predict(self.user_model, features, datadef.names)) if len(predictions.shape) > 1: class_names = get_class_names(self.user_model, predictions.shape[1]) else: class_names = [] data = array_to_grpc_datadef(predictions, class_names, request.data.WhichOneof("data_oneof")) return prediction_pb2.SeldonMessage(data=data)
def test_transformer_input_bin_data(): user_object = UserObject() app = get_rest_microservice(user_object,debug=True) client = app.test_client() bdata = b"123" bdata_base64 = base64.b64encode(bdata).decode('utf-8') rv = client.get('/transform-input?json={"binData":"'+bdata_base64+'"}') j = json.loads(rv.data) sm = prediction_pb2.SeldonMessage() # Check we can parse response assert sm == json_format.Parse(rv.data, sm, ignore_unknown_fields=False) print(j) assert rv.status_code == 200 assert "binData" in j assert j["meta"]["tags"] == {"mytag":1} assert j["meta"]["metrics"] == user_object.metrics()
def test_proto_tftensor_ok(): user_object = UserObject() app = SeldonModelGRPC(user_object) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData(tftensor=tf.make_tensor_proto(arr)) request = prediction_pb2.SeldonMessage(data=datadef) resp = app.Predict(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) print(j) assert j["meta"]["tags"] == {"mytag": 1} # add default type j["meta"]["metrics"][0]["type"] = "COUNTER" assert j["meta"]["metrics"] == user_object.metrics() arr2 = tf.make_ndarray(resp.data.tftensor) assert np.array_equal(arr, arr2)
def grpc_request_api_gateway(oauth_key, oauth_secret, rest_endpoint="localhost:8002", grpc_endpoint="localhost:8003", data_size=5, rows=1): token = get_token(oauth_key, oauth_secret, rest_endpoint) shape, arr = create_random_data(data_size, rows) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=shape, values=arr)) request = prediction_pb2.SeldonMessage(data=datadef) channel = grpc.insecure_channel(grpc_endpoint) stub = prediction_pb2_grpc.SeldonStub(channel) metadata = [('oauth_token', token)] response = stub.Predict(request=request, metadata=metadata) print(response)
def test_proto_ok(): user_object = UserObject() app = SeldonModelGRPC(user_object) arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr)) request = prediction_pb2.SeldonMessage(data=datadef) resp = app.Predict(request, None) jStr = json_format.MessageToJson(resp) j = json.loads(jStr) print(j) assert j["meta"]["tags"] == {"mytag": 1} # add default type j["meta"]["metrics"][0]["type"] = "COUNTER" assert j["meta"]["metrics"] == user_object.metrics() assert j["data"]["tensor"]["shape"] == [2, 1] assert j["data"]["tensor"]["values"] == [1, 2]
def getPrediction(self): batch_xs, batch_ys = self.mnist.train.next_batch(1) data = batch_xs[0].reshape((1, 784)) data = np.around(data, decimals=2) features = ["X" + str(i + 1) for i in range(0, self.data_size)] #request = {"data":{"names":features,"ndarray":data.tolist()}} datadef = prediction_pb2.DefaultData(names=features, tensor=prediction_pb2.Tensor( shape=[1, 784], values=data.flatten())) request = prediction_pb2.SeldonMessage(data=datadef) start_time = time.time() try: response = self.stub.Predict(request=request) if self.send_feedback == 1: response = MessageToJson(response) response = json.loads(response) route = response.get("meta").get("routing").get("eg-router") proba = response["data"]["tensor"]["values"] predicted = proba.index(max(proba)) correct = np.argmax(batch_ys[0]) j = json.loads(r.content) if predicted == correct: self.sendFeedback(request, j, 1.0) print("Correct!") else: self.sendFeedback(request, j, 0.0) print("Incorrect!") except Exception as e: total_time = int((time.time() - start_time) * 1000) print(e) events.request_failure.fire(request_type="grpc", name=HOST, response_time=total_time, exception=e) else: total_time = int((time.time() - start_time) * 1000) events.request_success.fire(request_type="grpc", name=HOST, response_time=total_time, response_length=0)
def test_model_tftensor_ok(): user_object = UserObject() app = get_rest_microservice(user_object, debug=True) client = app.test_client() arr = np.array([1, 2]) datadef = prediction_pb2.DefaultData(tftensor=tf.make_tensor_proto(arr)) request = prediction_pb2.SeldonMessage(data=datadef) jStr = json_format.MessageToJson(request) rv = client.get('/predict?json=' + jStr) j = json.loads(rv.data) print(j) assert rv.status_code == 200 assert j["meta"]["tags"] == {"mytag": 1} assert j["meta"]["metrics"] == user_object.metrics() assert 'tftensor' in j['data'] tfp = TensorProto() json_format.ParseDict(j['data'].get("tftensor"), tfp, ignore_unknown_fields=False) arr2 = tf.make_ndarray(tfp) assert np.array_equal(arr, arr2)
def Route(self,request,context): datadef = request.data features = grpc_datadef_to_array(datadef) routing = np.array([[route(self.user_model,features,datadef.names)]]) #TODO: check that predictions is 2 dimensional class_names = [] data = array_to_grpc_datadef(routing, class_names, request.data.WhichOneof("data_oneof")) # Construct meta data meta = prediction_pb2.Meta() metaJson = {} tags = get_custom_tags(self.user_model) if tags: metaJson["tags"] = tags metrics = get_custom_metrics(self.user_model) if metrics: metaJson["metrics"] = metrics json_format.ParseDict(metaJson,meta) return prediction_pb2.SeldonMessage(data=data,meta=meta)
def grpc_request_ambassador(deploymentName, namespace, endpoint="localhost:8004", data_size=5, rows=1, data=None): if data is None: shape, arr = create_random_data(data_size, rows) else: shape = data.shape arr = data.flatten() datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=shape, values=arr)) request = prediction_pb2.SeldonMessage(data=datadef) channel = grpc.insecure_channel(endpoint) stub = prediction_pb2_grpc.SeldonStub(channel) if namespace is None: metadata = [('seldon', deploymentName)] else: metadata = [('seldon', deploymentName), ('namespace', namespace)] response = stub.Predict(request=request, metadata=metadata) return response
def get_prediction(image, server_host='127.0.0.1', server_port=8080, deployment_name="server", timeout=10.0): """ Retrieve a prediction from a TensorFlow model server :param image: a MNIST image represented as a 1x784 array :param server_host: the address of the Seldon server :param server_port: the port used by the server :param deployment_name: the name of the deployment :param timeout: the amount of time to wait for a prediction to complete :return 0: the integer predicted in the MNIST image :return 1: the confidence scores for all classes :return 2: the version number of the model handling the request """ try: # build request chosen = 0 data = image[chosen].reshape(784) datadef = prediction_pb2.DefaultData( tensor=prediction_pb2.Tensor(shape=data.shape, values=data)) # retrieve results request = prediction_pb2.SeldonMessage(data=datadef) print("connecting to:%s:%i" % (server_host, server_port)) channel = grpc.insecure_channel(server_host + ":" + str(server_port)) stub = prediction_pb2_grpc.SeldonStub(channel) metadata = [('seldon', deployment_name)] response = stub.Predict(request=request, metadata=metadata) except Exception as e: # server connection failed print("Could Not Connect to Server: " + str(e)) return response.data.tensor.values