def Predict(self,request,context):
features = get_data_from_proto(request)
datadef = request.data
data_type = request.WhichOneof("data_oneof")
predictions = predict(self.user_model,features,datadef.names)
# Construct meta data
meta = prediction_pb2.Meta()
metaJson = {}
tags = get_custom_tags(self.user_model)
if tags:
metaJson["tags"] = tags
metrics = get_custom_metrics(self.user_model)
if metrics:
metaJson["metrics"] = metrics
json_format.ParseDict(metaJson,meta)
if isinstance(predictions, np.ndarray) or data_type == "data":
predictions = np.array(predictions)
if len(predictions.shape)>1:
class_names = get_class_names(self.user_model, predictions.shape[1])
else:
class_names = []
if data_type == "data":
default_data_type = request.data.WhichOneof("data_oneof")
else:
default_data_type = "tensor"
data = array_to_grpc_datadef(predictions, class_names, default_data_type)
return prediction_pb2.SeldonMessage(data=data,meta=meta)
else:
return prediction_pb2.SeldonMessage(binData=predictions,meta=meta)
def TransformOutput(self, request, context):
features = get_data_from_proto(request)
datadef = request.data
data_type = request.WhichOneof("data_oneof")
# Construct meta data
meta = prediction_pb2.Meta()
metaJson = {}
tags = get_custom_tags(self.user_model)
if tags:
metaJson["tags"] = tags
metrics = get_custom_metrics(self.user_model)
if metrics:
metaJson["metrics"] = metrics
json_format.ParseDict(metaJson, meta)
transformed = transform_output(self.user_model, features,
datadef.names)
if isinstance(transformed, np.ndarray) or data_type == "data":
transformed = np.array(transformed)
class_names = get_class_names(self.user_model, datadef.names)
if data_type == "data":
default_data_type = request.data.WhichOneof("data_oneof")
else:
default_data_type = "tensor"
data = array_to_grpc_datadef(transformed, class_names,
default_data_type)
return prediction_pb2.SeldonMessage(data=data, meta=meta)
else:
return prediction_pb2.SeldonMessage(binData=transformed, meta=meta)
def test_transform_output_proto_bin_data():
user_object = UserObject()
app = SeldonTransformerGRPC(user_object)
binData = b"\0\1"
request = prediction_pb2.SeldonMessage(binData = binData)
resp = app.TransformOutput(request,None)
assert resp.binData == binData
def get_prediction(self):
datadef = prediction_pb2.DefaultData(
names=["a", "b"],
tensor=prediction_pb2.Tensor(
shape=[3, 2], values=[1.0, 1.0, 2.0, 3.0, 4.0, 5.0]))
request = prediction_pb2.SeldonMessage(data=datadef)
start_time = time.time()
try:
if self.oauth_enabled:
metadata = [('oauth_token', self.access_token)]
response = self.stub.Predict(request=request,
metadata=metadata)
else:
response = self.stub.Predict(request=request)
except Exception as e:
total_time = int((time.time() - start_time) * 1000)
print(e)
events.request_failure.fire(request_type="grpc",
name=HOST,
response_time=total_time,
exception=e)
else:
total_time = int((time.time() - start_time) * 1000)
events.request_success.fire(request_type="grpc",
name=HOST,
response_time=total_time,
response_length=0)
def Predict(self, request, context):
datadef = request.data
features = grpc_datadef_to_array(datadef)
predictions = np.array(
predict(self.user_model, features, datadef.names))
if len(predictions.shape) > 1:
class_names = get_class_names(self.user_model,
predictions.shape[1])
else:
class_names = []
data = array_to_grpc_datadef(predictions, class_names,
request.data.WhichOneof("data_oneof"))
# Construct meta data
meta = prediction_pb2.Meta()
metaJson = {}
tags = get_custom_tags(self.user_model)
if tags:
metaJson["tags"] = tags
metrics = get_custom_metrics(self.user_model)
if metrics:
metaJson["metrics"] = metrics
json_format.ParseDict(metaJson, meta)
return prediction_pb2.SeldonMessage(data=data, meta=meta)
def get_prediction(image,
server_host='127.0.0.1',
server_port=8080,
deployment_name="server"):
"""
Retrieve a prediction from a TensorFlow model server
:param image: a MNIST image represented as a 1x784 array
:param server_host: the address of the Seldon server
:param server_port: the port used by the server
:param deployment_name: the name of the deployment
:return 0: the integer predicted in the MNIST image
"""
try:
# build request
chosen = 0
data = image[chosen].reshape(784)
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=data.shape, values=data))
# retrieve results
request = prediction_pb2.SeldonMessage(data=datadef)
logging.info("connecting to:%s:%i", server_host, server_port)
channel = grpc.insecure_channel(server_host + ":" + str(server_port))
stub = prediction_pb2_grpc.SeldonStub(channel)
metadata = [('seldon', deployment_name)]
response = stub.Predict(request=request, metadata=metadata)
except IOError as e:
# server connection failed
logging.error("Could Not Connect to Server: %s", str(e))
return response.data.tensor.values
def test_proto_bin_data():
user_object = UserObject()
app = SeldonModelGRPC(user_object)
bdata = b"123"
bdata_base64 = base64.b64encode(bdata)
request = prediction_pb2.SeldonMessage(binData=bdata_base64)
resp = app.Predict(request, None)
assert resp.binData == bdata_base64
def test_transform_output_proto_bin_data_nparray():
user_object = UserObject(ret_nparray=True)
app = SeldonTransformerGRPC(user_object)
binData = b"\0\1"
request = prediction_pb2.SeldonMessage(binData = binData)
resp = app.TransformOutput(request,None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
print(j)
assert j["data"]["tensor"]["values"] == list(user_object.nparray.flatten())
def SendFeedback(self,feedback,context):
datadef_request = feedback.request.data
features = grpc_datadef_to_array(datadef_request)
truth = grpc_datadef_to_array(feedback.truth)
reward = feedback.reward
send_feedback(self.user_model,features,datadef_request.names,truth,reward)
return prediction_pb2.SeldonMessage()
def Route(self,request,context):
datadef = request.data
features = grpc_datadef_to_array(datadef)
routing = np.array([[route(self.user_model,features,datadef.names)]])
#TODO: check that predictions is 2 dimensional
class_names = []
data = array_to_grpc_datadef(routing, class_names, request.data.WhichOneof("data_oneof"))
return prediction_pb2.SeldonMessage(data=data)
def grpc_request(deploymentName, data):
datadef = prediction_pb2.DefaultData(names=["a", "b"],
tensor=prediction_pb2.Tensor(
shape=[1, 784], values=data))
request = prediction_pb2.SeldonMessage(data=datadef)
channel = grpc.insecure_channel(AMBASSADOR_API_IP)
stub = prediction_pb2_grpc.SeldonStub(channel)
metadata = [('seldon', deploymentName)]
response = stub.Predict(request=request, metadata=metadata)
return response
def Predict(self,request,context):
datadef = request.data
features = grpc_datadef_to_array(datadef)
predictions = np.array(predict(self.user_model,features,datadef.names))
#TODO: check that predictions is 2 dimensional
class_names = get_class_names(self.user_model, predictions.shape[1])
data = array_to_grpc_datadef(predictions, class_names, request.data.WhichOneof("data_oneof"))
return prediction_pb2.SeldonMessage(data=data)
def TransformOutput(self,request,context):
datadef = request.data
features = grpc_datadef_to_array(datadef)
transformed = np.array(transform_output(self.user_model,features,datadef.names))
#TODO: check that predictions is 2 dimensional
class_names = get_class_names(self.user_model, datadef.names)
data = array_to_grpc_datadef(transformed, class_names, request.data.WhichOneof("data_oneof"))
return prediction_pb2.SeldonMessage(data=data)
def gen_GRPC_request(batch, features, tensor=True):
if tensor:
datadef = prediction_pb2.DefaultData(
names=features,
tensor=prediction_pb2.Tensor(shape=batch.shape,
values=batch.ravel().tolist()))
else:
datadef = prediction_pb2.DefaultData(
names=features, ndarray=array_to_list_value(batch))
request = prediction_pb2.SeldonMessage(data=datadef)
return request
def SendFeedback(self,feedback,context):
datadef_request = feedback.request.data
features = grpc_datadef_to_array(datadef_request)
truth = grpc_datadef_to_array(feedback.truth)
reward = feedback.reward
routing = feedback.response.meta.routing.get(PRED_UNIT_ID)
send_feedback(self.user_model,features,datadef_request.names,routing,reward,truth)
return prediction_pb2.SeldonMessage()
def gen_GRPC_request(batch, features, tensor=True):
if tensor:
datadef = prediction_pb2.DefaultData(
names=features,
tensor=prediction_pb2.Tensor(shape=batch.shape,
values=batch.ravel().tolist()))
else:
v = '{"names":' + json.dumps(features) + ',"ndarray":' + json.dumps(
batch.tolist()) + '}'
datadef = json_format.Parse(v, prediction_pb2.DefaultData())
request = prediction_pb2.SeldonMessage(data=datadef)
return request
def grpc_request_ambassador(deploymentName,
endpoint="localhost:8004",
data_size=5,
rows=1):
shape, arr = create_random_data(data_size, rows)
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=shape, values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
channel = grpc.insecure_channel(endpoint)
stub = prediction_pb2_grpc.SeldonStub(channel)
metadata = [('seldon', deploymentName)]
response = stub.Predict(request=request, metadata=metadata)
print(response)
def Predict(self, request, context):
datadef = request.data
features = grpc_datadef_to_array(datadef)
predictions = np.array(
predict(self.user_model, features, datadef.names))
if len(predictions.shape) > 1:
class_names = get_class_names(self.user_model,
predictions.shape[1])
else:
class_names = []
data = array_to_grpc_datadef(predictions, class_names,
request.data.WhichOneof("data_oneof"))
return prediction_pb2.SeldonMessage(data=data)
def test_transformer_input_bin_data():
user_object = UserObject()
app = get_rest_microservice(user_object,debug=True)
client = app.test_client()
bdata = b"123"
bdata_base64 = base64.b64encode(bdata).decode('utf-8')
rv = client.get('/transform-input?json={"binData":"'+bdata_base64+'"}')
j = json.loads(rv.data)
sm = prediction_pb2.SeldonMessage()
# Check we can parse response
assert sm == json_format.Parse(rv.data, sm, ignore_unknown_fields=False)
print(j)
assert rv.status_code == 200
assert "binData" in j
assert j["meta"]["tags"] == {"mytag":1}
assert j["meta"]["metrics"] == user_object.metrics()
def test_proto_tftensor_ok():
user_object = UserObject()
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(tftensor=tf.make_tensor_proto(arr))
request = prediction_pb2.SeldonMessage(data=datadef)
resp = app.Predict(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
print(j)
assert j["meta"]["tags"] == {"mytag": 1}
# add default type
j["meta"]["metrics"][0]["type"] = "COUNTER"
assert j["meta"]["metrics"] == user_object.metrics()
arr2 = tf.make_ndarray(resp.data.tftensor)
assert np.array_equal(arr, arr2)
def grpc_request_api_gateway(oauth_key,
oauth_secret,
rest_endpoint="localhost:8002",
grpc_endpoint="localhost:8003",
data_size=5,
rows=1):
token = get_token(oauth_key, oauth_secret, rest_endpoint)
shape, arr = create_random_data(data_size, rows)
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=shape, values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
channel = grpc.insecure_channel(grpc_endpoint)
stub = prediction_pb2_grpc.SeldonStub(channel)
metadata = [('oauth_token', token)]
response = stub.Predict(request=request, metadata=metadata)
print(response)
def test_proto_ok():
user_object = UserObject()
app = SeldonModelGRPC(user_object)
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=(2, 1), values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
resp = app.Predict(request, None)
jStr = json_format.MessageToJson(resp)
j = json.loads(jStr)
print(j)
assert j["meta"]["tags"] == {"mytag": 1}
# add default type
j["meta"]["metrics"][0]["type"] = "COUNTER"
assert j["meta"]["metrics"] == user_object.metrics()
assert j["data"]["tensor"]["shape"] == [2, 1]
assert j["data"]["tensor"]["values"] == [1, 2]
def getPrediction(self):
batch_xs, batch_ys = self.mnist.train.next_batch(1)
data = batch_xs[0].reshape((1, 784))
data = np.around(data, decimals=2)
features = ["X" + str(i + 1) for i in range(0, self.data_size)]
#request = {"data":{"names":features,"ndarray":data.tolist()}}
datadef = prediction_pb2.DefaultData(names=features,
tensor=prediction_pb2.Tensor(
shape=[1, 784],
values=data.flatten()))
request = prediction_pb2.SeldonMessage(data=datadef)
start_time = time.time()
try:
response = self.stub.Predict(request=request)
if self.send_feedback == 1:
response = MessageToJson(response)
response = json.loads(response)
route = response.get("meta").get("routing").get("eg-router")
proba = response["data"]["tensor"]["values"]
predicted = proba.index(max(proba))
correct = np.argmax(batch_ys[0])
j = json.loads(r.content)
if predicted == correct:
self.sendFeedback(request, j, 1.0)
print("Correct!")
else:
self.sendFeedback(request, j, 0.0)
print("Incorrect!")
except Exception as e:
total_time = int((time.time() - start_time) * 1000)
print(e)
events.request_failure.fire(request_type="grpc",
name=HOST,
response_time=total_time,
exception=e)
else:
total_time = int((time.time() - start_time) * 1000)
events.request_success.fire(request_type="grpc",
name=HOST,
response_time=total_time,
response_length=0)
def test_model_tftensor_ok():
user_object = UserObject()
app = get_rest_microservice(user_object, debug=True)
client = app.test_client()
arr = np.array([1, 2])
datadef = prediction_pb2.DefaultData(tftensor=tf.make_tensor_proto(arr))
request = prediction_pb2.SeldonMessage(data=datadef)
jStr = json_format.MessageToJson(request)
rv = client.get('/predict?json=' + jStr)
j = json.loads(rv.data)
print(j)
assert rv.status_code == 200
assert j["meta"]["tags"] == {"mytag": 1}
assert j["meta"]["metrics"] == user_object.metrics()
assert 'tftensor' in j['data']
tfp = TensorProto()
json_format.ParseDict(j['data'].get("tftensor"),
tfp,
ignore_unknown_fields=False)
arr2 = tf.make_ndarray(tfp)
assert np.array_equal(arr, arr2)
def Route(self,request,context):
datadef = request.data
features = grpc_datadef_to_array(datadef)
routing = np.array([[route(self.user_model,features,datadef.names)]])
#TODO: check that predictions is 2 dimensional
class_names = []
data = array_to_grpc_datadef(routing, class_names, request.data.WhichOneof("data_oneof"))
# Construct meta data
meta = prediction_pb2.Meta()
metaJson = {}
tags = get_custom_tags(self.user_model)
if tags:
metaJson["tags"] = tags
metrics = get_custom_metrics(self.user_model)
if metrics:
metaJson["metrics"] = metrics
json_format.ParseDict(metaJson,meta)
return prediction_pb2.SeldonMessage(data=data,meta=meta)
def grpc_request_ambassador(deploymentName,
namespace,
endpoint="localhost:8004",
data_size=5,
rows=1,
data=None):
if data is None:
shape, arr = create_random_data(data_size, rows)
else:
shape = data.shape
arr = data.flatten()
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=shape, values=arr))
request = prediction_pb2.SeldonMessage(data=datadef)
channel = grpc.insecure_channel(endpoint)
stub = prediction_pb2_grpc.SeldonStub(channel)
if namespace is None:
metadata = [('seldon', deploymentName)]
else:
metadata = [('seldon', deploymentName), ('namespace', namespace)]
response = stub.Predict(request=request, metadata=metadata)
return response
def get_prediction(image,
server_host='127.0.0.1',
server_port=8080,
deployment_name="server",
timeout=10.0):
"""
Retrieve a prediction from a TensorFlow model server
:param image: a MNIST image represented as a 1x784 array
:param server_host: the address of the Seldon server
:param server_port: the port used by the server
:param deployment_name: the name of the deployment
:param timeout: the amount of time to wait for a prediction to complete
:return 0: the integer predicted in the MNIST image
:return 1: the confidence scores for all classes
:return 2: the version number of the model handling the request
"""
try:
# build request
chosen = 0
data = image[chosen].reshape(784)
datadef = prediction_pb2.DefaultData(
tensor=prediction_pb2.Tensor(shape=data.shape, values=data))
# retrieve results
request = prediction_pb2.SeldonMessage(data=datadef)
print("connecting to:%s:%i" % (server_host, server_port))
channel = grpc.insecure_channel(server_host + ":" + str(server_port))
stub = prediction_pb2_grpc.SeldonStub(channel)
metadata = [('seldon', deployment_name)]
response = stub.Predict(request=request, metadata=metadata)
except Exception as e:
# server connection failed
print("Could Not Connect to Server: " + str(e))
return response.data.tensor.values
