def testModelWithStringBasedInputUsingSessionRun(self):
input_list = ["1, 3", "2, -4", "0, 0"]
model_path = os.path.join(
FLAGS.test_tmpdir, "testModelWithStringBasedInputUsingSessionRun")
export_model_with_single_string_input(model_path)
model = create_model_using_session(model_path)
# TODO(rhaertel): this is cheating because it's not possible to send
# serialized protos over JSON without first base64 encoding them.
# Leaving as is because this will be addressed as we implement the new
# input/output formats.
_, predictions = model.predict(input_list, stats=mlprediction.Stats())
# y1 = 0.5 * x1 + 2.0
# y2 = y1 * x2
# y3 = y1 * y2
self.assertEqual(list(predictions), [
{
"y1": 2.5,
"y2": 7.5,
"y3": 18.75
},
{
"y1": 3.0,
"y2": -12.0,
"y3": -36.0
},
{
"y1": 2.0,
"y2": 0.0,
"y3": 0.0
},
])
def testModelWithStringBasedInput(self, mock_model_server):
input_list = ["1, 3", "2, -4", "0, 0"]
expected_predict_response = make_response({
"y": [7.5, -12.0, 0.0],
"y1": [3.0, -8.0, 0.0]
})
mock_model_server.Predict.return_value = expected_predict_response
model_path = os.path.join(FLAGS.test_srcdir,
EXAMPLES_MULTIPLE_INPUTS_MODEL)
model = create_model_from_server(mock_model_server, model_path)
# TODO(rhaertel): this is cheating because it's not possible to send
# serialized protos over JSON without first base64 encoding them.
# Leaving as is because this will be addressed as we implement the new
# input/output formats.
_, predictions = model.predict(input_list, stats=mlprediction.Stats())
# y = 0.5 * x1 + 2.0
# y1 = x * x1
self.assertEqual(list(predictions), [
{
"y": 7.5,
"y1": 3.0
},
{
"y": -12.0,
"y1": -8 - 0
},
{
"y": 0.0,
"y1": 0.0
},
])
def predict(self,
inputs,
stats=None,
signature_name=(
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY),
**unused_kwargs):
"""Predicts over the input_list."""
stats = stats or mlprediction.Stats()
stats[mlprediction.ENGINE] = MODEL_SERVER_ENGINE_NAME
stats[mlprediction.FRAMEWORK] = TENSORFLOW_FRAMEWORK_NAME
with stats.time(MODEL_SERVER_CREATE_REQUEST_TIME):
try:
request = create_predict_request(
inputs, self._signature_map[signature_name].inputs,
signature_name)
except Exception as e: # pylint: disable=broad-except
raise mlprediction.PredictionError(
mlprediction.PredictionError.INVALID_INPUTS,
"Error processing input: " + str(e))
# TODO(b/33849399): Measure the actual session.run() time.
with stats.time(mlprediction.SESSION_RUN_TIME):
try:
response = self._stub.Predict(request, PREDICT_DEADLINE_SECS)
except Exception as e: # pylint: disable=broad-except
raise mlprediction.PredictionError(
mlprediction.PredictionError.FAILED_TO_RUN_MODEL,
"Error during model execution: " + e.message)
with stats.time(MODEL_SERVER_PARSE_RESPONSE_TIME):
return {
name: tensor_util.MakeNdarray(val)
for name, val in response.outputs.items()
}
def testModelWithExecutionError(self, mock_model_server):
input_list = ["1, 3", "2, -4", "0, 0"]
mock_model_server.Predict.side_effect = Exception("prediction failed")
model_path = os.path.join(FLAGS.test_srcdir,
EXAMPLES_MULTIPLE_INPUTS_MODEL)
model = create_model_from_server(mock_model_server, model_path)
with self.assertRaises(mlprediction.PredictionError) as error:
model.predict(input_list, stats=mlprediction.Stats())
self.assertIn("Error during model execution: prediction failed",
error.exception.error_detail)
def testCreateTFModelFromModelServerClient(self, mock_model_server):
env_updates = {"prediction_engine": "MODEL_SERVER"}
flag_values = {"tensorflow_session_parallelism": 3}
pseudo_flags = type("Flags", (object, ), flag_values)
# Create model's SignatureDef and expected response.
expected_response = get_model_metadata_pb2.GetModelMetadataResponse()
expected_response.model_spec.name = "default"
in_bytes = meta_graph_pb2.TensorInfo(name="x",
dtype=tf.string.as_datatype_enum)
out_bytes = meta_graph_pb2.TensorInfo(name="y",
dtype=tf.string.as_datatype_enum)
inputs = {"in_bytes": in_bytes}
outputs = {"out_bytes": out_bytes}
signatures_def = meta_graph_pb2.SignatureDef(inputs=inputs,
outputs=outputs)
signatures_def_map = get_model_metadata_pb2.SignatureDefMap()
signatures_def_map.signature_def["serving_default"].CopyFrom(
signatures_def)
expected_response.metadata["signature_def"].Pack(signatures_def_map)
mock_model_server.GetModelMetadata.return_value = expected_response
with mock.patch.dict("os.environ", env_updates):
with mock.patch.object(tf_prediction_server_lib,
"_start_model_server"):
tf_prediction_server_lib._start_model_server.return_value = (
None, mock_model_server)
model = tf_prediction_server_lib.create_tf_model(
"/dummy/model/path", pseudo_flags)
# model is a TensorflowModel instance with a ModelServer client.
expected_predict_response = make_response({"out_bytes": ["to encode"]})
mock_model_server.Predict.return_value = expected_predict_response
dummy_instances = []
_, predictions = model.predict(dummy_instances,
stats=mlprediction.Stats())
self.assertEqual(list(predictions), [{
"out_bytes": {
u"b64": base64.b64encode("to encode")
}
}])
def testTensorBasedInput(self, mock_model_server):
input_list = [
{
"x": 1,
"x1": 3
},
{
"x": 2,
"x1": -4
},
{
"x1": 0,
"x": 0
},
]
expected_predict_response = make_response({
"y": [7.5, -12, 0],
"y1": [3.0, -8.0, 0.0]
})
mock_model_server.Predict.return_value = expected_predict_response
model_path = os.path.join(FLAGS.test_srcdir,
DISCRETE_MULTI_INPUT_MODEL)
model = create_model_from_server(mock_model_server, model_path)
_, predictions = model.predict(input_list, stats=mlprediction.Stats())
# y = (0.5 * x + 2) * x1
# y1 = x * x1
self.assertEqual(list(predictions), [
{
"y": 7.5,
"y1": 3.0
},
{
"y": -12.0,
"y1": -8 - 0
},
{
"y": 0.0,
"y1": 0.0
},
])
def testUnexpectedTensorNameInput(self):
# The expected tensor name is "examples", or we can omit it.
# Other tensor names should not be given.
input_list = [{
"examples": "1, 3"
}, {
"examples": "2, -4"
}, {
"b64": "0, 0"
}]
model_path = os.path.join(FLAGS.test_tmpdir,
"testUnexpectedTensorNameInput")
export_model_with_single_string_input(model_path)
model = create_model_using_session(model_path)
expected_msg = (r".*Expected tensor name: examples, "
r"got tensor name: \['b64'\].*")
with self.assertRaisesRegexp(mlprediction.PredictionError,
expected_msg):
_ = model.predict(input_list, stats=mlprediction.Stats())
def testCreateTFModelFromSessionRunClient(self):
env_updates = {"prediction_engine": "TF_SESSION_RUN"}
flag_values = {"tensorflow_session_parallelism": 3}
pseudo_flags = type("Flags", (object, ), flag_values)
model_path = os.path.join(FLAGS.test_tmpdir,
"testCreateTFModelSessionRunClient")
export_model_with_single_string_input(model_path)
session, signature_map = mlprediction.load_model(
model_path, tags=[tf.saved_model.tag_constants.SERVING])
with mock.patch.dict("os.environ", env_updates):
with mock.patch.object(tf_prediction_server_lib,
"_get_session_and_signature_map"):
tf_prediction_server_lib._get_session_and_signature_map.return_value = (
session, signature_map)
model = tf_prediction_server_lib.create_tf_model(
model_path, pseudo_flags)
input_list = ["1, 3", "2, -4", "0, 0"]
# model is a TensorflowModel instance with a SessionClient.
_, predictions = model.predict(input_list, stats=mlprediction.Stats())
predictions = list(predictions)
self.assertEqual(list(predictions), [{
"y1": 2.5,
"y2": 7.5,
"y3": 18.75
}, {
"y1": 3.0,
"y2": -12.0,
"y3": -36.0
}, {
"y1": 2.0,
"y2": 0.0,
"y3": 0.0
}])
def testTensorBasedInputWithSessionRun(self):
input_list = [
{
"x": 1,
"x1": 3
},
{
"x": 2,
"x1": -4
},
{
"x": 0,
"x1": 0
},
]
model_path = os.path.join(FLAGS.test_srcdir,
DISCRETE_MULTI_INPUT_MODEL)
model = create_model_using_session(model_path)
_, predictions = model.predict(input_list, stats=mlprediction.Stats())
# y = (0.5 * x + 2) * x1
# y1 = x * x1
self.assertEqual(list(predictions), [
{
"y": 7.5,
"y1": 3.0
},
{
"y": -12.0,
"y1": -8 - 0
},
{
"y": 0.0,
"y1": 0.0
},
])
def post(self):
post_start_time = timeit.default_timer()
stats = mlprediction.Stats()
with stats.time(TOTAL_TIME):
request_count = self._increase_request_count()
logging.debug("Started POST.")
_load_model_if_not_loaded(self.app.config)
model = self.app.config[MODEL_KEY]
if not model:
self._write_error(self.app.config[MODEL_LOADING_ERROR_KEY])
return
self._instrumented_post(stats)
# Dumps stats as headers for collecting metrics in the frontend.
# Unit: microseconds.
self.response.headers[START_TIME] = str(post_start_time * 1000000)
self.response.headers[REQUEST_COUNT_HEADER] = str(request_count)
self._decrease_request_count()
for name, value in stats.iteritems():
self.response.headers[name] = str(value)
def testCanOmitTensorNameWithSingleStringInput(self):
# Test that for single string input, we can give or omit the tensor name.
input_list1 = ["1, 3", "2, -4", "0, 0"]
input_list2 = [{
"examples": "1, 3"
}, {
"examples": "2, -4"
}, {
"examples": "0, 0"
}]
input_list3 = [{"examples": "1, 3"}, {"examples": "2, -4"}, "0, 0"]
model_path = os.path.join(
FLAGS.test_tmpdir, "testCanOmitTensorNameWithSingleStringInput")
export_model_with_single_string_input(model_path)
model = create_model_using_session(model_path)
dummy_stats = mlprediction.Stats()
_, predictions1 = model.predict(input_list1, stats=dummy_stats)
_, predictions2 = model.predict(input_list2, stats=dummy_stats)
_, predictions3 = model.predict(input_list3, stats=dummy_stats)
predictions1 = list(predictions1)
self.assertEquals(predictions1, list(predictions2))
self.assertEquals(predictions1, list(predictions3))