class OpDefUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase):
@parameterized.parameters([
("any", "Foo", "Foo"),
("any", 12, 12),
("any", {2: 3}, {2: 3}),
("string", "Foo", "Foo"),
("string", b"Foo", b"Foo"),
("int", 12, 12),
("int", 12.3, 12),
("float", 12, 12.0),
("float", 12.3, 12.3),
("bool", True, True),
("shape", tensor_shape.TensorShape([3]), tensor_shape.TensorShape([3])),
("shape", [3], tensor_shape.TensorShape([3])),
("type", dtypes.int32, dtypes.int32),
("type", np.int32, dtypes.int32),
("type", "int32", dtypes.int32),
("tensor", tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT),
tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT)),
("tensor", "dtype: DT_FLOAT",
tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT)),
("list(any)", [1, "foo", 7.3, dtypes.int32],
[1, "foo", 7.3, dtypes.int32]),
("list(any)", (1, "foo"), [1, "foo"]),
("list(string)", ["foo", "bar"], ["foo", "bar"]),
("list(string)", ("foo", "bar"), ["foo", "bar"]),
("list(string)", iter("abcd"), ["a", "b", "c", "d"]),
("list(int)", (1, 2.3), [1, 2]),
("list(float)", (1, 2.3), [1.0, 2.3]),
("list(bool)", [True, False], [True, False]),
])
def testConvert(self, attr_type, value, expected):
result = _op_def_util.ConvertPyObjectToAttributeType(value, attr_type)
# Check that we get the expected value(s).
self.assertEqual(expected, result)
# Check that we get the expected type(s).
self.assertEqual(type(expected), type(result))
if isinstance(result, list):
for expected_item, result_item in zip(expected, result):
self.assertEqual(type(expected_item), type(result_item))
@parameterized.parameters([
("string", 12),
("int", "foo"),
("float", "foo"),
("bool", 1),
("dtype", None),
("shape", 12.0),
("tensor", [1, 2, 3]),
("list(any)", 12),
("list(int)", [1, "two"]),
("list(string)", [1, "two"]),
("tensor", "string that is not a text-formatted TensorProto"),
])
def testConvertError(self, attr_type, value):
with self.assertRaisesRegex(TypeError, "Failed to convert value"):
_op_def_util.ConvertPyObjectToAttributeType(value, attr_type)
def testInconvertibleTensorProto(self):
self.assertFalse(debug_data.has_inf_or_nan(
self._dummy_datum,
debug_data.InconvertibleTensorProto(tensor_pb2.TensorProto(),
initialized=False)))
self.assertFalse(debug_data.has_inf_or_nan(
self._dummy_datum,
debug_data.InconvertibleTensorProto(tensor_pb2.TensorProto(),
initialized=True)))
def rnet_serving(image):
host = '192.168.1.253'
port = 9000
channel = implementations.insecure_channel(host, port)
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'mtcnn'
request.model_spec.signature_name = 'rnet_predict'
image_data_arr = generate_input_string(image)
image_data_arr = np.asarray(image_data_arr).squeeze()
num_image = image_data_arr.shape[0]
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=num_image)]
tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor_proto = tensor_pb2.TensorProto(
dtype=types_pb2.DT_STRING,
tensor_shape=tensor_shape_proto,
string_val=[image_data for image_data in image_data_arr])
request.inputs['images'].CopyFrom(tensor_proto)
result = stub.Predict(request, 10.0)
return [
MakeNdarray(result.outputs['result1']),
MakeNdarray(result.outputs['result2'])
]
def run_inference_for_single_image_grpc(self,image,project):
# Get handles to input and output tensors
request = predict_pb2.PredictRequest()
# Specify model name (must be the same as when the TensorFlow serving serving was started)
request.model_spec.name = project.model
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=1)]
tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor_proto = tensor_pb2.TensorProto(
dtype=types_pb2.DT_STRING,
tensor_shape=tensor_shape_proto,
string_val=[image.tobytes()])
# Initalize prediction
# Specify signature name (should be the same as specified when exporting model)
request.model_spec.signature_name = ''
request.inputs['inputs'].CopyFrom(
tensor_proto)
result = project.stub.Predict(request, 10.0)
# Run inference
output_dict = {}
# all outputs are float32 numpy arrays, so convert types as appropriate
output_dict['num_detections'] = np.squeeze(result.outputs['num_detections'].float_val).astype(np.int32)
output_dict['detection_classes'] = np.squeeze(result.outputs['detection_classes'].float_val).astype(np.int32)
output_dict['detection_boxes'] = np.reshape(result.outputs['detection_boxes'].float_val,[100,4])
output_dict['detection_scores'] = result.outputs['detection_scores'].float_val
return output_dict
def _original_node_def(node_def, lnf, variable_values):
if lnf.original.t != graph_types_pb2.TFSavedModel:
raise ValueError("original node not a TFSavedModel: {0}".format(
lnf.name))
# Use serialized node def and fix inputs
node_def.ParseFromString(lnf.original.serialized_node)
del (node_def.input[:])
node_def.input.extend(ExportTFSavedModel._get_tensor_names(lnf.inputs))
for cntrl in lnf.control_inputs:
node_def.input.append("^" + cntrl)
# Get values from variables
if lnf.original.op == ops_pb2.VARIABLE:
if (tf_saved_model_base_importer.ImportTFSavedModelBase.TENSOR_PB
not in lnf.original.attr):
raise ValueError(
"Could not find value associated with variable node: {}".
format(lnf))
tensor_pb = tensor_pb2.TensorProto()
tensor_pb.ParseFromString(
lnf.original.attr[tf_saved_model_base_importer.
ImportTFSavedModelBase.TENSOR_PB].v)
variable_values[node_def.name] = tf.make_ndarray(tensor_pb)
def make_tensor_proto(data, dtype, size) -> tensor_pb2.TensorProto:
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=size)]
tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor_proto = tensor_pb2.TensorProto(dtype=dtype,
tensor_shape=tensor_shape_proto,
string_val=data)
return tensor_proto
def main():
if len(sys.argv) != 2:
print("error: only 1 argument accepted")
sys.exit(1)
ch = grpc.insecure_channel(sys.argv[1])
stub = prediction_service_pb2_grpc.PredictionServiceStub(ch)
# same example inputs from https://www.tensorflow.org/tfx/serving/docker
inputs = [1.0, 2.0, 5.0]
input_tensor = tensor_pb2.TensorProto(
dtype=types_pb2.DT_FLOAT,
tensor_shape=tensor_shape_pb2.TensorShapeProto(
dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=len(inputs), )]),
float_val=inputs,
)
req = predict_pb2.PredictRequest()
req.model_spec.name = "half_plus_two"
req.model_spec.signature_name = "serving_default"
req.inputs["x"].CopyFrom(input_tensor)
res = stub.Predict(req)
print(text_format.MessageToString(res))
sys.exit(0)
def testZerosLikeVariant(self):
# TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant
# copying between CPU and GPU is supported AND we register a
# ZerosLike callback for GPU for Variant storing primitive types
# in variant_op_registry.cc.
with self.test_session(use_gpu=False):
variant_tensor = tensor_pb2.TensorProto(
dtype=dtypes_lib.variant.as_datatype_enum,
tensor_shape=tensor_shape.TensorShape([]).as_proto(),
variant_val=[
tensor_pb2.VariantTensorDataProto(
# Match registration in variant_op_registry.cc
type_name=b"int",
metadata=np.array(1, dtype=np.int32).tobytes())
])
const_variant = constant_op.constant(variant_tensor)
zeros_like = array_ops.zeros_like(const_variant)
zeros_like_op = logging_ops.Print(
zeros_like, [const_variant, zeros_like],
message=
"Variant storing an int, input and output of zeros_like:").op
# Smoke test -- ensure this executes without trouble.
# Right now, non-numpy-compatible objects cannot be returned from a
# session.run call; similarly, objects that can't be converted to
# native numpy types cannot be passed to ops.convert_to_tensor.
# TODO(ebrevdo): Add registration mechanism for
# ops.convert_to_tensor and for session.run output.
zeros_like_op.run()
def construct_tensor(shapes, value):
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=dim) for dim in shapes]
tensor_shape = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor = tensor_pb2.TensorProto(dtype=types_pb2.DT_INT32,
tensor_shape=tensor_shape,
int_val=value)
return tensor
def testVariant(self):
# TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant
# copying between CPU and GPU is supported.
with self.test_session(use_gpu=False):
variant_tensor = tensor_pb2.TensorProto(
dtype=dtypes_lib.variant.as_datatype_enum,
tensor_shape=tensor_shape.TensorShape([]).as_proto(),
variant_val=[
tensor_pb2.VariantTensorDataProto(
# Match registration in variant_op_registry.cc
type_name=b"int",
metadata=np.array(1, dtype=np.int32).tobytes())
])
const = constant_op.constant(variant_tensor)
const_value = const.op.get_attr("value")
# Ensure we stored the tensor proto properly.
self.assertProtoEquals(variant_tensor, const_value)
# Smoke test -- ensure this executes without trouble.
# Right now, non-numpy-compatible objects cannot be returned from a
# session.run call; similarly, objects that can't be converted to
# native numpy types cannot be passed to ops.convert_to_tensor.
# TODO(ebrevdo): Add registration mechanism for
# ops.convert_to_tensor and for session.run output.
logging_const_op = logging_ops.Print(
const, [const],
message="Variant storing an int, decoded const value:").op
logging_const_op.run()
def facenet_serving(images):
host = '172.20.15.85'
port = 8500
channel = implementations.insecure_channel(host, port)
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = '1'
request.model_spec.signature_name = 'calculate_embeddings'
image_data_arr = np.asarray(images)
#print('image_data_arr',image_data_arr)
#print('image_data',image_data_arr.dtype)
input_size = image_data_arr.shape[0]
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=input_size)]
print('dims',dims)
tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor_proto = tensor_pb2.TensorProto(
dtype=types_pb2.DT_STRING,
tensor_shape=tensor_shape_proto,
string_val=[image_data for image_data in image_data_arr])
print('tensor_proto', tensor_proto)
request.inputs['images'].CopyFrom(tensor_proto)
result = stub.Predict(request, 10.0)
print(MakeNdarray(result.outputs['embeddings']))
return MakeNdarray(result.outputs['embeddings'])
def onet_serving(image, channel):
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'mtcnn'
request.model_spec.signature_name = 'onet_predict'
image_data_arr = generate_input_string(image)
image_data_arr = np.asarray(image_data_arr).squeeze()
num_image = image_data_arr.shape[0]
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=num_image)]
tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor_proto = tensor_pb2.TensorProto(
dtype=types_pb2.DT_STRING,
tensor_shape=tensor_shape_proto,
string_val=[image_data for image_data in image_data_arr])
request.inputs['images'].CopyFrom(tensor_proto)
result = stub.Predict(request, 10.0)
return [
MakeNdarray(result.outputs['result1']),
MakeNdarray(result.outputs['result2']),
MakeNdarray(result.outputs['result3'])
]
def pull_embedding_vectors(self, request, _):
result = tensor_pb2.TensorProto()
if not request.ids:
return result
embedding_vectors = self._parameters.get_embedding_param(
request.name, request.ids)
serialize_ndarray(embedding_vectors, result)
return result
def __call__(self, stream, content_type):
try:
data = stream.read()
finally:
stream.close()
try:
return json_format.Parse(data, tensor_pb2.TensorProto())
except json_format.ParseError:
return json.loads(data.decode())
def make_tensor_proto_float(data, shape):
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=i) for i in shape]
tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor_proto = tensor_pb2.TensorProto(
dtype=types_pb2.DT_FLOAT,
tensor_shape=tensor_shape_proto,
float_val=np.array(data).ravel().tolist())
return tensor_proto
def testFormatResourceTypeTensor(self):
tensor_proto = tensor_pb2.TensorProto(
dtype=types_pb2.DataType.Value("DT_RESOURCE"),
tensor_shape=tensor_shape_pb2.TensorShapeProto(
dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]))
out = tensor_format.format_tensor(
debug_data.InconvertibleTensorProto(tensor_proto), "a")
self.assertEqual(["Tensor \"a\":", ""], out.lines[:2])
self.assertEqual(str(tensor_proto).split("\n"), out.lines[2:])
def make_tensor_proto_wrapped(values, dtype=None, shape=None, verify_shape=False, allow_broadcast=False):
try:
return make_tensor_proto_original(values, dtype, shape, verify_shape, allow_broadcast)
except ValueError:
if dtype is None:
dtype = tf.dtypes.as_dtype(values.dtype).as_datatype_enum
tensor_proto = tensor_pb2.TensorProto(
dtype=dtype,
tensor_shape=tensor_shape.as_shape(values.shape).as_proto())
tensor_proto.tensor_content = values.tobytes()
return tensor_proto
def make_tensor(v, arg_name):
"""Ensure v is a TensorProto."""
if isinstance(v, tensor_pb2.TensorProto):
return v
elif isinstance(v, six.string_types):
pb = tensor_pb2.TensorProto()
text_format.Merge(v, pb)
return pb
raise TypeError(
"Don't know how to convert %s to a TensorProto for argument '%s'." %
(repr(v), arg_name))
def create_constant_variant(value):
return constant_op.constant(
tensor_pb2.TensorProto(
dtype=dtypes.variant.as_datatype_enum,
tensor_shape=tensor_shape.TensorShape([]).as_proto(),
variant_val=[
tensor_pb2.VariantTensorDataProto(
# Match registration in variant_op_registry.cc
type_name=b"int",
metadata=np.array(value, dtype=np.int32).tobytes())
]))
def make_tensor_proto(data):
shape = data.shape
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=i) for i in shape]
proto_shape = tensor_shape_pb2.TensorShapeProto(dim=dims)
proto_dtype = dtypes_as_dtype(data.dtype)
tensor_proto = tensor_pb2.TensorProto(dtype=proto_dtype, tensor_shape=proto_shape)
tensor_proto.tensor_content = data.tostring()
return tensor_proto
def testFormatUninitializedTensor(self):
tensor_proto = tensor_pb2.TensorProto(
dtype=types_pb2.DataType.Value("DT_FLOAT"),
tensor_shape=tensor_shape_pb2.TensorShapeProto(
dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]))
out = tensor_format.format_tensor(
debug_data.InconvertibleTensorProto(tensor_proto, False), "a")
self.assertEqual(["Tensor \"a\":", "", "Uninitialized tensor:"],
out.lines[:3])
self.assertEqual(str(tensor_proto).split("\n"), out.lines[3:])
def add_text(self, tag, text, step):
"""Log a text."""
pluginData = tf.SummaryMetadata.PluginData(plugin_name='text')
smd = tf.SummaryMetadata(plugin_data=pluginData)
tensor = tensor_pb2.TensorProto(dtype='DT_STRING',
string_val=[text.encode(encoding='utf_8')],
tensor_shape=tensor_shape_pb2.TensorShapeProto(dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]))
summary = tf.Summary(value=[tf.Summary.Value(tag=tag, metadata=smd, tensor=tensor)])
self._add_summary(summary, step)
def make_tensor_proto(img):
tensor_shape = list(img.shape)
dims = [
tensor_shape_pb2.TensorShapeProto.Dim(size=dim) for dim in tensor_shape
]
tensor_shape = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor = tensor_pb2.TensorProto(
dtype=types_pb2.DT_FLOAT,
tensor_shape=tensor_shape,
float_val=list(img.reshape(-1)),
)
return tensor
def testLocateTensorElementAnnotationsUnavailable(self):
tensor_proto = tensor_pb2.TensorProto(
dtype=types_pb2.DataType.Value("DT_FLOAT"),
tensor_shape=tensor_shape_pb2.TensorShapeProto(
dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]))
out = tensor_format.format_tensor(
debug_data.InconvertibleTensorProto(tensor_proto, False), "a")
self.assertEqual(["Tensor \"a\":", "", "Uninitialized tensor:"],
out.lines[:3])
with self.assertRaisesRegexp(
AttributeError, "tensor_metadata is not available in annotations"):
tensor_format.locate_tensor_element(out, [0])
def prepare_output_as_list(inference_output, model_available_outputs):
response = predict_pb2.PredictResponse()
for key, value in model_available_outputs.items():
if value in inference_output:
dtype = dtypes.as_dtype(inference_output[value].dtype)
output_tensor = tensor_pb2.TensorProto(
dtype=dtype.as_datatype_enum,
tensor_shape=tensor_shape.as_shape(
inference_output[value].shape).as_proto())
result = inference_output[value].flatten()
tensor_util._NP_TO_APPEND_FN[dtype.as_numpy_dtype](output_tensor,
result)
response.outputs[key].CopyFrom(output_tensor)
return response
def make_tensor_proto(data):
# replace tf.make_tensor_proto(data, shape=data.shape) to optimize client request
shape = data.shape
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=i) for i in shape]
proto_shape= tensor_shape_pb2.TensorShapeProto(dim=dims)
proto_dtype = dtypes_as_dtype(data.dtype)
tensor_proto = tensor_pb2.TensorProto(
dtype=proto_dtype,
tensor_shape=proto_shape)
tensor_proto.tensor_content = data.tostring()
return tensor_proto
def _parse_json_request(serialized_data):
"""
json deserialization works in the following order:
1 - tries to deserialize the payload as a tensor using google.protobuf.json_format.Parse(
payload, tensor_pb2.TensorProto())
2 - in case it fails uses common json.loads deserialization
Args:
serialized_data: (str) json data to be deserialized
Returns:
deserialized object
"""
try:
return json_format.Parse(serialized_data, tensor_pb2.TensorProto())
except json_format.ParseError:
return json.loads(serialized_data)
def _prepare_output_as_AppendArrayToTensorProto(inference_output,
model_available_outputs):
response = predict_pb2.PredictResponse()
for response_output_name, model_output_name in \
model_available_outputs.items():
if model_output_name in inference_output:
dtype = dtypes.as_dtype(inference_output[model_output_name].dtype)
output_tensor = tensor_pb2.TensorProto(
dtype=dtype.as_datatype_enum,
tensor_shape=tensor_shape.as_shape(
inference_output[model_output_name].shape).as_proto())
result = inference_output[model_output_name].flatten()
tensor_util._NP_TO_APPEND_FN[dtype.as_numpy_dtype](output_tensor,
result)
response.outputs[response_output_name].CopyFrom(output_tensor)
return response
def do_inference(hostport, image_str):
"""Tests PredictionService with concurrent requests.
Args:
hostport: Host:port address of the PredictionService.
path: The full path of working directory for test data set.
Returns:
The classification error rate.
Raises:
IOError: An error occurred processing test data set.
"""
#t0 = time.clock()
channel = grpc.insecure_channel(hostport)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'image_class'
request.model_spec.signature_name = 'predict_images'
#t1 = time.clock()
image = load_image_and_preprocess(image_str)
#t2 = time.clock()
#request.inputs['input'].CopyFrom(
# tf.contrib.util.make_tensor_proto(image, shape=[1, 224, 224, 3]))
# use this approach can speed up over 100 times
dims = [
tensor_shape_pb2.TensorShapeProto.Dim(size=1),
tensor_shape_pb2.TensorShapeProto.Dim(size=-1),
tensor_shape_pb2.TensorShapeProto.Dim(size=-1),
tensor_shape_pb2.TensorShapeProto.Dim(size=3)
]
tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor_proto = tensor_pb2.TensorProto(dtype=types_pb2.DT_FLOAT,
tensor_shape=tensor_shape_proto,
float_val=image)
request.inputs['images'].CopyFrom(tensor_proto)
#t3 = time.clock()
result = stub.Predict(request, 5.0) # 5 seconds
#t4 = time.clock()
response = numpy.array(result.outputs['scores'].float_val)
print(numpy.shape(response))
#t5 = time.clock()
prediction = numpy.argmax(response)
#print((t1-t0),(t2-t1),(t3-t2),(t4-t3),(t5-t4))
return prediction, response
def predict():
# MODEL PARAMS
max_seq_length = 128
channel = grpc.insecure_channel("bert-scores:8500")
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
# Parse Description
tokenizer = tokenization.FullTokenizer(vocab_file="asset/vocab.txt",
do_lower_case=True)
processor = classifiers.CompetencyProcessor()
label_list = processor.get_labels()
content = request.get_json()
request_id = str(random.randint(1, 9223372036854775807))
inputExample = processor._create_example(
[request_id, content['description']], 'test')
tf_example = classifiers.from_record_to_tf_example(3, inputExample,
label_list,
max_seq_length,
tokenizer)
model_input = tf_example.SerializeToString()
# Send request
# See prediction_service.proto for gRPC request/response details.
model_request = predict_pb2.PredictRequest()
model_request.model_spec.name = 'bert'
model_request.model_spec.signature_name = 'serving_default'
dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=1)]
tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
tensor_proto = tensor_pb2.TensorProto(dtype=types_pb2.DT_STRING,
tensor_shape=tensor_shape_proto,
string_val=[model_input])
model_request.inputs['examples'].CopyFrom(tensor_proto)
result = stub.Predict(model_request, 10.0) # 10 secs timeout
result = tf.make_ndarray(result.outputs["probabilities"])
pretty_result = "Predicted Label: " + label_list[result[0].argmax(axis=0)]
app.logger.info("Predicted Label: %s",
label_list[result[0].argmax(axis=0)])
return pretty_result
