class OpDefUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): @parameterized.parameters([ ("any", "Foo", "Foo"), ("any", 12, 12), ("any", {2: 3}, {2: 3}), ("string", "Foo", "Foo"), ("string", b"Foo", b"Foo"), ("int", 12, 12), ("int", 12.3, 12), ("float", 12, 12.0), ("float", 12.3, 12.3), ("bool", True, True), ("shape", tensor_shape.TensorShape([3]), tensor_shape.TensorShape([3])), ("shape", [3], tensor_shape.TensorShape([3])), ("type", dtypes.int32, dtypes.int32), ("type", np.int32, dtypes.int32), ("type", "int32", dtypes.int32), ("tensor", tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT), tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT)), ("tensor", "dtype: DT_FLOAT", tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT)), ("list(any)", [1, "foo", 7.3, dtypes.int32], [1, "foo", 7.3, dtypes.int32]), ("list(any)", (1, "foo"), [1, "foo"]), ("list(string)", ["foo", "bar"], ["foo", "bar"]), ("list(string)", ("foo", "bar"), ["foo", "bar"]), ("list(string)", iter("abcd"), ["a", "b", "c", "d"]), ("list(int)", (1, 2.3), [1, 2]), ("list(float)", (1, 2.3), [1.0, 2.3]), ("list(bool)", [True, False], [True, False]), ]) def testConvert(self, attr_type, value, expected): result = _op_def_util.ConvertPyObjectToAttributeType(value, attr_type) # Check that we get the expected value(s). self.assertEqual(expected, result) # Check that we get the expected type(s). self.assertEqual(type(expected), type(result)) if isinstance(result, list): for expected_item, result_item in zip(expected, result): self.assertEqual(type(expected_item), type(result_item)) @parameterized.parameters([ ("string", 12), ("int", "foo"), ("float", "foo"), ("bool", 1), ("dtype", None), ("shape", 12.0), ("tensor", [1, 2, 3]), ("list(any)", 12), ("list(int)", [1, "two"]), ("list(string)", [1, "two"]), ("tensor", "string that is not a text-formatted TensorProto"), ]) def testConvertError(self, attr_type, value): with self.assertRaisesRegex(TypeError, "Failed to convert value"): _op_def_util.ConvertPyObjectToAttributeType(value, attr_type)
def testInconvertibleTensorProto(self): self.assertFalse(debug_data.has_inf_or_nan( self._dummy_datum, debug_data.InconvertibleTensorProto(tensor_pb2.TensorProto(), initialized=False))) self.assertFalse(debug_data.has_inf_or_nan( self._dummy_datum, debug_data.InconvertibleTensorProto(tensor_pb2.TensorProto(), initialized=True)))
def rnet_serving(image): host = '192.168.1.253' port = 9000 channel = implementations.insecure_channel(host, port) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'mtcnn' request.model_spec.signature_name = 'rnet_predict' image_data_arr = generate_input_string(image) image_data_arr = np.asarray(image_data_arr).squeeze() num_image = image_data_arr.shape[0] dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=num_image)] tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DT_STRING, tensor_shape=tensor_shape_proto, string_val=[image_data for image_data in image_data_arr]) request.inputs['images'].CopyFrom(tensor_proto) result = stub.Predict(request, 10.0) return [ MakeNdarray(result.outputs['result1']), MakeNdarray(result.outputs['result2']) ]
def run_inference_for_single_image_grpc(self,image,project): # Get handles to input and output tensors request = predict_pb2.PredictRequest() # Specify model name (must be the same as when the TensorFlow serving serving was started) request.model_spec.name = project.model dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=1)] tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DT_STRING, tensor_shape=tensor_shape_proto, string_val=[image.tobytes()]) # Initalize prediction # Specify signature name (should be the same as specified when exporting model) request.model_spec.signature_name = '' request.inputs['inputs'].CopyFrom( tensor_proto) result = project.stub.Predict(request, 10.0) # Run inference output_dict = {} # all outputs are float32 numpy arrays, so convert types as appropriate output_dict['num_detections'] = np.squeeze(result.outputs['num_detections'].float_val).astype(np.int32) output_dict['detection_classes'] = np.squeeze(result.outputs['detection_classes'].float_val).astype(np.int32) output_dict['detection_boxes'] = np.reshape(result.outputs['detection_boxes'].float_val,[100,4]) output_dict['detection_scores'] = result.outputs['detection_scores'].float_val return output_dict
def _original_node_def(node_def, lnf, variable_values): if lnf.original.t != graph_types_pb2.TFSavedModel: raise ValueError("original node not a TFSavedModel: {0}".format( lnf.name)) # Use serialized node def and fix inputs node_def.ParseFromString(lnf.original.serialized_node) del (node_def.input[:]) node_def.input.extend(ExportTFSavedModel._get_tensor_names(lnf.inputs)) for cntrl in lnf.control_inputs: node_def.input.append("^" + cntrl) # Get values from variables if lnf.original.op == ops_pb2.VARIABLE: if (tf_saved_model_base_importer.ImportTFSavedModelBase.TENSOR_PB not in lnf.original.attr): raise ValueError( "Could not find value associated with variable node: {}". format(lnf)) tensor_pb = tensor_pb2.TensorProto() tensor_pb.ParseFromString( lnf.original.attr[tf_saved_model_base_importer. ImportTFSavedModelBase.TENSOR_PB].v) variable_values[node_def.name] = tf.make_ndarray(tensor_pb)
def make_tensor_proto(data, dtype, size) -> tensor_pb2.TensorProto: dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=size)] tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto(dtype=dtype, tensor_shape=tensor_shape_proto, string_val=data) return tensor_proto
def main(): if len(sys.argv) != 2: print("error: only 1 argument accepted") sys.exit(1) ch = grpc.insecure_channel(sys.argv[1]) stub = prediction_service_pb2_grpc.PredictionServiceStub(ch) # same example inputs from https://www.tensorflow.org/tfx/serving/docker inputs = [1.0, 2.0, 5.0] input_tensor = tensor_pb2.TensorProto( dtype=types_pb2.DT_FLOAT, tensor_shape=tensor_shape_pb2.TensorShapeProto( dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=len(inputs), )]), float_val=inputs, ) req = predict_pb2.PredictRequest() req.model_spec.name = "half_plus_two" req.model_spec.signature_name = "serving_default" req.inputs["x"].CopyFrom(input_tensor) res = stub.Predict(req) print(text_format.MessageToString(res)) sys.exit(0)
def testZerosLikeVariant(self): # TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant # copying between CPU and GPU is supported AND we register a # ZerosLike callback for GPU for Variant storing primitive types # in variant_op_registry.cc. with self.test_session(use_gpu=False): variant_tensor = tensor_pb2.TensorProto( dtype=dtypes_lib.variant.as_datatype_enum, tensor_shape=tensor_shape.TensorShape([]).as_proto(), variant_val=[ tensor_pb2.VariantTensorDataProto( # Match registration in variant_op_registry.cc type_name=b"int", metadata=np.array(1, dtype=np.int32).tobytes()) ]) const_variant = constant_op.constant(variant_tensor) zeros_like = array_ops.zeros_like(const_variant) zeros_like_op = logging_ops.Print( zeros_like, [const_variant, zeros_like], message= "Variant storing an int, input and output of zeros_like:").op # Smoke test -- ensure this executes without trouble. # Right now, non-numpy-compatible objects cannot be returned from a # session.run call; similarly, objects that can't be converted to # native numpy types cannot be passed to ops.convert_to_tensor. # TODO(ebrevdo): Add registration mechanism for # ops.convert_to_tensor and for session.run output. zeros_like_op.run()
def construct_tensor(shapes, value): dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=dim) for dim in shapes] tensor_shape = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor = tensor_pb2.TensorProto(dtype=types_pb2.DT_INT32, tensor_shape=tensor_shape, int_val=value) return tensor
def testVariant(self): # TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant # copying between CPU and GPU is supported. with self.test_session(use_gpu=False): variant_tensor = tensor_pb2.TensorProto( dtype=dtypes_lib.variant.as_datatype_enum, tensor_shape=tensor_shape.TensorShape([]).as_proto(), variant_val=[ tensor_pb2.VariantTensorDataProto( # Match registration in variant_op_registry.cc type_name=b"int", metadata=np.array(1, dtype=np.int32).tobytes()) ]) const = constant_op.constant(variant_tensor) const_value = const.op.get_attr("value") # Ensure we stored the tensor proto properly. self.assertProtoEquals(variant_tensor, const_value) # Smoke test -- ensure this executes without trouble. # Right now, non-numpy-compatible objects cannot be returned from a # session.run call; similarly, objects that can't be converted to # native numpy types cannot be passed to ops.convert_to_tensor. # TODO(ebrevdo): Add registration mechanism for # ops.convert_to_tensor and for session.run output. logging_const_op = logging_ops.Print( const, [const], message="Variant storing an int, decoded const value:").op logging_const_op.run()
def facenet_serving(images): host = '172.20.15.85' port = 8500 channel = implementations.insecure_channel(host, port) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = '1' request.model_spec.signature_name = 'calculate_embeddings' image_data_arr = np.asarray(images) #print('image_data_arr',image_data_arr) #print('image_data',image_data_arr.dtype) input_size = image_data_arr.shape[0] dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=input_size)] print('dims',dims) tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DT_STRING, tensor_shape=tensor_shape_proto, string_val=[image_data for image_data in image_data_arr]) print('tensor_proto', tensor_proto) request.inputs['images'].CopyFrom(tensor_proto) result = stub.Predict(request, 10.0) print(MakeNdarray(result.outputs['embeddings'])) return MakeNdarray(result.outputs['embeddings'])
def onet_serving(image, channel): stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'mtcnn' request.model_spec.signature_name = 'onet_predict' image_data_arr = generate_input_string(image) image_data_arr = np.asarray(image_data_arr).squeeze() num_image = image_data_arr.shape[0] dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=num_image)] tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DT_STRING, tensor_shape=tensor_shape_proto, string_val=[image_data for image_data in image_data_arr]) request.inputs['images'].CopyFrom(tensor_proto) result = stub.Predict(request, 10.0) return [ MakeNdarray(result.outputs['result1']), MakeNdarray(result.outputs['result2']), MakeNdarray(result.outputs['result3']) ]
def pull_embedding_vectors(self, request, _): result = tensor_pb2.TensorProto() if not request.ids: return result embedding_vectors = self._parameters.get_embedding_param( request.name, request.ids) serialize_ndarray(embedding_vectors, result) return result
def __call__(self, stream, content_type): try: data = stream.read() finally: stream.close() try: return json_format.Parse(data, tensor_pb2.TensorProto()) except json_format.ParseError: return json.loads(data.decode())
def make_tensor_proto_float(data, shape): dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=i) for i in shape] tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DT_FLOAT, tensor_shape=tensor_shape_proto, float_val=np.array(data).ravel().tolist()) return tensor_proto
def testFormatResourceTypeTensor(self): tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DataType.Value("DT_RESOURCE"), tensor_shape=tensor_shape_pb2.TensorShapeProto( dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)])) out = tensor_format.format_tensor( debug_data.InconvertibleTensorProto(tensor_proto), "a") self.assertEqual(["Tensor \"a\":", ""], out.lines[:2]) self.assertEqual(str(tensor_proto).split("\n"), out.lines[2:])
def make_tensor_proto_wrapped(values, dtype=None, shape=None, verify_shape=False, allow_broadcast=False): try: return make_tensor_proto_original(values, dtype, shape, verify_shape, allow_broadcast) except ValueError: if dtype is None: dtype = tf.dtypes.as_dtype(values.dtype).as_datatype_enum tensor_proto = tensor_pb2.TensorProto( dtype=dtype, tensor_shape=tensor_shape.as_shape(values.shape).as_proto()) tensor_proto.tensor_content = values.tobytes() return tensor_proto
def make_tensor(v, arg_name): """Ensure v is a TensorProto.""" if isinstance(v, tensor_pb2.TensorProto): return v elif isinstance(v, six.string_types): pb = tensor_pb2.TensorProto() text_format.Merge(v, pb) return pb raise TypeError( "Don't know how to convert %s to a TensorProto for argument '%s'." % (repr(v), arg_name))
def create_constant_variant(value): return constant_op.constant( tensor_pb2.TensorProto( dtype=dtypes.variant.as_datatype_enum, tensor_shape=tensor_shape.TensorShape([]).as_proto(), variant_val=[ tensor_pb2.VariantTensorDataProto( # Match registration in variant_op_registry.cc type_name=b"int", metadata=np.array(value, dtype=np.int32).tobytes()) ]))
def make_tensor_proto(data): shape = data.shape dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=i) for i in shape] proto_shape = tensor_shape_pb2.TensorShapeProto(dim=dims) proto_dtype = dtypes_as_dtype(data.dtype) tensor_proto = tensor_pb2.TensorProto(dtype=proto_dtype, tensor_shape=proto_shape) tensor_proto.tensor_content = data.tostring() return tensor_proto
def testFormatUninitializedTensor(self): tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DataType.Value("DT_FLOAT"), tensor_shape=tensor_shape_pb2.TensorShapeProto( dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)])) out = tensor_format.format_tensor( debug_data.InconvertibleTensorProto(tensor_proto, False), "a") self.assertEqual(["Tensor \"a\":", "", "Uninitialized tensor:"], out.lines[:3]) self.assertEqual(str(tensor_proto).split("\n"), out.lines[3:])
def add_text(self, tag, text, step): """Log a text.""" pluginData = tf.SummaryMetadata.PluginData(plugin_name='text') smd = tf.SummaryMetadata(plugin_data=pluginData) tensor = tensor_pb2.TensorProto(dtype='DT_STRING', string_val=[text.encode(encoding='utf_8')], tensor_shape=tensor_shape_pb2.TensorShapeProto(dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)])) summary = tf.Summary(value=[tf.Summary.Value(tag=tag, metadata=smd, tensor=tensor)]) self._add_summary(summary, step)
def make_tensor_proto(img): tensor_shape = list(img.shape) dims = [ tensor_shape_pb2.TensorShapeProto.Dim(size=dim) for dim in tensor_shape ] tensor_shape = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor = tensor_pb2.TensorProto( dtype=types_pb2.DT_FLOAT, tensor_shape=tensor_shape, float_val=list(img.reshape(-1)), ) return tensor
def testLocateTensorElementAnnotationsUnavailable(self): tensor_proto = tensor_pb2.TensorProto( dtype=types_pb2.DataType.Value("DT_FLOAT"), tensor_shape=tensor_shape_pb2.TensorShapeProto( dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)])) out = tensor_format.format_tensor( debug_data.InconvertibleTensorProto(tensor_proto, False), "a") self.assertEqual(["Tensor \"a\":", "", "Uninitialized tensor:"], out.lines[:3]) with self.assertRaisesRegexp( AttributeError, "tensor_metadata is not available in annotations"): tensor_format.locate_tensor_element(out, [0])
def prepare_output_as_list(inference_output, model_available_outputs): response = predict_pb2.PredictResponse() for key, value in model_available_outputs.items(): if value in inference_output: dtype = dtypes.as_dtype(inference_output[value].dtype) output_tensor = tensor_pb2.TensorProto( dtype=dtype.as_datatype_enum, tensor_shape=tensor_shape.as_shape( inference_output[value].shape).as_proto()) result = inference_output[value].flatten() tensor_util._NP_TO_APPEND_FN[dtype.as_numpy_dtype](output_tensor, result) response.outputs[key].CopyFrom(output_tensor) return response
def make_tensor_proto(data): # replace tf.make_tensor_proto(data, shape=data.shape) to optimize client request shape = data.shape dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=i) for i in shape] proto_shape= tensor_shape_pb2.TensorShapeProto(dim=dims) proto_dtype = dtypes_as_dtype(data.dtype) tensor_proto = tensor_pb2.TensorProto( dtype=proto_dtype, tensor_shape=proto_shape) tensor_proto.tensor_content = data.tostring() return tensor_proto
def _parse_json_request(serialized_data): """ json deserialization works in the following order: 1 - tries to deserialize the payload as a tensor using google.protobuf.json_format.Parse( payload, tensor_pb2.TensorProto()) 2 - in case it fails uses common json.loads deserialization Args: serialized_data: (str) json data to be deserialized Returns: deserialized object """ try: return json_format.Parse(serialized_data, tensor_pb2.TensorProto()) except json_format.ParseError: return json.loads(serialized_data)
def _prepare_output_as_AppendArrayToTensorProto(inference_output, model_available_outputs): response = predict_pb2.PredictResponse() for response_output_name, model_output_name in \ model_available_outputs.items(): if model_output_name in inference_output: dtype = dtypes.as_dtype(inference_output[model_output_name].dtype) output_tensor = tensor_pb2.TensorProto( dtype=dtype.as_datatype_enum, tensor_shape=tensor_shape.as_shape( inference_output[model_output_name].shape).as_proto()) result = inference_output[model_output_name].flatten() tensor_util._NP_TO_APPEND_FN[dtype.as_numpy_dtype](output_tensor, result) response.outputs[response_output_name].CopyFrom(output_tensor) return response
def do_inference(hostport, image_str): """Tests PredictionService with concurrent requests. Args: hostport: Host:port address of the PredictionService. path: The full path of working directory for test data set. Returns: The classification error rate. Raises: IOError: An error occurred processing test data set. """ #t0 = time.clock() channel = grpc.insecure_channel(hostport) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) request = predict_pb2.PredictRequest() request.model_spec.name = 'image_class' request.model_spec.signature_name = 'predict_images' #t1 = time.clock() image = load_image_and_preprocess(image_str) #t2 = time.clock() #request.inputs['input'].CopyFrom( # tf.contrib.util.make_tensor_proto(image, shape=[1, 224, 224, 3])) # use this approach can speed up over 100 times dims = [ tensor_shape_pb2.TensorShapeProto.Dim(size=1), tensor_shape_pb2.TensorShapeProto.Dim(size=-1), tensor_shape_pb2.TensorShapeProto.Dim(size=-1), tensor_shape_pb2.TensorShapeProto.Dim(size=3) ] tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto(dtype=types_pb2.DT_FLOAT, tensor_shape=tensor_shape_proto, float_val=image) request.inputs['images'].CopyFrom(tensor_proto) #t3 = time.clock() result = stub.Predict(request, 5.0) # 5 seconds #t4 = time.clock() response = numpy.array(result.outputs['scores'].float_val) print(numpy.shape(response)) #t5 = time.clock() prediction = numpy.argmax(response) #print((t1-t0),(t2-t1),(t3-t2),(t4-t3),(t5-t4)) return prediction, response
def predict(): # MODEL PARAMS max_seq_length = 128 channel = grpc.insecure_channel("bert-scores:8500") stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) # Parse Description tokenizer = tokenization.FullTokenizer(vocab_file="asset/vocab.txt", do_lower_case=True) processor = classifiers.CompetencyProcessor() label_list = processor.get_labels() content = request.get_json() request_id = str(random.randint(1, 9223372036854775807)) inputExample = processor._create_example( [request_id, content['description']], 'test') tf_example = classifiers.from_record_to_tf_example(3, inputExample, label_list, max_seq_length, tokenizer) model_input = tf_example.SerializeToString() # Send request # See prediction_service.proto for gRPC request/response details. model_request = predict_pb2.PredictRequest() model_request.model_spec.name = 'bert' model_request.model_spec.signature_name = 'serving_default' dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=1)] tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims) tensor_proto = tensor_pb2.TensorProto(dtype=types_pb2.DT_STRING, tensor_shape=tensor_shape_proto, string_val=[model_input]) model_request.inputs['examples'].CopyFrom(tensor_proto) result = stub.Predict(model_request, 10.0) # 10 secs timeout result = tf.make_ndarray(result.outputs["probabilities"]) pretty_result = "Predicted Label: " + label_list[result[0].argmax(axis=0)] app.logger.info("Predicted Label: %s", label_list[result[0].argmax(axis=0)]) return pretty_result