def _evaluateTFLiteModel(self, tflite_model, input_data, input_shapes=None): """Evaluates the model on the `input_data`. Args: tflite_model: TensorFlow Lite model. input_data: List of EagerTensor const ops containing the input data for each input tensor. input_shapes: List of tuples representing the `shape_signature` and the new shape of each input tensor that has unknown dimensions. Returns: [np.ndarray] """ interpreter = Interpreter(model_content=tflite_model) input_details = interpreter.get_input_details() if input_shapes: for idx, (shape_signature, final_shape) in enumerate(input_shapes): self.assertTrue( (input_details[idx]['shape_signature'] == shape_signature ).all()) interpreter.resize_tensor_input(idx, final_shape) interpreter.allocate_tensors() output_details = interpreter.get_output_details() for input_tensor, tensor_data in zip(input_details, input_data): interpreter.set_tensor(input_tensor['index'], tensor_data.numpy()) interpreter.invoke() return [ interpreter.get_tensor(details['index']) for details in output_details ]
def testFunctionalSequentialModel(self): """Test a Functional tf.keras model containing a Sequential model.""" with session.Session().as_default(): model = keras.models.Sequential() model.add(keras.layers.Dense(2, input_shape=(3,))) model.add(keras.layers.RepeatVector(3)) model.add(keras.layers.TimeDistributed(keras.layers.Dense(3))) model = keras.models.Model(model.input, model.output) model.compile( loss=keras.losses.MSE, optimizer=keras.optimizers.RMSprop(), metrics=[keras.metrics.categorical_accuracy], sample_weight_mode='temporal') x = np.random.random((1, 3)) y = np.random.random((1, 3, 3)) model.train_on_batch(x, y) model.predict(x) model.predict(x) fd, keras_file = tempfile.mkstemp('.h5') try: keras.models.save_model(model, keras_file) finally: os.close(fd) # Convert to TFLite model. converter = lite.TFLiteConverter.from_keras_model_file(keras_file) tflite_model = converter.convert() self.assertTrue(tflite_model) # Check tensor details of converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertEqual(1, len(input_details)) self.assertEqual('dense_input', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) self.assertTrue(([1, 3] == input_details[0]['shape']).all()) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() self.assertEqual(1, len(output_details)) self.assertEqual('time_distributed/Reshape_1', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) self.assertTrue(([1, 3, 3] == output_details[0]['shape']).all()) self.assertEqual((0., 0.), output_details[0]['quantization']) # Check inference of converted model. input_data = np.array([[1, 2, 3]], dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], input_data) interpreter.invoke() tflite_result = interpreter.get_tensor(output_details[0]['index']) keras_model = keras.models.load_model(keras_file) keras_result = keras_model.predict(input_data) np.testing.assert_almost_equal(tflite_result, keras_result, 5) os.remove(keras_file)
def testDeprecatedFlags(self): with ops.Graph().as_default(): in_tensor = array_ops.placeholder(shape=[1, 4], dtype=dtypes.float32) out_tensor = in_tensor + in_tensor sess = session.Session() # Convert model and ensure model is not None. converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) converter.target_ops = set([lite.OpsSet.SELECT_TF_OPS]) # Ensure `target_ops` is set to the correct value after flag deprecation. self.assertEqual(converter.target_ops, set([lite.OpsSet.SELECT_TF_OPS])) self.assertEqual(converter.target_spec.supported_ops, set([lite.OpsSet.SELECT_TF_OPS])) tflite_model = converter.convert() self.assertTrue(tflite_model) # Check the model works with TensorFlow ops. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() test_input = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], test_input) interpreter.invoke() output_details = interpreter.get_output_details() expected_output = np.array([[2.0, 4.0, 6.0, 8.0]], dtype=np.float32) output_data = interpreter.get_tensor(output_details[0]['index']) self.assertTrue((expected_output == output_data).all())
class LiteModel: def __init__(self, model_content): model_content = bytes(model_content) self.interpreter = Interpreter(model_content=model_content) input_details = self.interpreter.get_input_details() output_details = self.interpreter.get_output_details() self.input_shape = input_details[0]['shape'][1:] self.input_index = input_details[0]['index'] self.output_index = output_details[0]['index'] self.input_scale, self.input_zero_point = input_details[0][ 'quantization'] self.output_scale, self.output_zero_point = output_details[0][ 'quantization'] self.interpreter.allocate_tensors() def predict(self, X): X = self.input_map(X) self.interpreter.set_tensor(self.input_index, X) self.interpreter.invoke() Y = self.interpreter.get_tensor(self.output_index) Y = self.output_map(Y) return Y def input_map(self, x): return np.array(x, dtype=np.float32) # return np.array(x / self.input_scale + self.input_zero_point, dtype=np.uint8) def output_map(self, y): return np.array(y, dtype=np.float32)
def testL2LossOp(self, tf_quantization_mode): root = tracking.AutoTrackable() root.l2_loss_func = def_function.function(lambda x: nn_ops.l2_loss(x)) # pylint: disable=unnecessary-lambda input_data = tf.range(4, dtype=tf.float32) concrete_func = root.l2_loss_func.get_concrete_function(input_data) converter = lite.TFLiteConverterV2.from_concrete_functions([concrete_func], root) converter._experimental_tf_quantization_mode = tf_quantization_mode tflite_model = converter.convert() self.assertTrue(tflite_model) self.assertIn('FlexL2Loss', tflite_test_util.get_ops_list(tflite_model)) # Check the model works. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() test_input = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], test_input) interpreter.invoke() output_details = interpreter.get_output_details() expected_output = np.array([15.0], dtype=np.float32) output_data = interpreter.get_tensor(output_details[0]['index']) self.assertTrue((expected_output == output_data).all())
def _set_input_tensors(self, interpreter: _interpreter.Interpreter, tensor_data: Sequence[np.ndarray], initialize: bool) -> None: """Sets input tensors into TFLite model Interpreter. Args: interpreter: a tf.lite.Interpreter object with allocated tensors. tensor_data: a list of Numpy array data. initialize: set to true when input is first set for the interpreter, to set input shapes and allocate tensors. Raises: ValueError: when inputs can't be set, or size of provided inputs does not match size of model inputs. """ input_details = interpreter.get_input_details() if len(input_details) != len(tensor_data): raise ValueError( 'Number of inputs provided ({}) does not match number of inputs to ' 'the model ({})'.format(len(tensor_data), len(input_details))) if initialize: for input_detail, tensor in zip(input_details, tensor_data): interpreter.resize_tensor_input(input_detail['index'], tensor.shape) interpreter.allocate_tensors() for input_detail, tensor in zip(input_details, tensor_data): if tensor.dtype == np.float32 and input_detail['dtype'] == np.int8: quant_params = _get_quant_params(input_detail) if quant_params: scale, zero_point = quant_params tensor = np.round((tensor / scale) + zero_point).astype( np.int8) interpreter.set_tensor(input_detail['index'], tensor)
def testAddOp(self, tf_quantization_mode): root = tracking.AutoTrackable() root.add_func = def_function.function(lambda x: x + x) input_data = tf.reshape(tf.range(4, dtype=tf.float32), [1, 4]) concrete_func = root.add_func.get_concrete_function(input_data) # Convert model and check if the op is not flex. converter = lite.TFLiteConverterV2.from_concrete_functions([concrete_func], root) converter._experimental_tf_quantization_mode = tf_quantization_mode tflite_model = converter.convert() self.assertTrue(tflite_model) if tf_quantization_mode == 'LEGACY_INTEGER': self.assertIn('ADD', tflite_test_util.get_ops_list(tflite_model)) else: self.assertIn('FlexAddV2', tflite_test_util.get_ops_list(tflite_model)) # Check the model works. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() test_input = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], test_input) interpreter.invoke() output_details = interpreter.get_output_details() expected_output = np.array([[2.0, 4.0, 6.0, 8.0]], dtype=np.float32) output_data = interpreter.get_tensor(output_details[0]['index']) self.assertTrue((expected_output == output_data).all())
def __classify_frames(model_path, predictions_to_avg, classification_queue, has_started, should_stop, predictions, labels): # Load the model, then signal that the classifier has started interpreter = Interpreter(model_path) interpreter.allocate_tensors() input_tensor_index = interpreter.get_input_details()[0]['index'] output_tensor_index = interpreter.get_output_details()[0]['index'] has_started.set() # Set up class state tracking ys = collections.deque(maxlen=predictions_to_avg) # Loop over the supplied frames and process detections while not should_stop.is_set(): # Get the next input and prepare it for classification X = __drain(classification_queue) if X is None: time.sleep(1.0 / 25.0) continue # Run the classifier and add the result to the set of recent predictions interpreter.set_tensor(input_tensor_index, X) interpreter.invoke() y = interpreter.get_tensor(output_tensor_index)[0] ys.append(y) # Update the shared predictions using the average of recent predictions if len(ys) == predictions_to_avg: predictions[:] = np.mean(ys, axis=0)
def testFloat(self, enable_mlir): input_data = constant_op.constant(1., shape=[1]) root = tracking.AutoTrackable() root.v1 = variables.Variable(3.) root.v2 = variables.Variable(2.) root.f = def_function.function(lambda x: root.v1 * root.v2 * x) concrete_func = root.f.get_concrete_function(input_data) # Convert model. converter = lite.TFLiteConverterV2.from_concrete_functions( [concrete_func], root) converter.target_spec.supported_ops = set([lite.OpsSet.SELECT_TF_OPS]) converter.experimental_new_converter = enable_mlir tflite_model = converter.convert() # Check the model works with TensorFlow ops. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() test_input = np.array([4.0], dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], test_input) interpreter.invoke() output_details = interpreter.get_output_details() expected_output = np.array([24.0], dtype=np.float32) output_data = interpreter.get_tensor(output_details[0]['index']) self.assertTrue((expected_output == output_data).all())
def testFlexWithDoubleOp(self): # Create a graph that has one double op. saved_model_dir = os.path.join(self.get_temp_dir(), 'model2') with ops.Graph().as_default(): with session.Session() as sess: in_tensor = array_ops.placeholder(shape=[1, 4], dtype=dtypes.int32, name='input') out_tensor = double_op.double(in_tensor) inputs = {'x': in_tensor} outputs = {'z': out_tensor} saved_model.simple_save(sess, saved_model_dir, inputs, outputs) converter = lite.TFLiteConverterV2.from_saved_model(saved_model_dir) converter.target_spec.supported_ops = set([lite.OpsSet.SELECT_TF_OPS]) converter.target_spec.experimental_select_user_tf_ops = ['Double'] tflite_model = converter.convert() self.assertTrue(tflite_model) self.assertIn('FlexDouble', tflite_test_util.get_ops_list(tflite_model)) # Check the model works with TensorFlow ops. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() test_input = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.int32) interpreter.set_tensor(input_details[0]['index'], test_input) interpreter.invoke() output_details = interpreter.get_output_details() expected_output = np.array([[2.0, 4.0, 6.0, 8.0]], dtype=np.int32) output_data = interpreter.get_tensor(output_details[0]['index']) self.assertTrue((expected_output == output_data).all())
class TfLiteModel: def __init__(self, model_content): self.model_content = bytes(model_content) self.interpreter = Interpreter(model_content=self.model_content) input_details = self.interpreter.get_input_details() output_details = self.interpreter.get_output_details() print(input_details) self.input_index = input_details[0]['index'] self.output_index = output_details[0]['index'] self.input_scale, self.input_zero_point = input_details[0][ 'quantization'] self.output_scale, self.output_zero_point = output_details[0][ 'quantization'] self.interpreter.allocate_tensors() def forward(self, data_in): test_input = np.array(data_in / self.input_scale + self.input_zero_point, dtype=np.uint8).reshape(1, -1) self.interpreter.set_tensor(self.input_index, test_input) self.interpreter.invoke() output_data = self.interpreter.get_tensor(self.output_index)[0] return (np.array(output_data, dtype=np.float32) - self.output_zero_point) * self.output_scale
def testScalarValid(self): # Construct a graph using a scalar (empty shape) input. in_tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[]) out_tensor = in_tensor + in_tensor sess = session.Session() # Test conversion with the scalar input shape. converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) tflite_model = converter.convert() # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertEqual(1, len(input_details)) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) self.assertEqual(len(input_details[0]['shape']), 0) output_details = interpreter.get_output_details() self.assertEqual(1, len(output_details)) self.assertEqual('add', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) self.assertEqual(len(output_details[0]['shape']), 0) # Validate inference using the scalar inputs/outputs. test_input = np.array(4.0, dtype=np.float32) expected_output = np.array(8.0, dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], test_input) interpreter.invoke() output_data = interpreter.get_tensor(output_details[0]['index']) self.assertTrue((expected_output == output_data).all())
def infer(model_path, image): # Load the TensorFlow Lite model. interpreter = Interpreter(model_path=model_path) interpreter.allocate_tensors() # Get model details input_details = interpreter.get_input_details( ) # Dimensions of image the model expects output_details = interpreter.get_output_details() # Output Tensors input_height = input_details[0]['shape'][1] input_width = input_details[0]['shape'][2] floating_model = (input_details[0]['dtype'] == np.float32 ) # Checks if model is quantised # Load image and resize to expected shape [1xHxWx3] image = cv2.imread(image) image_height, image_width, _ = image.shape new_image = cv2.resize(image, (input_width, input_height)) input_data = np.expand_dims(new_image, axis=0) # Normalize pixel values if using a floating model (i.e. if model is non-quantized) input_mean = 127.5 input_std = 127.5 if floating_model: input_data = (np.float32(input_data) - input_mean) / input_std # Perform the actual detection by running the model with the image as input interpreter.set_tensor(input_details[0]['index'], input_data) interpreter.invoke() # Retrieve detection results bboxes = (interpreter.get_tensor( output_details[0]['index'])[0]).tolist() # Get all 4 bbox coordinates x_bboxes = [round(x[1], 2) for x in bboxes] # Extract x-coordinates classes = (interpreter.get_tensor(output_details[1]['index'])[0]).tolist() classes = [int(x) for x in classes] confidences = (interpreter.get_tensor( output_details[2]['index'])[0]).tolist() confidences = [round(x, 2) for x in confidences] prediction = sorted(zip(classes, confidences, x_bboxes), key=lambda t: t[1], reverse=True) # check for more than 9 detections if len(prediction) != 9: prediction = prediction[:-1] prediction = sorted(prediction, key=lambda t: t[2]) # sort by x-coordinate return prediction
def _evaluateTFLiteModel(self, tflite_model, input_data): """Evaluates the model on the `input_data`.""" interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() for input_tensor, tensor_data in zip(input_details, input_data): interpreter.set_tensor(input_tensor['index'], tensor_data.numpy()) interpreter.invoke() return interpreter.get_tensor(output_details[0]['index'])
class ClassifyBird(): def __init__(self): model_path = f"{os.path.dirname(__file__)}/aiy/lite-model_aiy_vision_classifier_birds_V1_3.tflite" label_path = f"{os.path.dirname(__file__)}/aiy/probability-labels-en.txt" print(f"[model_path] {model_path}") print(f"[label_path] {label_path}") self.interpreter = Interpreter(model_path=model_path) self.labels = self.load_labels(label_path) self.interpreter.allocate_tensors() self.input_details = self.interpreter.get_input_details() self.output_details = self.interpreter.get_output_details() self.height = self.input_details[0]['shape'][1] self.width = self.input_details[0]['shape'][2] self.floating_model = (self.input_details[0]['dtype'] == np.float32) self.input_mean = 127.5 self.input_std = 127.5 def load_labels(self, filename): with open(filename, 'r') as f: return [line.strip() for line in f.readlines()] def classify_path(self, path): image = cv2.imread(path) return self.__classify_array(image) def classify_image(self, image): return self.__classify_array(image) def __classify_array(self, image) -> ClassifyResultSet: image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) imH, imW, _ = image.shape image_resized = cv2.resize(image_rgb, (self.width, self.height)) input_data = np.expand_dims(image_resized, axis=0) if self.floating_model: input_data = (np.float32(input_data) - input_mean) / input_std self.interpreter.set_tensor(self.input_details[0]['index'], input_data) self.interpreter.invoke() output_data = self.interpreter.get_tensor( self.output_details[0]['index']) results = np.squeeze(output_data) return ClassifyResultSet(results, self.labels)
def testConvOpWithBias(self, tf_quantization_mode): class ConvModel(tracking.AutoTrackable): @def_function.function def conv_func(self, in_tensor, filter_tensor): bias = constant_op.constant(3., shape=[1]) conv_tensor = tf.nn.conv2d( in_tensor, filter_tensor, strides=[1, 1, 1, 1], dilations=[1, 1, 1, 1], padding='VALID', data_format='NHWC') conv_tensor = conv_tensor + bias return tf.nn.relu(conv_tensor) root = ConvModel() input_data = tf.reshape(tf.range(4, dtype=tf.float32), [1, 2, 2, 1]) filter_data = tf.reshape(tf.range(2, dtype=tf.float32), [1, 2, 1, 1]) concrete_func = root.conv_func.get_concrete_function( input_data, filter_data) converter = lite.TFLiteConverterV2.from_concrete_functions([concrete_func], root) converter._experimental_tf_quantization_mode = tf_quantization_mode tflite_model = converter.convert() self.assertTrue(tflite_model) self.assertCountEqual(['CONV_2D', 'RESHAPE'], tflite_test_util.get_ops_list(tflite_model)) # Check the model works. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() test_input = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32).reshape( (1, 2, 2, 1)) interpreter.set_tensor(input_details[0]['index'], test_input) test_filter = np.array([1.0, 0.0], dtype=np.float32).reshape((1, 2, 1, 1)) interpreter.set_tensor(input_details[1]['index'], test_filter) interpreter.invoke() output_details = interpreter.get_output_details() expected_output = np.array([[[[4.]], [[6.]]]], dtype=np.float32) output_data = interpreter.get_tensor(output_details[0]['index']) self.assertTrue((expected_output == output_data).all())
class RealtimeStereo(): def __init__(self, model_path): self.model = self.load_model(model_path) def load_model(self, model_path): self.interpreter = Interpreter(model_path, num_threads=4) self.interpreter.allocate_tensors() self.input_details = self.interpreter.get_input_details() input_shape = self.input_details[0]['shape'] self.input_height = input_shape[1] self.input_width = input_shape[2] self.channels = input_shape[2] self.output_details = self.interpreter.get_output_details() self.output_shape = self.output_details[0]['shape'] def preprocess(self, image): img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) img_input = cv2.resize( img, (self.input_width, self.input_height)).astype(np.float32) # Scale input pixel values to -1 to 1 mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] img_input = ((img_input / 255.0 - mean) / std) # img_input = img_input.transpose(2, 0, 1) img_input = img_input[np.newaxis, :, :, :] return img_input.astype(np.float32) def run(self, left, right): input_left = self.preprocess(left) input_right = self.preprocess(right) self.interpreter.set_tensor(self.input_details[0]['index'], input_left) self.interpreter.set_tensor(self.input_details[1]['index'], input_right) self.interpreter.invoke() disparity = self.interpreter.get_tensor( self.output_details[0]['index']) return np.squeeze(disparity)
def image_process(image): model_file = "/tmp/mobilenet_v1_1.0_224.tflite" label_file = "/tmp/labels.txt" interpreter = Interpreter(model_path=model_file) interpreter.allocate_tensors() input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() # check the type of the input tensor floating_model = input_details[0]['dtype'] == np.float32 # NxHxWxC, H:1, W:2 height = input_details[0]['shape'][1] width = input_details[0]['shape'][2] img = Image.open(image).resize((width, height)) # add N dim input_data = np.expand_dims(img, axis=0) if floating_model: input_data = (np.float32(input_data) - 127.5) / 127.5 interpreter.set_tensor(input_details[0]['index'], input_data) interpreter.invoke() output_data = interpreter.get_tensor(output_details[0]['index']) results = np.squeeze(output_data) top_k = results.argsort()[-3:][::-1] labels = load_labels(label_file) phrase = "There is a " + str( labels[top_k[0]]).split(":")[1] + " in front of you" return phrase for i in top_k: if floating_model: print('{:08.6f}: {}'.format(float(results[i]), labels[i])) else: print('{:08.6f}: {}'.format(float(results[i] / 255.0), labels[i]))
class Model: def __init__(self, PATH_TO_CKPT): # Load the Tensorflow Lite model. self.interpreter = Interpreter(model_path=PATH_TO_CKPT) self.interpreter.allocate_tensors() # Get model details self.input_details = self.interpreter.get_input_details() self.output_details = self.interpreter.get_output_details() self.height = self.input_details[0]['shape'][1] self.width = self.input_details[0]['shape'][2] self.floating_model = (self.input_details[0]['dtype'] == np.float32) self.input_mean = 127.5 self.input_std = 127.5 def model_out(self, frame): # Acquire frame and resize to expected shape [1xHxWx3] frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frame_resized = cv2.resize(frame_rgb, (self.width, self.height)) input_data = np.expand_dims(frame_resized, axis=0) # Normalize pixel values if using a floating model (i.e. if model is non-quantized) if self.floating_model: input_data = (np.float32(input_data) - self.input_mean) / self.input_std # Perform the actual detection by running the model with the image as input self.interpreter.set_tensor(self.input_details[0]['index'], input_data) self.interpreter.invoke() # Retrieve detection results boxes = self.interpreter.get_tensor(self.output_details[0]['index'])[ 0] # Bounding box coordinates of detected objects classes = self.interpreter.get_tensor(self.output_details[1]['index'])[ 0] # Class index of detected objects scores = self.interpreter.get_tensor(self.output_details[2]['index'])[ 0] # Confidence of detected objects # num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed) return scores, classes, boxes
def testDisableFlexTensorMemoryReusing(self): @tf.function(input_signature=[ tf.TensorSpec(shape=[2, 3], dtype=tf.float32, name='x') ]) def model(x): l = list_ops.tensor_list_reserve(element_dtype=tf.int64, element_shape=[None, 1], num_elements=2) init_state = (0, x, l) condition = lambda i, x, l: i < 2 def body(i, x, l): element = tf.where(x[i]) l = list_ops.tensor_list_set_item(l, i, element) return i + 1, x, l _, _, l_final = tf.while_loop(condition, body, init_state) return list_ops.tensor_list_stack(l_final, element_dtype=tf.int64) # Convert model. converter = lite.TFLiteConverterV2.from_concrete_functions( [model.get_concrete_function()]) converter.target_spec.supported_ops = set( [lite.OpsSet.TFLITE_BUILTINS, lite.OpsSet.SELECT_TF_OPS]) tflite_model = converter.convert() # Check the model produces correct result. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() test_input = np.array([[1.0, 2.0, 0.0], [0.0, 5.0, 6.0]], dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], test_input) interpreter.invoke() output_details = interpreter.get_output_details() expected_output = np.array([0, 1, 1, 2], dtype=np.int64) output_data = interpreter.get_tensor(output_details[0]['index']) self.assertTrue( (expected_output == np.ndarray.flatten(output_data)).all())
def testSequentialModel(self): """Test a Sequential tf.keras model with default inputs.""" keras_file = self._getSequentialModel() converter = lite.TFLiteConverter.from_keras_model_file(keras_file) tflite_model = converter.convert() self.assertTrue(tflite_model) # Check tensor details of converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertEqual(1, len(input_details)) self.assertEqual('dense_input', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) self.assertTrue(([1, 3] == input_details[0]['shape']).all()) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() self.assertEqual(1, len(output_details)) self.assertEqual('time_distributed/Reshape_1', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) self.assertTrue(([1, 3, 3] == output_details[0]['shape']).all()) self.assertEqual((0., 0.), output_details[0]['quantization']) # Check inference of converted model. input_data = np.array([[1, 2, 3]], dtype=np.float32) interpreter.set_tensor(input_details[0]['index'], input_data) interpreter.invoke() tflite_result = interpreter.get_tensor(output_details[0]['index']) keras_model = keras.models.load_model(keras_file) keras_result = keras_model.predict(input_data) np.testing.assert_almost_equal(tflite_result, keras_result, 5) os.remove(keras_file)
frames = backbone.getFrames() #get the frame from the camera frames = backbone.getAlignedFrame(frames) timeStamp = frames.get_timestamp() / 1000 #retrives the depth image from camera depth_frame = backbone.getDepthFrame(frames) # retrives color image as a np array color_image = backbone.colorImageCV2(frames) origH = color_image.shape[0] origW = color_image.shape[1] image_data = resize_image(color_image, width, height) #perform detection on the resized image interpreter.set_tensor(inputDetails[0]['index'], image_data) interpreter.invoke() #retrieve the model output first index correlates to output array boxes = interpreter.get_tensor(outputDetails[0]['index'])[0] # Bounding box cordinates #understand get tensor classes = interpreter.get_tensor(outputDetails[1]["index"])[0] # Class index of object/label score = interpreter.get_tensor(outputDetails[2]["index"])[0] # Score/ prediction a class was detected or confidence count = int(interpreter.get_tensor(outputDetails[3]["index"])[0]) # Number of objects detected """Retrieve the bounding box and scale it to the original image draw the boundiung box onto the image and apply the label above the object **************************************** To optimize for use on full implementation only check till an object is found to be to close set bool true inform user. Give a delay to allow user to adjust before warning the user of any other objects
ret = video.set(3,imW) ret = video.set(4,imH) while(True): # Acquire frame and resize to expected shape [1xHxWx3] ret, frame = video.read() frame_resized = cv2.resize(frame, (width, height)) input_data = np.expand_dims(frame_resized, axis=0) # Normalize pixel values if using a floating model (i.e. if model is non-quantized) if floating_model: input_data = (np.float32(input_data) - input_mean) / input_std # Perform the actual detection by running the model with the image as input interpreter.set_tensor(input_details[0]['index'],input_data) interpreter.invoke() # Retrieve detection results boxes = interpreter.get_tensor(output_details[0]['index'])[0] # Bounding box coordinates of detected objects classes = interpreter.get_tensor(output_details[1]['index'])[0] # Class index of detected objects scores = interpreter.get_tensor(output_details[2]['index'])[0] # Confidence of detected objects #num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed) # Loop over all detections and draw detection box if confidence is above minimum threshold for i in range(len(scores)): if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)): # Get bounding box coordinates and draw box # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min() ymin = int(max(1,(boxes[i][0] * imH)))
class Gesture: def __init__(self, width=None, height=None, pnet=None): pygame.init() self.clock = pygame.time.Clock() if width and height: self.WIDTH = width self.HEIGHT = height self.window = pygame.display.set_mode((self.WIDTH, self.HEIGHT)) else: info_object = pygame.display.Info() self.WIDTH = info_object.current_w self.HEIGHT = info_object.current_h flags = pygame.FULLSCREEN | pygame.DOUBLEBUF | pygame.HWSURFACE self.window = pygame.display.set_mode(flags=flags) pygame.display.set_caption('Posenet Hands') if pnet: self.pnet = pnet else: self.pnet = posenet_interface.posenetInterface(257) self.myfont = pygame.font.SysFont("Comic Sans MS", 40) self.image = None self.interpreter = Interpreter(model_path='hand_landmark.tflite') self.interpreter.allocate_tensors() self.input_details = self.interpreter.get_input_details() self.output_details = self.interpreter.get_output_details() def run(self): while 1: self.draw() for event in pygame.event.get(): if event.type == pygame.QUIT: pygame.quit() raise SystemExit elif event.type == pygame.KEYDOWN: if event.key == pygame.K_ESCAPE: pygame.quit() return pygame.display.update() self.clock.tick(60) def blit_cam_frame(self, frame, screen): # frame = np.fliplr(frame) frame = np.rot90(frame) frame = pygame.surfarray.make_surface(frame) screen.blit(pygame.transform.scale(frame, (self.WIDTH, self.HEIGHT)), (0, 0)) def scale_keypoints(self, original_shape, keypoints): scale = np.array([self.HEIGHT / original_shape[0], self.WIDTH / original_shape[1]]) return keypoints * scale def get_adjacent_keypoints(self, keypoints): results = [] for left, right in CONNECTED_POINTS: results.append( np.array([keypoints[left].pt, keypoints[right].pt]).astype(np.int32), ) return results def hand_landmark(self, input_image, right_hand): target_width = target_height = 256 input_image = input_image[0] scale_x = self.image.shape[1] / 256 scale_y = self.image.shape[0] / 256 shift_x = 0 shift_y = 0 # If hand keypoint is available if right_hand[0] != 0.0 or right_hand[1] != 0.0: right_hand = np.int0(right_hand) # Take square region near hand keypoint which is located at wrist square_size = 150 square_size_half = square_size // 2 shift_x = max(right_hand[1] - square_size_half, 0) shift_y = max(right_hand[0] - square_size, 0) input_image = self.image[shift_y:right_hand[0], shift_x:right_hand[1] + square_size_half] scale_x = input_image.shape[1] / 256 scale_y = input_image.shape[0] / 256 # TODO Find hand and rotate so hand is always upright # input_image = find_hand(new_img) if input_image is None: return # cv2.imshow('hand', input_image) input_image = input_image * (2.0 / 255.0) - 1.0 input_img = cv2.resize(input_image, (target_width, target_height), interpolation=cv2.INTER_LINEAR).astype( np.float32) input_img = np.expand_dims(input_img, 0) self.interpreter.set_tensor(self.input_details[0]['index'], input_img) start = time.time() self.interpreter.invoke() print('infer time:', time.time() - start) hand_points = self.interpreter.get_tensor(self.output_details[0]['index'])[0] hand_confidence = self.interpreter.get_tensor(self.output_details[1]['index']) if hand_confidence[0] < 0.1: return norm_hand_points = [] for i in range(0, len(hand_points), 2): x = (hand_points[i] * scale_x) + shift_x y = (hand_points[i + 1] * scale_y) + shift_y # cv2.putText(self.image, str(idx), (int(x), int(y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, # (255, 0, 0)) norm_hand_points.append(cv2.KeyPoint(x, y, 10)) return norm_hand_points def draw(self): self.image, keypoints, input_image = self.pnet.get_image(return_input_img=True) norm_hand_points = self.hand_landmark(input_image, keypoints[0][10]) if norm_hand_points is not None: self.image = cv2.drawKeypoints( self.image, norm_hand_points, outImage=np.array([]), color=(0, 255, 255), flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS) adjacent_keypoints = self.get_adjacent_keypoints(norm_hand_points) self.image = cv2.polylines(self.image, adjacent_keypoints, isClosed=False, color=(0, 255, 255), thickness=2) self.blit_cam_frame(self.image, self.window) frames = self.myfont.render(str(int(self.clock.get_fps())) + " FPS", True, pygame.Color('green')) self.window.blit(frames, (self.WIDTH - 100, 20))
class PeopleDetector(Thread, FrameHandler): """ An interface describing an object that can handle the result of the people detection process """ __pkg: object __labels: list __interpreter: Interpreter __height: float __width: float __alive: bool __head: HeadController __detection_handler: Optional[PeopleDetectionHandler] __tracker: CentroidTracker __led_controller: LedController __detection_state: DetectionState # Keep a reference ot the state def __init__(self): super().__init__() self.__alive = True self.__pkg = importlib.util.find_spec("tflite_runtime") with open(PATH_TO_LABELS, "r") as f: self.__labels = [line.strip() for line in f.readlines()] if self.__labels[0] == '???': del (self.__labels[0]) # Then load tensorflow lite model self.__interpreter = Interpreter(model_path=PATH_TO_CKPT) self.__interpreter.allocate_tensors() self.__head = HeadController() self.__head.start() self.__tracker = CentroidTracker() self.__detection_handler = None self.__led_controller = LedController() self.__detection_state = PersonNotPresentState(self.__detection_handler) def run(self) -> None: super().run() input_details = self.__interpreter.get_input_details() output_details = self.__interpreter.get_output_details() height = input_details[0]['shape'][1] width = input_details[0]['shape'][2] floating_model = (input_details[0]['dtype'] == np.float32) input_mean = 127.5 input_std = 127.5 counter = {} gone_left = False while self.__alive: ret, image = self.get_next_frame() if ret and image is not None: # image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) imH, imW, _ = image.shape image_resized = cv2.resize(image, (width, height)) input_data = np.expand_dims(image_resized, axis=0) if floating_model: input_data = (np.float32(input_data) - input_mean) / input_std self.__interpreter.set_tensor(input_details[0]['index'], input_data) self.__interpreter.invoke() boxes = self.__interpreter.get_tensor(output_details[0]['index'])[0] classes = self.__interpreter.get_tensor(output_details[1]['index'])[0] scores = self.__interpreter.get_tensor(output_details[2]['index'])[0] frame = image frame_w = frame.shape[1] frame_h = frame.shape[0] r = [] track_id = 0 for i in range(len(scores)): object_name = self.__labels[int(classes[i])] if (object_name == "person" and scores[i] > min_conf_threshold) and (scores[i] <= 1.0): rects = [] ymin = int(max(1, (boxes[i][0] * imH))) rects.append(ymin) xmin = int(max(1, (boxes[i][1] * imW))) rects.append(xmin) ymax = int(min(imH, (boxes[i][2] * imH))) rects.append(ymax) xmax = int(min(imW, (boxes[i][3] * imW))) rects.append(xmax) rects = [xmin, ymin, xmax, ymax] val = np.array(rects) r.append(val.astype("int")) # ---------------------------------Head random rotation, rotate to 90 if any people detected---------------------------------# left = randint(0, 45) right = randint(135, 180) if len(r) > 0: self.__head.rotate(90) else: self.__head.rotate(right if gone_left else left) # Here we tell the detection state that no person is present self.__detection_state = self.__detection_state.on_detection_result(False) counter = {} # And then toggle the rotation gone_left = not gone_left # --------------------------------- Choose an ID, Check if present for atleast 10 frames ---------------------------------# objects = self.__tracker.update(r) flag = 0 next_id = 0 i = 0 new_coord = [] next_coord = [] coord = [] for (objectID, centroid) in objects.items(): if objectID == track_id: flag = 1 new_coord = centroid if i == 0: next_id = objectID next_coord = centroid i += 1 if objectID in counter: counter[objectID] += 1 else: counter[objectID] = 0 if len(objects.items()) > 0: if flag == 0: track_id = next_id coord = next_coord else: coord = new_coord # --------------------------------- Control LED till 10 frames ---------------------------------# if len(coord) > 0: # If a person exists compute the index of the led to turn on x_pos = coord[0] led_index = math.floor((x_pos * 3.0) / frame_w) animation: LedAnimation = LedAnimation.ANIM_EYE_0 if led_index == 0: animation = LedAnimation.ANIM_EYE_2 if led_index == 1: animation = LedAnimation.ANIM_EYE_1 if led_index == 2: animation = LedAnimation.ANIM_EYE_0 # Ask the led manager to play the animation #self.__led_controller.play_animation(animation) # --------------------------------- Set the handler as true if a person presemt for more than 20 frames ---------------------------------# if counter[track_id] > 20: self.__detection_state = self.__detection_state.on_detection_result(True) cv2.imshow("Detector", frame) if cv2.waitKey(1) == ord('q'): break def stop(self) -> None: self.__alive = False def set_detection_handler(self, detection_handler: PeopleDetectionHandler): self.__detection_handler = detection_handler self.__detection_state.set_detection_handler(detection_handler)
'quantization_parameters': {'quantized_dimension': 0, 'scales': array([], dtype=float32), 'zero_points': array([], dtype=int32)}, 'shape': array([ 1, 60, 60, 3], dtype=int32), 'shape_signature': array([ 1, 60, 60, 3], dtype=int32), 'sparsity_parameters': {}}] """ pprint(input_blob) pprint(output_blob) img = cv2.imread("test.jpg") img = cv2.resize(img, (60, 60)) img = img.astype(np.float32) img = img[np.newaxis, :, :, :] interpreter.set_tensor(input_blob[0]['index'], img) interpreter.invoke() out0 = interpreter.get_tensor(output_blob[0]['index']) out1 = interpreter.get_tensor(output_blob[1]['index']) out2 = interpreter.get_tensor(output_blob[2]['index']) """ [{'dtype': <class 'numpy.float32'>, 'index': 85, 'name': 'Identity', 'quantization': (0.0, 0), 'quantization_parameters': {'quantized_dimension': 0, 'scales': array([], dtype=float32), 'zero_points': array([], dtype=int32)}, 'shape': array([1, 1], dtype=int32),
def TFlite(frameCount): # grab global references to the output frame, and # lock variables global outputFrame, lock, _reset ct = CentroidTracker() #Funtion Select cam_select, cam_rtsp, min_conf_threshold, notify, detect_list, resW, resH = select_option( ) # warmup if cam_select == 0: vs = cv2.VideoCapture(0) else: vs = cv2.VideoCapture(cam_rtsp) imW, imH = int(resW), int(resH) vs.set(3, imW) vs.set(4, imH) time.sleep(2.0) ################## args = argsparser() MODEL_NAME = args.modeldir GRAPH_NAME = args.graph LABELMAP_NAME = args.labelsTF ####################### # min_conf_threshold = args.threshold ####################### CWD_PATH = os.getcwd() # Path to .tflite file, which contains the model that is used for object detection PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME) # Path to label map file PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME) # Load the label map with open(PATH_TO_LABELS, 'r') as f: labels = [line.strip() for line in f.readlines()] if labels[0] == '???': del (labels[0]) # Load the Tensorflow Lite model and get details interpreter = Interpreter(model_path=PATH_TO_CKPT) interpreter.allocate_tensors() input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() height = input_details[0]['shape'][1] width = input_details[0]['shape'][2] floating_model = (input_details[0]['dtype'] == np.float32) input_mean = 127.5 input_std = 127.5 # Initialize frame rate calculation frame_rate_calc = 1 freq = cv2.getTickFrequency() font = cv2.FONT_HERSHEY_SIMPLEX ########################################## t1_image = time.time() t2_image = time.time() objects_first = 0 ########################################## log.info("Detect On") while True: t1 = cv2.getTickCount() ret, frame_q = vs.read() frame_resized = cv2.resize(frame_q, (width, height)) input_data = np.expand_dims(frame_resized, axis=0) # Normalize pixel values if using a floating model (i.e. if model is non-quantized) if floating_model: input_data = (np.float32(input_data) - input_mean) / input_std # Perform the actual detection by running the model with the image as input interpreter.set_tensor(input_details[0]['index'], input_data) interpreter.invoke() # Retrieve detection results boxes = interpreter.get_tensor(output_details[0]['index'])[ 0] # Bounding box coordinates of detected objects classes = interpreter.get_tensor( output_details[1]['index'])[0] # Class index of detected objects scores = interpreter.get_tensor( output_details[2]['index'])[0] # Confidence of detected objects #num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed) ckname = '' object_name = None rects = [] # Loop over all detections and draw detection box if confidence is above minimum threshold for i in range(len(scores)): if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)): # Draw label object_name = labels[int(classes[i])] if object_name in detect_list: ckname = ckname + object_name + ',' ymin = int(max(1, (boxes[i][0] * imH))) xmin = int(max(1, (boxes[i][1] * imW))) ymax = int(min(imH, (boxes[i][2] * imH))) xmax = int(min(imW, (boxes[i][3] * imW))) cv2.rectangle(frame_q, (xmin, ymin), (xmax, ymax), (10, 255, 0), 2) label = '%s: %d%%' % (object_name, int(scores[i] * 100)) #print(label) labelSize, baseLine = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) label_ymin = max(ymin, labelSize[1] + 10) color = color_name(object_name) cv2.rectangle( frame_q, (xmin, label_ymin - labelSize[1] - 10), (xmin + labelSize[0] + 45, label_ymin + baseLine - 10), color, cv2.FILLED) # Draw white box to put label text in cv2.putText(frame_q, label, (xmin + 45, label_ymin - 7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) x = np.array([xmin, ymin, xmax, ymax]) # print(x) rects.append(x.astype("int")) cv2.putText(frame_q, "FPS: {0:.2f}".format(frame_rate_calc), (30, 50), font, 1, (255, 255, 0), 2, cv2.LINE_AA) #Funtion display_show t1_image = time.time() frame_q, objects_first, t2_image = display_show( rects, t1_image, t2_image, ckname, frame_q, objects_first, notify, ct) with lock: outputFrame = frame_q.copy() if _reset == 1: sys.exit() t2 = cv2.getTickCount() time1 = (t2 - t1) / freq frame_rate_calc = 1 / time1 vs.stop()
frame = frame_receiver.receive_array() print(frame.shape) if frame.shape == np.array([999888]).shape: print('End of service.') frame_receiver.endServer() response_sender.endServer() print('All server closed.') frame_receiver = SocketNumpyArray() response_sender = SocketNumpyArray() frame_receiver.initalize_receiver(9999) response_sender.initialize_sender('localhost', 8848) response_sender.send_numpy_array([height, width]) continue # Perform the actual detection by running the model with the image as input interpreter.set_tensor(input_details[0]['index'], frame) interpreter.invoke() # Retrieve detection results boxes = interpreter.get_tensor(output_details[0]['index'])[ 0] # Bounding box coordinates of detected objects classes = interpreter.get_tensor( output_details[1]['index'])[0] # Class index of detected objects scores = interpreter.get_tensor( output_details[2]['index'])[0] # Confidence of detected objects print(boxes.shape, classes.shape, scores.shape) response_sender.send_numpy_array([boxes, classes, scores]) # response_sender.send_numpy_array( classes ) # response_sender.send_numpy_array( scores )
with open("../example.png", "rb") as image_file: encoded_string = b64encode(image_file.read()) image_binary = pil_open(BytesIO(b64decode(encoded_string))) image_binary = image_binary.resize([96, 96]) full_image_np = asarray(image_binary, dtype='float32')/255. arr = [] BATCH_SIZE = 1 arr = [] for i in range(BATCH_SIZE): arr.append(full_image_np) full_image_np = asarray(arr, dtype='float32') start_time = int(round(time.time() * 1000)) for i in range(16384//BATCH_SIZE): INTERPRETER.set_tensor(INPUT_DETAILS[0]['index'], full_image_np) INTERPRETER.invoke() output_data = [] for od in OUTPUT_DETAILS: for tensor in INTERPRETER.get_tensor(od['index']).tolist(): output_data.append(tensor) print(f'{(int(round(time.time() * 1000))-start_time)/1000.0} seconds')
input_details = interpreter.get_input_details()[0]['index'] deeplabv3_predictions = interpreter.get_output_details()[0]['index'] color_image = cv2.imread("/home/pi/03_integer_quantization/soren_images/2020_10_02_10_58_15/color.png") # path to color image color_image = cv2.resize(color_image, (image_width, image_height)) cv2.imshow("Input color image",color_image) prepimg_deep = cv2.resize(color_image, (image_width, image_height)) prepimg_deep = cv2.cvtColor(prepimg_deep, cv2.COLOR_BGR2RGB) prepimg_deep = np.expand_dims(prepimg_deep, axis=0) prepimg_deep = prepimg_deep.astype(np.uint8) # Run model - DeeplabV3-plus start_time = time.time() interpreter.set_tensor(input_details, prepimg_deep) interpreter.invoke() # Get results predictions = interpreter.get_tensor(deeplabv3_predictions)[0] # Segmentation outputimg = np.uint8(predictions) outputimg = cv2.resize(outputimg, (image_width, image_height)) outputimg = Image.fromarray(outputimg, mode="P") outputimg.putpalette(DEEPLAB_PALETTE) outputimg = outputimg.convert("RGB") outputimg = np.asarray(outputimg) outputimg = cv2.cvtColor(outputimg, cv2.COLOR_RGB2BGR)