Ejemplo n.º 1
0
img = cv2.imread('Capture.png')
h = img.shape[0]
w = img.shape[1]

img = cv2.resize(img, (256, 144))
img = np.asarray(img)
img = img / 255.
img = img.astype(np.float32)
img = img[np.newaxis, :, :, :]

# Tensorflow Lite
interpreter = Interpreter(model_path='model_float16_quant.tflite',
                          num_threads=4)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()[0]['index']
output_details = interpreter.get_output_details()[0]['index']

interpreter.set_tensor(input_details, img)
interpreter.invoke()
output = interpreter.get_tensor(output_details)

print(output.shape)
out1 = output[0][:, :, 0]
out2 = output[0][:, :, 1]

out1 = np.invert((out1 > 0.5) * 255)
out2 = np.invert((out2 > 0.5) * 255)

print('out1:', out1.shape)
print('out2:', out2.shape)
Ejemplo n.º 2
0
class TFInferenceEngine:
    """Thin wrapper around TFLite Interpreter.

    The official TFLite API is moving fast and still changes frequently.
    This class intends to abstract out underlying TF changes to some extend.

    It dynamically detects if EdgeTPU is available and uses it.
    Otherwise falls back to TFLite Runtime.
    """
    def __init__(self,
                 model=None,
                 labels=None,
                 confidence_threshold=0.8,
                 top_k=10):
        """Create an instance of Tensorflow inference engine.

        :Parameters:
        ----------
        model: dict
            {
                'tflite': path,
                'edgetpu': path,
            }
            Where path is of type string and points to the
            location of frozen graph file (AI model).
        labels : string
            Location of file with model labels.
        confidence_threshold : float
            Inference confidence threshold.
        top_k : type
            Inference top-k threshold.

        """
        assert model
        assert model['tflite'], 'TFLite AI model path required.'
        model_tflite = model['tflite']
        assert os.path.isfile(model_tflite), \
            'TFLite AI model file does not exist: {}' \
            .format(model_tflite)
        self._model_tflite_path = model_tflite
        model_edgetpu = model.get('edgetpu', None)
        if model_edgetpu:
            assert os.path.isfile(model_edgetpu), \
                'EdgeTPU AI model file does not exist: {}' \
                .format(model_edgetpu)
        self._model_edgetpu_path = model_edgetpu
        assert labels, 'AI model labels path required.'
        assert os.path.isfile(labels), \
            'AI model labels file does not exist: {}' \
            .format(labels)
        self._model_labels_path = labels
        self._confidence_threshold = confidence_threshold
        self._top_k = top_k
        log.debug(
            'Loading AI model:\n'
            'TFLite graph: %r\n'
            'EdgeTPU graph: %r\n'
            'Labels %r.'
            'Condidence threshod: %.0f%%'
            'top-k: %d', model_tflite, model_edgetpu, labels,
            confidence_threshold * 100, top_k)
        # EdgeTPU is not available in testing and other environments
        # load dynamically as needed
        #        edgetpu_class = 'DetectionEngine'
        #        module_object = import_module('edgetpu.detection.engine',
        #                                      packaage=edgetpu_class)
        #        target_class = getattr(module_object, edgetpu_class)
        self._tf_interpreter = _get_edgetpu_interpreter(model=model_edgetpu)
        if not self._tf_interpreter:
            log.debug('EdgeTPU not available. Will use TFLite CPU runtime.')
            self._tf_interpreter = Interpreter(model_path=model_tflite)
        assert self._tf_interpreter
        self._tf_interpreter.allocate_tensors()
        # check the type of the input tensor
        self._tf_input_details = self._tf_interpreter.get_input_details()
        self._tf_output_details = self._tf_interpreter.get_output_details()
        self._tf_is_quantized_model = \
            self.input_details[0]['dtype'] != np.float32

    @property
    def input_details(self):
        return self._tf_input_details

    @property
    def output_details(self):
        return self._tf_output_details

    @property
    def is_quantized(self):
        return self._tf_is_quantized_model

    @property
    def labels_path(self):
        """
        Location of labels file.

        :Returns:
        -------
        string
            Path to AI model labels.

        """
        return self._model_labels_path

    @property
    def confidence_threshold(self):
        """
        Inference confidence threshold.

        :Returns:
        -------
        float
            Confidence threshold for inference results.
            Only results at or above
            this threshold should be returned by each engine inference.

        """
        return self._confidence_threshold

    @property
    def top_k(self):
        """
        Inference top-k threshold.

        :Returns:
        -------
        int
            Max number of results to be returned by each inference.
            Ordered by confidence score.

        """
        return self._top_k

    def infer(self):
        """Invoke model inference on current input tensor."""
        return self._tf_interpreter.invoke()

    def set_tensor(self, index=None, tensor_data=None):
        """Set tensor data at given reference index."""
        assert isinstance(index, int)
        self._tf_interpreter.set_tensor(index, tensor_data)

    def get_tensor(self, index=None):
        """Return tensor data at given reference index."""
        assert isinstance(index, int)
        return self._tf_interpreter.get_tensor(index)
Ejemplo n.º 3
0
model_dir = './mobilenet-v1.1.0-128quant/'
model_name = 'mobilenet_v1_1.0_128_quant.tflite'

model_dir = download_model_zoo(model_dir, model_name)
tflite_model_file = os.path.join(model_dir, model_name)
tflite_model_buf = open(tflite_model_file, "rb").read()
try:
    import tflite
    tflite_model = tflite.Model.GetRootAsModel(tflite_model_buf, 0)
except AttributeError:
    import tflite.Model
    tflite_model = tflite.Model.Model.GetRootAsModel(tflite_model_buf, 0)

interpreter = Interpreter(tflite_model_file)
interpreter.allocate_tensors()

_, height, width, _ = interpreter.get_input_details()[0]['shape']
image = load_test_image('uint8', height, width)

numpy_time = np.zeros(repeat)

for i in range(0, repeat):
    start_time = time.time()
    results = classify_image(interpreter, image)

    elapsed_ms = (time.time() - start_time) * 1000
    numpy_time[i] = elapsed_ms

print("tflite %-20s %-19s (%s)" % (model_name, "%.2f ms" % np.mean(numpy_time),
                                   "%.2f ms" % np.std(numpy_time)))
class PoseEngine:
    """Engine used for pose tasks."""

    def __init__(self, model_path, mirror=False):
        """Creates a PoseEngine with given model.

        Args:
          model_path: String, path to TF-Lite Flatbuffer file.
          mirror: Flip keypoints horizontally

        Raises:
          ValueError: An error occurred when model output is invalid.
        """
        self._mirror = mirror

        edgetpu_delegate = load_delegate(EDGETPU_SHARED_LIB)
        posenet_decoder_delegate = load_delegate(POSENET_SHARED_LIB)
        self._interpreter = Interpreter(
            model_path, experimental_delegates=[edgetpu_delegate, posenet_decoder_delegate])
        self._interpreter.allocate_tensors()
        self._input_tensor_shape = self._interpreter.get_input_details()[0]['shape']
        self._input_details = self._interpreter.get_input_details()
        if (self._input_tensor_shape.size != 4 or
                self._input_tensor_shape[3] != 3 or
                self._input_tensor_shape[0] != 1):
            raise ValueError(
                ('Image model should have input shape [1, height, width, 3]!'
                 ' This model has {}.'.format(self._input_tensor_shape)))
        _, self.image_height, self.image_width, self.image_depth = self._input_tensor_shape


        # Auto-detect stride size
        def calcStride(h,w,L):
          return int((2*h*w)/(math.sqrt(h**2 + 4*h*L*w - 2*h*w + w**2) - h - w))

        details = self._interpreter.get_output_details()[4]
        self.heatmap_zero_point = details['quantization_parameters']['zero_points'][0]
        self.heatmap_scale = details['quantization_parameters']['scales'][0]
        heatmap_size = self._interpreter.tensor(details['index'])().nbytes
        self.stride = calcStride(self.image_height, self.image_width, heatmap_size)
        self.heatmap_size = (self.image_width // self.stride + 1, self.image_height // self.stride + 1)
        details = self._interpreter.get_output_details()[5]
        self.parts_zero_point = details['quantization_parameters']['zero_points'][0]
        self.parts_scale = details['quantization_parameters']['scales'][0]

        print("Heatmap size: ", self.heatmap_size)
        print("Stride: ", self.stride, self.heatmap_size)


    def DetectPosesInImage(self, img):
        """Detects poses in a given image.

           For ideal results make sure the image fed to this function is close to the
           expected input size - it is the caller's responsibility to resize the
           image accordingly.

        Args:
          img: numpy array containing image
        """

        # Extend or crop the input to match the input shape of the network.
        if img.shape[0] < self.image_height or img.shape[1] < self.image_width:
            pads = [[0, max(0, self.image_height - img.shape[0])],
                    [0, max(0, self.image_width - img.shape[1])], [0, 0]]
            img = np.pad(img, pads, mode='constant')
        img = img[0:self.image_height, 0:self.image_width]
        assert (img.shape == tuple(self._input_tensor_shape[1:]))

        # Run the inference (API expects the data to be flattened)
        inference_time, outputs = self.run_inference(img)
        poses = self._parse_poses(outputs)
        heatmap, bodyparts = self._parse_heatmaps(outputs)
        return inference_time, poses, heatmap, bodyparts

    def ParseOutputs(self, outputs):
        poses = self._parse_poses(outputs)
        heatmap, bodyparts = self._parse_heatmaps(outputs)
        return poses, heatmap, bodyparts

    def _parse_poses(self, outputs):
        keypoints = outputs[0].reshape(-1, len(KEYPOINTS), 2)
        keypoint_scores = outputs[1].reshape(-1, len(KEYPOINTS))
        pose_scores = outputs[2].flatten()
        nposes = int(outputs[3][0])

        # Convert the poses to a friendlier format of keypoints with associated
        # scores.
        poses = []
        for pose_i in range(nposes):
            keypoint_dict = {}
            for point_i, point in enumerate(keypoints[pose_i]):
                keypoint = Keypoint(KEYPOINTS[point_i], point,
                                    keypoint_scores[pose_i, point_i])
                if self._mirror: keypoint.yx[1] = self.image_width - keypoint.yx[1]
                keypoint_dict[KEYPOINTS[point_i]] = keypoint
            poses.append(Pose(keypoint_dict, pose_scores[pose_i]))

        return poses

    def softmax(self, y, axis):
        y = y - np.expand_dims(np.max(y, axis = axis), axis)
        y = np.exp(y)
        return y / np.expand_dims(np.sum(y, axis = axis), axis)

    def _parse_heatmaps(self, outputs):
        # Heatmaps are really float32.
        heatmap = (outputs[4].astype(np.float32) - self.heatmap_zero_point) * self.heatmap_scale
        heatmap = np.reshape(heatmap, [self.heatmap_size[1], self.heatmap_size[0]])
        part_heatmap = (outputs[5].astype(np.float32) - self.parts_zero_point) * self.parts_scale
        part_heatmap = np.reshape(part_heatmap, [self.heatmap_size[1], self.heatmap_size[0], -1])
        part_heatmap = self.softmax(part_heatmap, axis=2)
        return heatmap, part_heatmap

    def run_inference(self, input):
        start_time = time.monotonic()
        self._interpreter.set_tensor(self._input_details[0]['index'], np.expand_dims(input, axis=0))
        self._interpreter.invoke()
        duration_ms = (time.monotonic() - start_time) * 1000

        output = []
        for details in self._interpreter.get_output_details():
            tensor = self._interpreter.get_tensor(details['index'])
            output.append(tensor)

        return (duration_ms, output)
Ejemplo n.º 5
0
def setup_ssd_edgetpu(modelParas):
    # Get Args
    MODEL_NAME = modelParas[0]
    GRAPH_NAME = modelParas[1]
    LABELMAP_NAME = modelParas[2]
    min_conf_threshold = float(modelParas[3])
    resW, resH = modelParas[4:6]
    imW, imH = int(resW), int(resH)
    use_TPU = modelParas[6]

    # Import TensorFlow libraries
    # If tensorflow is not installed, import interpreter from tflite_runtime, else import from regular tensorflow
    # If using Coral Edge TPU, import the load_delegate library

    # If using Edge TPU, assign filename for Edge TPU model
    if use_TPU:
        # If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
        if (GRAPH_NAME == 'detect.tflite'):
            GRAPH_NAME = 'edgetpu.tflite'

    # Get path to current working directory
    CWD_PATH = os.getcwd()

    # Path to .tflite file, which contains the model that is used for object detection
    PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)

    # Path to label map file
    PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)

    # Load the label map
    with open(PATH_TO_LABELS, 'r') as f:
        labels = [line.strip() for line in f.readlines()]

    # Have to do a weird fix for label map if using the COCO "starter model" from
    # https://www.tensorflow.org/lite/models/object_detection/overview
    # First label is '???', which has to be removed.
    if labels[0] == '???':
        del (labels[0])

    # Load the Tensorflow Lite model.
    # If using Edge TPU, use special load_delegate argument
    if use_TPU:
        interpreter = Interpreter(
            model_path=PATH_TO_CKPT,
            experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
        print(PATH_TO_CKPT)
    else:
        interpreter = Interpreter(model_path=PATH_TO_CKPT)

    interpreter.allocate_tensors()

    # Get model details
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()
    height = input_details[0]['shape'][1]
    width = input_details[0]['shape'][2]

    floating_model = (input_details[0]['dtype'] == np.float32)

    input_mean = 127.5
    input_std = 127.5

    tfParas = [
        height, width, floating_model, labels, input_mean, input_std,
        input_details, min_conf_threshold, imH, imW, interpreter,
        output_details
    ]

    return tfParas
class PoseEngine:
    """Engine used for pose tasks."""
    def __init__(self,
                 model_path,
                 mirror=False,
                 offsetRefineStep=2,
                 scoreThreshold=0.8,
                 maxPoseDetections=5,
                 nmsRadius=30,
                 minPoseConfidence=0.15):
        """Creates a PoseEngine with given model.

        Args:
          model_path: String, path to TF-Lite Flatbuffer file.
          mirror: Flip keypoints horizontally

        Raises:
          ValueError: An error occurred when model output is invalid.
        """
        self.interpreter = Interpreter(model_path)
        self.interpreter.allocate_tensors()

        self._mirror = mirror

        self._input_tensor_shape = self.get_input_tensor_shape()
        if (self._input_tensor_shape.size != 4
                or self._input_tensor_shape[3] != 3
                or self._input_tensor_shape[0] != 1):
            raise ValueError(
                ('Image model should have input shape [1, height, width, 3]!'
                 ' This model has {}.'.format(self._input_tensor_shape)))
        _, self.image_height, self.image_width, self.image_depth = self.get_input_tensor_shape(
        )

        self.heatmaps_nx = self.interpreter.get_output_details()[0]['shape'][2]
        self.heatmaps_ny = self.interpreter.get_output_details()[0]['shape'][1]
        self.heatmaps_stride_x = self.getStride(self.image_width,
                                                self.heatmaps_nx)
        self.heatmaps_stride_y = self.getStride(self.image_height,
                                                self.heatmaps_ny)
        self.quant_heatmaps_r, self.quant_heatmaps_off = self.interpreter.get_output_details(
        )[0]['quantization']
        self.quant_offsets_short_r, self.quant_offsets_short_off = self.interpreter.get_output_details(
        )[1]['quantization']
        self.quant_offsets_mid_r, self.quant_offsets_mid_off = self.interpreter.get_output_details(
        )[2]['quantization']

        self.offsetRefineStep = offsetRefineStep
        self.scoreThreshold = scoreThreshold
        self.maxPoseDetections = maxPoseDetections
        self.nmsRadius = nmsRadius
        self.sqRadius = self.nmsRadius * self.nmsRadius
        self.minPoseConfidence = minPoseConfidence

        # The API returns all the output tensors flattened and concatenated. We
        # have to figure out the boundaries from the tensor shapes & sizes.
        offset = 0
        self._output_offsets = [0]
        for size in self.get_all_output_tensors_sizes():
            offset += size
            self._output_offsets.append(offset)

    def getStride(self, l, n):
        strides = (8, 16, 32)
        return strides[np.argmin(np.abs(strides - l / n))]

    def get_input_tensor_shape(self):
        return self.interpreter.get_input_details()[0]['shape']

    def get_all_output_tensors_sizes(self):
        sizes = np.array([], dtype='int32')
        for d in self.interpreter.get_output_details():
            s = np.squeeze(self.interpreter.get_tensor(
                d['index'])).flatten().size
            sizes = np.append(sizes, int(s))
        return sizes

    def DetectPosesInImage(self, img):
        """Detects poses in a given image.

           For ideal results make sure the image fed to this function is close to the
           expected input size - it is the caller's responsibility to resize the
           image accordingly.

        Args:
          img: numpy array containing image
        """

        # Extend or crop the input to match the input shape of the network.
        if img.shape[0] < self.image_height or img.shape[1] < self.image_width:
            img = np.pad(
                img, [[0, max(0, self.image_height - img.shape[0])],
                      [0, max(0, self.image_width - img.shape[1])], [0, 0]],
                mode='constant')
        img = img[0:self.image_height, 0:self.image_width]
        assert (img.shape == tuple(self._input_tensor_shape[1:]))

        # Run the inference (API expects the data to be flattened)
        return self.ParseOutput(self.run_inference(img))

    def run_inference(self, img):
        if img.shape[0] < self.image_height or img.shape[1] < self.image_width:
            img = np.pad(
                img, [[0, max(0, self.image_height - img.shape[0])],
                      [0, max(0, self.image_width - img.shape[1])], [0, 0]],
                mode='constant')
        img = img[0:self.image_height, 0:self.image_width]
        assert (img.shape == tuple(self._input_tensor_shape[1:]))

        tensor_index = self.interpreter.get_input_details()[0]['index']
        input_tensor = self.interpreter.tensor(tensor_index)
        input_tensor()[:, :, :, :] = img
        start_time = time.monotonic()
        self.interpreter.invoke()
        elapsed_ms = (time.monotonic() - start_time) * 1000
        out = np.empty(0)
        for d in self.interpreter.get_output_details():
            o = np.squeeze(self.interpreter.get_tensor(d['index'])).flatten()
            out = np.append(out, o)
        return (elapsed_ms, out)

    def logistic(self, x):
        return 1 / (1 + np.exp(-x))

    def isPeak(self, heatmaps_flat, index):
        maxindex = index // len(KEYPOINTS)
        maxkeypoint = index % len(KEYPOINTS)

        y_index = maxindex // self.heatmaps_nx
        x_index = maxindex % self.heatmaps_nx

        y_index_min = np.max((y_index - 1, 0))
        y_index_max = np.min((y_index + 1, self.heatmaps_ny - 1))
        x_index_min = np.max((x_index - 1, 0))
        x_index_max = np.min((x_index + 1, self.heatmaps_nx - 1))

        for y_current in range(y_index_min, y_index_max + 1):
            for x_current in range(x_index_min, x_index_max + 1):
                index_current = len(KEYPOINTS) * (
                    y_current * self.heatmaps_nx + x_current) + maxkeypoint
                if (heatmaps_flat[index_current] >
                        heatmaps_flat[index]) and (index_current != index):
                    return False
        return True

    def ParseOutput(self, output):
        inference_time, output = output
        outputs = [
            output[i:j]
            for i, j in zip(self._output_offsets, self._output_offsets[1:])
        ]

        heatmaps = outputs[0].reshape(-1, len(KEYPOINTS))
        offsets_short_y = outputs[1].reshape(
            -1, 2 * len(KEYPOINTS))[:, 0:len(KEYPOINTS)]
        offsets_short_x = outputs[1].reshape(
            -1, 2 * len(KEYPOINTS))[:, len(KEYPOINTS):2 * len(KEYPOINTS)]
        offsets_mid_fwd_y = outputs[2].reshape(
            -1, 4 * len(poseChain))[:, 0:len(poseChain)]
        offsets_mid_fwd_x = outputs[2].reshape(
            -1, 4 * len(poseChain))[:, len(poseChain):2 * len(poseChain)]
        offsets_mid_bwd_y = outputs[2].reshape(
            -1, 4 * len(poseChain))[:, 2 * len(poseChain):3 * len(poseChain)]
        offsets_mid_bwd_x = outputs[2].reshape(
            -1, 4 * len(poseChain))[:, 3 * len(poseChain):4 * len(poseChain)]
        heatmaps = self.logistic(
            (heatmaps - self.quant_heatmaps_off) * self.quant_heatmaps_r)
        heatmaps_flat = heatmaps.flatten()
        offsets_short_y = (offsets_short_y - self.quant_offsets_short_off
                           ) * self.quant_offsets_short_r
        offsets_short_x = (offsets_short_x - self.quant_offsets_short_off
                           ) * self.quant_offsets_short_r
        offsets_mid_fwd_y = (offsets_mid_fwd_y - self.quant_offsets_mid_off
                             ) * self.quant_offsets_mid_r
        offsets_mid_fwd_x = (offsets_mid_fwd_x - self.quant_offsets_mid_off
                             ) * self.quant_offsets_mid_r
        offsets_mid_bwd_y = (offsets_mid_bwd_y - self.quant_offsets_mid_off
                             ) * self.quant_offsets_mid_r
        offsets_mid_bwd_x = (offsets_mid_bwd_x - self.quant_offsets_mid_off
                             ) * self.quant_offsets_mid_r

        # Obtaining the peaks of heatmaps larger than scoreThreshold
        orderedindices = np.argsort(heatmaps_flat)[::-1]
        largeheatmaps_indices = np.empty(0, dtype='int32')
        for i in range(len(orderedindices)):
            if heatmaps_flat[orderedindices[i]] < self.scoreThreshold:
                break
            if self.isPeak(heatmaps_flat, orderedindices[i]):
                largeheatmaps_indices = np.append(largeheatmaps_indices,
                                                  orderedindices[i])

        pose_list = np.full(self.maxPoseDetections * 2 * len(KEYPOINTS),
                            0.0,
                            dtype='float32').reshape(-1, len(KEYPOINTS), 2)
        maxindex_list = np.full(self.maxPoseDetections * len(KEYPOINTS),
                                -1,
                                dtype='int32').reshape(-1, len(KEYPOINTS))
        score_list = np.full(self.maxPoseDetections * len(KEYPOINTS),
                             0.0,
                             dtype='float32').reshape(-1, len(KEYPOINTS))
        pose_score_list = np.full(self.maxPoseDetections, 0.0, dtype='float32')

        nPoses = 0
        # obtaining at most maxPoseDetections poses
        for point in range(len(largeheatmaps_indices)):
            if nPoses >= self.maxPoseDetections:
                break

            # obtain a root canidate
            maxindex = largeheatmaps_indices[point] // len(KEYPOINTS)
            maxkeypoint = largeheatmaps_indices[point] % len(KEYPOINTS)
            y = self.heatmaps_stride_y * (maxindex // self.heatmaps_nx)
            x = self.heatmaps_stride_x * (maxindex % self.heatmaps_nx)
            y += offsets_short_y[maxindex, maxkeypoint]
            x += offsets_short_x[maxindex, maxkeypoint]

            # skip keypoint with (x, y) that is close to the existing keypoints
            skip = 0
            for p in range(nPoses):
                y_exist = pose_list[p, maxkeypoint, 0]
                x_exist = pose_list[p, maxkeypoint, 1]
                if (y_exist - y) * (y_exist - y) + (x_exist - x) * (
                        x_exist - x) < self.sqRadius:
                    skip = 1
                    break
            if skip == 1:
                continue

            # setting the maxkeypoint as root
            pose_list[nPoses, maxkeypoint, 0] = y
            pose_list[nPoses, maxkeypoint, 1] = x
            maxindex_list[nPoses, maxkeypoint] = maxindex
            score_list[nPoses, maxkeypoint] = heatmaps[maxindex, maxkeypoint]

            # backward decoding
            for edge in reversed(range(len(poseChain))):
                sourceKeypointId = parentToChildEdges[edge]
                targetKeypointId = childToParentEdges[edge]
                if maxindex_list[nPoses,
                                 sourceKeypointId] != -1 and maxindex_list[
                                     nPoses, targetKeypointId] == -1:
                    maxindex = maxindex_list[nPoses, sourceKeypointId]
                    y = pose_list[nPoses, sourceKeypointId, 0]
                    x = pose_list[nPoses, sourceKeypointId, 1]
                    y += offsets_mid_bwd_y[maxindex, edge]
                    x += offsets_mid_bwd_x[maxindex, edge]

                    y_index = np.clip(round(y / self.heatmaps_stride_y), 0,
                                      self.heatmaps_ny - 1)
                    x_index = np.clip(round(x / self.heatmaps_stride_x), 0,
                                      self.heatmaps_nx - 1)
                    maxindex_list[
                        nPoses,
                        targetKeypointId] = self.heatmaps_nx * y_index + x_index
                    for i in range(self.offsetRefineStep):
                        y_index = np.clip(round(y / self.heatmaps_stride_y), 0,
                                          self.heatmaps_ny - 1)
                        x_index = np.clip(round(x / self.heatmaps_stride_x), 0,
                                          self.heatmaps_nx - 1)
                        maxindex_list[
                            nPoses,
                            targetKeypointId] = self.heatmaps_nx * y_index + x_index
                        y = self.heatmaps_stride_y * y_index
                        x = self.heatmaps_stride_x * x_index
                        y += offsets_short_y[maxindex_list[nPoses,
                                                           targetKeypointId],
                                             targetKeypointId]
                        x += offsets_short_x[maxindex_list[nPoses,
                                                           targetKeypointId],
                                             targetKeypointId]

                    pose_list[nPoses, targetKeypointId, 0] = y
                    pose_list[nPoses, targetKeypointId, 1] = x
                    score_list[nPoses, targetKeypointId] = heatmaps[
                        maxindex_list[nPoses,
                                      targetKeypointId], targetKeypointId]

            # forward decoding
            for edge in range(len(poseChain)):
                sourceKeypointId = childToParentEdges[edge]
                targetKeypointId = parentToChildEdges[edge]
                if maxindex_list[nPoses,
                                 sourceKeypointId] != -1 and maxindex_list[
                                     nPoses, targetKeypointId] == -1:
                    maxindex = maxindex_list[nPoses, sourceKeypointId]
                    y = pose_list[nPoses, sourceKeypointId, 0]
                    x = pose_list[nPoses, sourceKeypointId, 1]
                    y += offsets_mid_fwd_y[maxindex, edge]
                    x += offsets_mid_fwd_x[maxindex, edge]

                    y_index = np.clip(round(y / self.heatmaps_stride_y), 0,
                                      self.heatmaps_ny - 1)
                    x_index = np.clip(round(x / self.heatmaps_stride_x), 0,
                                      self.heatmaps_nx - 1)
                    maxindex_list[
                        nPoses,
                        targetKeypointId] = self.heatmaps_nx * y_index + x_index
                    for i in range(self.offsetRefineStep):
                        y_index = np.clip(round(y / self.heatmaps_stride_y), 0,
                                          self.heatmaps_ny - 1)
                        x_index = np.clip(round(x / self.heatmaps_stride_x), 0,
                                          self.heatmaps_nx - 1)
                        maxindex_list[
                            nPoses,
                            targetKeypointId] = self.heatmaps_nx * y_index + x_index
                        y = self.heatmaps_stride_y * y_index
                        x = self.heatmaps_stride_x * x_index
                        y += offsets_short_y[maxindex_list[nPoses,
                                                           targetKeypointId],
                                             targetKeypointId]
                        x += offsets_short_x[maxindex_list[nPoses,
                                                           targetKeypointId],
                                             targetKeypointId]

                    pose_list[nPoses, targetKeypointId, 0] = y
                    pose_list[nPoses, targetKeypointId, 1] = x
                    score_list[nPoses, targetKeypointId] = heatmaps[
                        maxindex_list[nPoses,
                                      targetKeypointId], targetKeypointId]

            # calclate pose score
            score = 0
            for k in range(len(KEYPOINTS)):
                y = pose_list[nPoses, k, 0]
                x = pose_list[nPoses, k, 1]
                closekeypoint_exists = False
                for p in range(nPoses):
                    y_exist = pose_list[p, k, 0]
                    x_exist = pose_list[p, k, 1]
                    if (y_exist - y) * (y_exist - y) + (x_exist - x) * (
                            x_exist - x) < self.sqRadius:
                        closekeypoint_exists = True
                        break
                if not closekeypoint_exists:
                    score += score_list[nPoses, k]
            score /= len(KEYPOINTS)

            if score > self.minPoseConfidence:
                pose_score_list[nPoses] = score
                nPoses += 1
            else:
                for k in range(len(KEYPOINTS)):
                    maxindex_list[nPoses, k] = -1

        # Convert the poses to a friendlier format of keypoints with associated
        # scores.
        poses = []
        for pose_i in range(nPoses):
            keypoint_dict = {}
            for point_i, point in enumerate(pose_list[pose_i]):
                keypoint = Keypoint(KEYPOINTS[point_i], point,
                                    score_list[pose_i, point_i])
                if self._mirror:
                    keypoint.yx[1] = self.image_width - keypoint.yx[1]
                keypoint_dict[KEYPOINTS[point_i]] = keypoint
            poses.append(Pose(keypoint_dict, pose_score_list[pose_i]))

        return poses, inference_time
Ejemplo n.º 7
0
class SsdMobileNet:
    def __init__(self, frozenGraphFilename, device='cpu'):
        self.boxes = None
        self.scores = None
        self.classes = None
        self.num_detections = None
        #self.interpreter = tf.lite.Interpreter(frozenGraphFilename)
        self.interpreter = Interpreter(frozenGraphFilename)
        self.__load_graph(device)
        self.__init_predictor()

    def __load_graph(self, device):

        # TFLITE INTERPRETER CON.
        #tf.logging.set_verbosity(tf.logging.DEBUG)
        self.interpreter.allocate_tensors()

    def __init_predictor(self):
        # obtaining the input-output shapes and types
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()

    def predict(self,
                images,
                params,
                thresh=0,
                batch=1,
                warmup=0,
                iterations=1):
        # Warmup
        x_matrix = np.array(images)
        if params[const.PRECISION] == const.FP32:
            x_matrix = np.array(images, dtype=np.float32)

        self.interpreter.set_tensor(self.input_details[0]['index'], x_matrix)
        for i in range(warmup):
            self.interpreter.invoke()
            matrix_0 = self.interpreter.get_tensor(
                self.output_details[0]['index'])
            matrix_1 = self.interpreter.get_tensor(
                self.output_details[1]['index'])
            matrix_2 = self.interpreter.get_tensor(
                self.output_details[2]['index'])
            matrix_3 = self.interpreter.get_tensor(
                self.output_details[3]['index'])

        # Measure
        times = []
        for i in range(iterations):
            t0 = time.time()
            self.interpreter.invoke()
            boxes = self.interpreter.get_tensor(
                self.output_details[0]['index'])
            classes = self.interpreter.get_tensor(
                self.output_details[1]['index'])
            scores = self.interpreter.get_tensor(
                self.output_details[2]['index'])
            num_detections = self.interpreter.get_tensor(
                self.output_details[3]['index'])

            t1 = time.time()
            ts = t1 - t0
            times.append(ts)
        # Report
        results = {"seconds": times, "predictions": []}

        for i in range(len(num_detections)):
            thisResult = []
            for d in range(int(num_detections[0])):
                # Note the weird bbox coords: y1, x1, y2, x2 !!
                box = boxes[i][d].tolist()
                x = {
                    'score': scores[i][d].tolist(),
                    'box': [box[1], box[0], box[3], box[2]],
                    'class': (classes[i][d] + 1).tolist()
                }
                thisResult.append(x)
            results['predictions'].append(thisResult)
        return results

    def predict_runtime(self, images, params, max=5):
        x_matrix = np.array(images)
        if params[const.PRECISION] == const.FP32:
            x_matrix = np.array(images, dtype=np.float32)

        start = time.time()
        self.interpreter.invoke()
        boxes = self.interpreter.get_tensor(self.output_details[0]['index'])
        classes = self.interpreter.get_tensor(self.output_details[1]['index'])
        scores = self.interpreter.get_tensor(self.output_details[2]['index'])
        num_detections = self.interpreter.get_tensor(
            self.output_details[3]['index'])
        end = time.time() - start
        return end
Ejemplo n.º 8
0
def main():

  history = []
  state = ''
  previous_state = ''

  parser = argparse.ArgumentParser(
      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  parser.add_argument(
      '--model', help='File path of .tflite file.', default=os.path.join(dirname, 'model/model.tflite'))
  parser.add_argument(
      '--labels', help='File path of labels file.', default=os.path.join(dirname, 'model/labels.txt'))
  args = parser.parse_args()

  labels = load_labels(args.labels)

  interpreter = Interpreter(args.model)
  interpreter.allocate_tensors()
  _, height, width, _ = interpreter.get_input_details()[0]['shape']

  with picamera.PiCamera(resolution=(600, 600), framerate=30) as camera:
    camera.iso = 100
    camera.vflip = 1
    camera.hflip = 1
    camera.crop = (0.0, 0.3, 0.7, 0.5)
    #camera.exposure_mode = "sport"
    camera.exposure_compensation = 0
    camera.exposure_mode = 'auto'
    camera.start_preview()
    try:
      stream = io.BytesIO()
      for _ in camera.capture_continuous(
          stream, format='jpeg', use_video_port=True):
        stream.seek(0)
        image = Image.open(stream).convert('RGB').resize((width, height), Image.ANTIALIAS)
        start_time = time.time()
        results = classify_image(interpreter, image)
        elapsed_ms = (time.time() - start_time) * 1000
        label_id, prob = results[0]
        stream.seek(0)
        stream.truncate()

        prob = int(round(prob,2)*100)


        if prob > 70:
            history.append(labels[label_id])
            history = history[-5:]
            print(history)
            if history.count(history[0]) == len(history):
                state = history[0] # on considère que si 10 détections indiquent la meme info on peut s'y fier
                if previous_state != state:

                    if state == 'publicite':
                        print("on coupe le son")
                        os.system('/usr/local/bin/irsend SEND_ONCE TV KEY_MUTE -#2')
                        time.sleep(0.1)
                        os.system('/usr/local/bin/irsend SEND_ONCE TV KEY_VOLUMEDOWN -#2')
                    elif previous_state == 'publicite':
                        print("on remet le son")
                        os.system('/usr/local/bin/irsend SEND_ONCE TV KEY_MUTE -#2')
                        time.sleep(0.1)
                        os.system('/usr/local/bin/irsend SEND_ONCE TV KEY_VOLUMEUP -#2')

                    previous_state = state
         
        sys.stdout.flush()

        camera.annotate_text = '%s\nscore:%s%%' % (labels[label_id], prob)
    finally:
        GPIO.output(40, GPIO.LOW)
        print("fin")
        camera.stop_preview()
Ejemplo n.º 9
0
    return final_result


print("Loading classification interpreter")
classify_interpreter = Interpreter(
    model_path="converted_currency_model.tflite")
print("Loaded classification interpreter")
print("Loading watermark interpreter")
watermark_interpreter = Interpreter(
    model_path="converted_watermark_model.tflite")
print("Loaded watermark interpreter")
print("Loading UV interpreter")
uv_interpreter = Interpreter(model_path="converted_uv_model.tflite")
print("Loaded UV interpreter")

classify_input_details = classify_interpreter.get_input_details()
classify_output_details = classify_interpreter.get_output_details()
watermark_input_details = watermark_interpreter.get_input_details()
watermark_output_details = watermark_interpreter.get_output_details()
uv_input_details = uv_interpreter.get_input_details()
uv_output_details = uv_interpreter.get_output_details()

print("Allocating classification model tensors...")
classify_interpreter.allocate_tensors()
print("Allocated classification model tensors...")
print("Allocating watermark model tensors...")
watermark_interpreter.allocate_tensors()
print("Allocated watermark model tensors...")
print("Allocating UV model tensors...")
uv_interpreter.allocate_tensors()
print("Allocated UV model tensors...")
def gen_frames():
# Define VideoStream class to handle streaming of video from webcam in separate processing thread
# Source - Adrian Rosebrock, PyImageSearch: https://www.pyimagesearch.com/2015/12/28/increasing-raspberry-pi-fps-with-python-and-opencv/
    class VideoStream(object):
        """Camera object that controls video streaming from the Picamera"""
        def __init__(self,resolution=(640,480),framerate=30,target=None,args=()):
            global capture_image_limit
            capture_image_limit = 2000
            
            global file_save_id
            file_save_id =0
            
            # Initialize the PiCamera and the camera image stream
            self.stream = cv2.VideoCapture(0)
 
            #VideoStream Instance
            instance = VideoStream.__qualname__
            print('The class instance is: ',instance)
            #print('\nVIDEOSTREAM: locals() value inside class\n', locals())
            #print(dir(VideoStream))
 
            #Reload
            reloadClass = os.environ.get('reload')
            if reloadClass == 'True':
                print('Delete Self:')
                del self
                os.environ['reload'] = 'False'          
                        
            ret = self.stream.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
            ret = self.stream.set(3,resolution[0])
            ret = self.stream.set(4,resolution[1])
                
            # Read first frame from the stream
            (self.grabbed, self.frame) = self.stream.read()

            # Variable to control when the camera is stopped
            self.stopped = False            
            
        def __del__(self):
            print ("Object destroyed");   

        def start(self):
        # Start the thread that reads frames from the video stream
            Thread(target=self.update,args=()).start()
            
            return self

        def update(self):
            # Keep looping indefinitely until the thread is stopped
            while True:
                # If the camera is stopped, stop the thread
                if self.stopped:
                    # Close camera resources
                    self.stream.release()
                    return
                    
                # Otherwise, grab the next frame from the stream
                (self.grabbed, self.frame) = self.stream.read()

        def read(self):
        # Return the most recent frame
            this_instance = self
            return self.frame

        def stop(self):
        # Indicate that the camera and thread should be stopped
            self.stopped = True


    # Define and parse input arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('--modeldir', help='Folder the .tflite file is located in',
                        required=True)
    parser.add_argument('--graph', help='Name of the .tflite file, if different than detect.tflite',
                        default='detect.tflite')
    parser.add_argument('--labels', help='Name of the labelmap file, if different than labelmap.txt',
                        default='labelmap.txt')
    parser.add_argument('--threshold', help='Minimum confidence threshold for displaying detected objects',
                        default=0.5)
    parser.add_argument('--resolution', help='Desired webcam resolution in WxH. If the webcam does not support the resolution entered, errors may occur.',
                        default='1280x720')
    parser.add_argument('--edgetpu', help='Use Coral Edge TPU Accelerator to speed up detection',
                        action='store_true')

    args = parser.parse_args()
    

    MODEL_NAME = args.modeldir
    print('~~~~ Param Default Model Name: ' + str(MODEL_NAME))
    GRAPH_NAME = args.graph
    LABELMAP_NAME = args.labels
    min_conf_threshold = float(args.threshold)
    resW, resH = args.resolution.split('x')
    imW, imH = int(resW), int(resH)
    use_TPU = args.edgetpu


    # Import TensorFlow libraries
    # If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
    # If using Coral Edge TPU, import the load_delegate library
    pkg = importlib.util.find_spec('tflite_runtime')
    print('TPU Runtime' + str(pkg))

    if pkg:
        from tflite_runtime.interpreter import Interpreter 
        if use_TPU:
            from tflite_runtime.interpreter import load_delegate 
    else:
        from tensorflow.lite.python.interpreter import Interpreter 
        if use_TPU:
            from tensorflow.lite.python.interpreter import load_delegate 

    # If using Edge TPU, assign filename for Edge TPU model
    if use_TPU:
        # If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
        if (GRAPH_NAME == 'detect.tflite'):
            GRAPH_NAME = 'edgetpu.tflite'       

    # Get path to current working directory
    # Multi-Model
    # Demo90 /home/pi/SensorFusion/Demo90
    # Deer: /home/pi/SensorFusion/PreLoadedModels/Model01.Deer
    # Head: /home/pi/SensorFusion/PreLoadedModels/Model02.Head
    # Eyes: /home/pi/SensorFusion/PreLoadedModels/Model03.Eyes
    # Tree: /home/pi/SensorFusion/PreLoadedModels/Model04.Tree
    # check.id - cd /home/pi/SensorFusion/checkid

    CWD_PATH = os.getcwd()
    print("Default Path: "+ CWD_PATH)
     
    newModel = str(os.environ.get('run_model'))
    
    print("New Model Name: "+ newModel)
    
    if newModel == "Demo90":
        CWD_PATH = "/home/pi/SensorFusion/"+ newModel 
    elif newModel == 'Check.ID':
        CWD_PATH = "/home/pi/SensorFusion/checkid" 
    else:
        CWD_PATH = "/home/pi/SensorFusion/PreLoadedModels/"+ newModel 
            
        print("Current Model Path: "+ CWD_PATH)

    # Path to .tflite file, which contains the model that is used for object detection
    PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,GRAPH_NAME)

    # Path to label map file
    PATH_TO_LABELS = os.path.join(CWD_PATH,MODEL_NAME,LABELMAP_NAME)
    print("Current Path to Label Map: "+ PATH_TO_LABELS)

    # Load the label map
    with open(PATH_TO_LABELS, 'r') as f:
        labels = [line.strip() for line in f.readlines()]

    # Have to do a weird fix for label map if using the COCO "starter model" from
    # https://www.tensorflow.org/lite/models/object_detection/overview
    # First label is '???', which has to be removed.
    if labels[0] == '???':
        del(labels[0])

    # Load the Tensorflow Lite model.
    # If using Edge TPU, use special load_delegate argument
    
    #if video_camera_flag:#Using a Flag here - for future use
    if use_TPU:
        interpreter = Interpreter(model_path=PATH_TO_CKPT,
                                  experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
        print('TPU Detected' + PATH_TO_CKPT)
    else:
        interpreter = Interpreter(model_path=PATH_TO_CKPT)
        print('No TPU detected!'+ PATH_TO_CKPT)            

    interpreter.allocate_tensors()

    # Get model details
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()
    height = input_details[0]['shape'][1]
    width = input_details[0]['shape'][2]

    floating_model = (input_details[0]['dtype'] == np.float32)

    input_mean = 127.5
    input_std = 127.5

    # Initialize frame rate calculation
    frame_rate_calc = 1
    freq = cv2.getTickFrequency()

    # Initialize video stream
    videostream = VideoStream(resolution=(imW,imH),framerate=30).start()
    time.sleep(1)
    
    global img_counter
    img_counter = 0
  
    #for frame1 in camera.capture_continuous(rawCapture, format="bgr",use_video_port=True):
    try:
        while True:
        #while video_camera_flag:
            
            # Start timer (for calculating frame rate)
            t1 = cv2.getTickCount()

            # Grab frame from video stream
            frame1 = videostream.read()

            # Acquire frame and resize to expected shape [1xHxWx3]
            global frame
            frame = frame1.copy()
            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            frame_resized = cv2.resize(frame_rgb, (width, height))
            input_data = np.expand_dims(frame_resized, axis=0)
            
            # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
            if floating_model:
                input_data = (np.float32(input_data) - input_mean) / input_std

            # Perform the actual detection by running the model with the image as input
            interpreter.set_tensor(input_details[0]['index'],input_data)
            interpreter.invoke()

            # Retrieve detection results           
            person_found = False
            
            boxes = interpreter.get_tensor(output_details[0]['index'])[0] # Bounding box coordinates of detected objects
            classes = interpreter.get_tensor(output_details[1]['index'])[0] # Class index of detected objects
            scores = interpreter.get_tensor(output_details[2]['index'])[0] # Confidence of detected objects
            #num = interpreter.get_tensor(output_details[3]['index'])[0]  # Total number of detected objects (inaccurate and not needed)
            
            #Kill TensofFlow while Annotating
            
            kill_tensorFlow = os.environ.get('kill_tensorFlow')
            #print("TensofFlow Status: " + str(kill_tensorFlow))

            # Loop over all detections and draw detection box if confidence is above minimum threshold
            for i in range(len(scores)):
                if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
                    
                    # Get bounding box coordinates and draw box
                    # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
                    ymin = int(max(1,(boxes[i][0] * imH)))
                    xmin = int(max(1,(boxes[i][1] * imW)))
                    ymax = int(min(imH,(boxes[i][2] * imH)))
                    xmax = int(min(imW,(boxes[i][3] * imW)))
                    
                    #print("Kill TF Flag: "+ str(kill_tensorFlow))
                    if kill_tensorFlow != 'True':
                        try:
                            cv2.rectangle(frame, (xmin,ymin), (xmax,ymax), (10, 255, 0), 3)
                        except:
                            pass

                    # Draw label (object_name) and score (%)    
                    
                    object_name = labels[int(classes[i])] # Look up object name from "labels" array using class index 
                    #print(labels[int(classes[i])]+": "+str(i))  
                    
                    
                    if labels[int(classes[0])]== 'person':#NOTE - The bar is for one person only
                        #print('Person Found!')
                        person_found = True# used for bar below
                    
                    scores_flag = os.environ.get('scores_flag')
                    labels_flag = os.environ.get('labels_flag')
                    
                    #states
                    state_ = 11 #both on by default
                    if labels_flag == 'labels_off' and scores_flag == 'scores_off':
                        state_ = 0#00
                        label = object()
                    
                    if labels_flag == 'labels_on' and scores_flag == 'scores_on':                    
                        state_ = 11#11
                        label = '%s: %d%%' % (object_name.capitalize(), int(scores[i]*100)) # Example: 'person: 72%'
                    
                    if labels_flag == 'labels_off' and scores_flag == 'scores_on':
                        label = '%d%%' % (int(scores[i]*100)) # Example: '72%'
                        state_ = 1#01    
                        
                    if labels_flag == 'labels_on' and scores_flag == 'scores_off':
                        state_= 10 #10   
                        label = '%s: ' % (object_name.capitalize()) # Example: 'person: '
                    
                    #draw the labels, background score and box
                    if state_ != 0:
                        labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size                        
                        label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window                    
                        #cv2.rectangle(frame, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (237,237,237), cv2.FILLED) # Draw white box to put label text in                   
                        if kill_tensorFlow != 'True':
                            cv2.rectangle(frame, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (128,128,128), cv2.FILLED) # Draw gray box to put label text in                   
                            cv2.putText(frame, label, (xmin, label_ymin-7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) # Draw label text
                    else:
                        if kill_tensorFlow != 'True':
                            cv2.rectangle(frame, (xmin,ymin), (xmin,ymin), (237,237,237), cv2.FILLED) # Draw frame with no label OR score text !
                        
            # Draw framerate in corner of frame - use 'F' key to toggle on/off
            try:
                if fps_flag:
                    cv2.putText(frame,'FPS: {0:.2f}'.format(frame_rate_calc),(30,50),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,0),2,cv2.LINE_AA)
                else:
                    pass
            except:
                pass
            
            #If Capture Image Draw status text
            capture_flag = os.environ.get('cap_flag')    
            try:
                if capture_flag == "True":    
                    cv2.putText(frame,'Saving File: '+str(img_counter),(520,50),cv2.FONT_HERSHEY_SIMPLEX,0.6,(0,0,255),2)
                else:
                    pass
                
            except:
                pass
                
            # All the results have been drawn on the frame, so it's time to display it.
            #cv2.imshow('Object detector', frame) ## Commented for the FLASK API 
 
            #Module widgets.meter()
            if kill_tensorFlow != 'True':
                #window_name ='Object detector'                     
                top = int(scores[0]*100)
                color = (0,0,255)
                if person_found == True:
                     widgets.meter(frame,top)#module
            #End Module

            # Displaying the image - DO NOT USE!
            #cv2.imshow(window_name, image)  
          
            #SENSOR FUSION Flask VIDEO API 
            #Brute Force Motion JPEG, OpenCV defaults to capture raw images,
            #so we must encode it into JPEG in order to correctly display the
            #video stream - NOTE need to work on this cv2.imencode tobytes slows the apparent frame rate by about 50%, plus the UI takes some
            #See: https://www.pyimagesearch.com/2017/02/06/faster-video-file-fps-with-cv2-videocapture-and-opencv/
            
            ret, buffer = cv2.imencode('.jpg', frame)
            frame2 = buffer.tobytes() #the image that is saved           
           
            #Capture Images and save to Annotate Named subdirectory under ~/Pictures
            #capture_flag = os.environ.get('cap_flag')
            annotate_name = os.environ.get('annotate_name')           
                        
            
            if capture_flag == 'True':
                #Check limit
                try:
                    print("image limit: " + anno_images)                    
                    capture_image_limit = int(anno_images)
                except:
                    pass
      
            if capture_flag == 'True' and img_counter < capture_image_limit:
                #Create new or use existing directory
                path_to_directory = '../Pictures/' + annotate_name
                print("Saving to ", path_to_directory)
                try:
                    os.makedirs(path_to_directory)
                except FileExistsError:
                    #dir already exists, so overwrite existing (unless we datestamp)!
                    pass
                img_name="../Pictures/"+annotate_name+"/"+annotate_name+"sf-frame_{}.jpg".format(img_counter)
            
                cv2.namedWindow("Capture Window")
                cv2.moveWindow("Capture Window", -500, -500)# push it off screen :)
                
                cv2.imwrite(img_name, frame1)
                print('Wrote Image-'+ img_name)
                
                
                img_counter +=1
                
            #Clear Capture Flag when done grabbing images
            if capture_flag == 'True' and img_counter >= capture_image_limit: 
                   os.environ['cap_flag'] = 'False'
                   img_counter = 0
                   
            yield (b'--frame\r\n'
                   b'Content-Type: image/jpeg\r\n\r\n' + frame2 + b'\r\n')  # concat frame one by one and show result
            ## End Video Stream API ###

            # Calculate framerate
            t2 = cv2.getTickCount()
            time1 = (t2-t1)/freq
            frame_rate_calc= 1/time1
                       
            # Press 'q' to quit
            if cv2.waitKey(1) == ord('q'):
                print("CV2 Break")
                break
        
            
            # Press 'q' to quit
            quit_flag = os.environ.get('quit_flag')
            if quit_flag == 'quit':#
                os.environ['quit_flag'] = ''
                print("CV2 Quit " + quit_flag)
                cv2.destroyAllWindows()
                if videostream:
                    #videostream.release()
                    videostream.stop()
                    print('Videostream stopped')
                break
            
                
            #print("quit_flag " + str(quit_flag))    
        # Clean up
        cv2.destroyAllWindows()
        if videostream:
            #videostream.release()
            videostream.stop()
            #os.system("pkill chromium")
        #webbrowser.open('http://localhost:5000', new=0)
            
    except KeyboardInterrupt:
        pass
Ejemplo n.º 11
0
def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--model',
                        help='File path of .tflite file.',
                        required=True)
    parser.add_argument('--labels',
                        help='File path of labels file.',
                        required=True)
    parser.add_argument('--threshold',
                        help='Score threshold for detected objects.',
                        required=False,
                        type=float,
                        default=0.4)  # originally 0.4
    args = parser.parse_args()

    labels = load_labels(args.labels)
    interpreter = Interpreter(
        args.model,
        experimental_delegates=[load_delegate('libedgetpu.so.1.0')])  #coral
    interpreter.allocate_tensors()
    _, input_height, input_width, _ = interpreter.get_input_details(
    )[0]['shape']

    # initialize variables to calculate FPS
    instantaneous_frame_rates = []

    # initialize variable for tracker use
    counter = 1

    start_time = time.monotonic()

    t = []

    # begin video stream internally
    ######vs = VideoStream(usePiCamera=True).start()
    vs = cv2.VideoCapture(
        '/home/pi/Desktop/object_detection/vision_system_multi_tracking_slow/test_video_11.mp4'
    )

    # uncomment next two lines for exporting video
    # fourcc = cv2.VideoWriter_fourcc(*'XVID')
    #out = cv2.VideoWriter('output2.avi', fourcc, 30.0, (640,480))

    # wait 1 second to give the camera time to adjust to lighting
    time.sleep(1.0)

    # main loop
    while True:  #(vs.isOpened()):

        # calculating instantaneous FPS
        total_time = (time.monotonic() - start_time)
        start_time = time.monotonic()
        print("FPS: " + str(1 / (total_time)))

        # Keep track of loop number
        counter += 1

        # get and resize current frame from camera
        ret, frame = vs.read()
        if ret == False:
            break
        frame = cv2.resize(frame, (input_width, input_height))
        (H, W) = frame.shape[:2]

        # if no tracker exits
        if len(t) == 0:

            # formating the frame as an RGB image for the TensorFlow detector
            image_detector = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

            # get object detection results from TensorFlow Lite object detection model
            results = detect_objects(interpreter, image_detector,
                                     args.threshold)

            # get coordinates of bounding boxes
            rects = get_rects(results)

            # loops through results
            for i in np.arange(0, len(results)):
                #format bounding box coordinates for OpenCV tracker
                box = np.array(rects[i])
                (startY, startX, endY, endX) = box.astype("int")
                cv_rect = (startX, startY, endX - startX, endY - startY)

                #Note on tracker types:
                #KCF: Average speed, Average accuracy
                #TLD: Average speed, works well is occlusion and scale changes
                #MOSSE: High speed, low accuracy
                #MedianFlow: High speed, good accuracy only on slow moving objects (current best)

                # initialize tracker
                #tracker = cv2.TrackerMedianFlow_create()
                #tracker.init(frame, cv_rect)
                #t.append(tracker)

                # draw bounding box from the detector on frame
                cv2.rectangle(frame, (startX, startY), (endX, endY),
                              (0, 255, 0), 2)

                # return active objects from the centroid tracker
                objects = ct.update(rects)

                # display object centroid on screen
                for (objectID, centroid) in objects.items():
                    text = "ID {}".format(objectID)
                    #annotator.text([centroid[0],centroid[1]], text)
                    cv2.putText(frame, text,
                                (centroid[0] - 10, centroid[1] - 10),
                                cv2.FONT_HERSHEY_SIMPLEX, .5, (0, 255, 0))
                    cv2.circle(frame, (centroid[0], centroid[1]), 5,
                               (0, 255, 0))

        # if a tracker has already been set up
        else:
            for tracker in t:
                # update the tracker is new frame and get new results
                (success, box) = tracker.update(frame)

                # if tracker was successful
                if success:
                    # draw bounding box; box format [xmin, ymin, width, height], cv2.rectangle format [xmin, ymin, xmax, ymax]
                    cv2.rectangle(frame, (int(box[0]), int(box[1])),
                                  (int(box[0] + box[2]), int(box[1] + box[3])),
                                  (0, 255, 0), 2)

                    # update centroud tracker; centroid format [ymin, xmin, ymax, xmax]
                    # TODO: Fix formating!
                    objects = ct.update([[
                        int(box[1]),
                        int(box[0]),
                        int(box[1] + box[3]),
                        int(box[0] + box[2])
                    ]])

                    # draw centorid
                    for (objectID, centroid) in objects.items():
                        text = "ID {}".format(objectID)
                        #annotator.text([centroid[0],centroid[1]], text)
                        cv2.putText(frame, text,
                                    (centroid[0] - 10, centroid[1] - 10),
                                    cv2.FONT_HERSHEY_SIMPLEX, .5, (0, 255, 0))
                        cv2.circle(frame, (centroid[0], centroid[1]), 5,
                                   (0, 255, 0))

            # Every n frames the tracker will be erased and the object detector will run again to re-initialize the tracker
            # n=15 for MedianFlow
            if (counter % 15) == 0:
                t = []

        # resize frame for display
        frame = cv2.resize(frame, (640, 480))

        # uncomment next time to export video
        # out.write(frame)

        cv2.imshow("Frame", frame)
        key = cv2.waitKey(1) & 0xFF

        # key "q" quits main loop
        if key == ord("q"):
            break

    # once out of main loop program ends
    cv2.destroyAllWindows()
    #vs.stop
    vs.release
def objectsCount(MODEL_NAME, GRAPH_NAME, LABELMAP_NAME, min_conf_threshold,
                 use_TPU, IM_NAME, IM_DIR):
    import os
    import cv2
    import numpy as np
    import sys
    import glob
    import importlib.util

    # If both an image AND a folder are specified, throw an error
    if (IM_NAME and IM_DIR):
        print(
            'Error! Please only use the --image argument or the --imagedir argument, not both. Issue "python TFLite_detection_image.py -h" for help.'
        )
        sys.exit()

    # If neither an image or a folder are specified, default to using 'test1.jpg' for image name
    if (not IM_NAME and not IM_DIR):
        IM_NAME = 'test1.jpg'

    # Import TensorFlow libraries
    # If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
    # If using Coral Edge TPU, import the load_delegate library
    pkg = importlib.util.find_spec('tflite_runtime')
    if pkg:
        from tflite_runtime.interpreter import Interpreter
        if use_TPU:
            from tflite_runtime.interpreter import load_delegate
    else:
        from tensorflow.lite.python.interpreter import Interpreter
        if use_TPU:
            from tensorflow.lite.python.interpreter import load_delegate

    # Get path to current working directory
    CWD_PATH = os.getcwd()

    # Define path to images and grab all image filenames
    if IM_DIR:
        PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_DIR)
        images = glob.glob(PATH_TO_IMAGES + '/*')

    elif IM_NAME:
        PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_NAME)
        images = glob.glob(PATH_TO_IMAGES)

    # Path to .tflite file, which contains the model that is used for object detection
    PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)

    # Path to label map file
    PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)

    # Load the label map
    with open(PATH_TO_LABELS, 'r') as f:
        labels = [line.strip() for line in f.readlines()]

    # Have to do a weird fix for label map if using the COCO "starter model" from
    # https://www.tensorflow.org/lite/models/object_detection/overview
    # First label is '???', which has to be removed.
    if labels[0] == '???':
        del (labels[0])

    # Load the Tensorflow Lite model.
    # If using Edge TPU, use special load_delegate argument
    if use_TPU:
        interpreter = Interpreter(
            model_path=PATH_TO_CKPT,
            experimental_delegates=[load_delegate('libedgetpu.so.1.0')])

    else:
        interpreter = Interpreter(model_path=PATH_TO_CKPT)

    interpreter.allocate_tensors()

    # Get model details
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()
    height = input_details[0]['shape'][1]
    width = input_details[0]['shape'][2]

    floating_model = (input_details[0]['dtype'] == np.float32)

    input_mean = 127.5
    input_std = 127.5

    objects_list = {
    }  #create the dictionary where the traffic names and number of cars detected will be saved

    # Loop over every image and perform detection
    for image_path in images:

        # Load image and resize to expected shape [1xHxWx3]
        image = cv2.imread(image_path)
        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        imH, imW, _ = image.shape
        image_resized = cv2.resize(image_rgb, (width, height))
        input_data = np.expand_dims(image_resized, axis=0)

        # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
        if floating_model:
            input_data = (np.float32(input_data) - input_mean) / input_std

        # Perform the actual detection by running the model with the image as input
        interpreter.set_tensor(input_details[0]['index'], input_data)
        interpreter.invoke()

        # Retrieve detection results
        boxes = interpreter.get_tensor(output_details[0]['index'])[
            0]  # Bounding box coordinates of detected objects
        classes = interpreter.get_tensor(
            output_details[1]['index'])[0]  # Class index of detected objects
        scores = interpreter.get_tensor(
            output_details[2]['index'])[0]  # Confidence of detected objects
        #num = interpreter.get_tensor(output_details[3]['index'])[0]  # Total number of detected objects (inaccurate and not needed)

        objects_count = 0  #instantiate detected object counts

        # Loop over all detections and draw detection box if confidence is above minimum threshold
        for i in range(len(scores)):
            if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):

                # Draw label
                object_name = labels[int(
                    classes[i]
                )]  # Look up object name from "labels" array using class index
                if (object_name == 'car'):
                    objects_count = objects_count + 1  #get the count of cars detected in the image

        objects_list[image_path] = objects_count
    return (objects_list)
def object_detection():
    label_out = []
    mid_x_out = []
    mid_y_out = []

    class VideoStream:
        """Camera object that controls video streaming from the Picamera"""
        def __init__(self, resolution=(640, 480), framerate=30):
            # Initialize the PiCamera and the camera image stream
            self.stream = cv2.VideoCapture(0)
            ret = self.stream.set(cv2.CAP_PROP_FOURCC,
                                  cv2.VideoWriter_fourcc(*'MJPG'))
            ret = self.stream.set(3, resolution[0])
            ret = self.stream.set(4, resolution[1])

            # Read first frame from the stream
            (self.grabbed, self.frame) = self.stream.read()

            # Variable to control when the camera is stopped
            self.stopped = False

        def start(self):
            # Start the thread that reads frames from the video stream
            Thread(target=self.update, args=()).start()
            return self

        def update(self):
            # Keep looping indefinitely until the thread is stopped
            while True:
                # If the camera is stopped, stop the thread
                if self.stopped:
                    # Close camera resources
                    self.stream.release()
                    return

                # Otherwise, grab the next frame from the stream
                (self.grabbed, self.frame) = self.stream.read()

        def read(self):
            # Return the most recent frame
            return self.frame

        def stop(self):
            # Indicate that the camera and thread should be stopped
            self.stopped = True

    # Define and parse input arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('--modeldir',
                        help='Folder the .tflite file is located in',
                        required=True)
    parser.add_argument(
        '--graph',
        help='Name of the .tflite file, if different than detect.tflite',
        default='detect.tflite')
    parser.add_argument(
        '--labels',
        help='Name of the labelmap file, if different than labelmap.txt',
        default='labelmap.txt')
    parser.add_argument(
        '--threshold',
        help='Minimum confidence threshold for displaying detected objects',
        default=0.5)
    parser.add_argument(
        '--resolution',
        help=
        'Desired webcam resolution in WxH. If the webcam does not support the resolution entered, errors may occur.',
        default='1280x720')
    parser.add_argument(
        '--edgetpu',
        help='Use Coral Edge TPU Accelerator to speed up detection',
        action='store_true')

    args = parser.parse_args()

    MODEL_NAME = args.modeldir
    GRAPH_NAME = args.graph
    LABELMAP_NAME = args.labels
    min_conf_threshold = float(args.threshold)
    resW, resH = args.resolution.split('x')
    imW, imH = int(resW), int(resH)
    use_TPU = args.edgetpu

    # Import TensorFlow libraries
    # If tensorflow is not installed, import interpreter from tflite_runtime, else import from regular tensorflow
    # If using Coral Edge TPU, import the load_delegate library
    pkg = importlib.util.find_spec('tensorflow')
    if pkg is None:
        from tflite_runtime.interpreter import Interpreter
        if use_TPU:
            from tflite_runtime.interpreter import load_delegate
    else:
        from tensorflow.lite.python.interpreter import Interpreter
        if use_TPU:
            from tensorflow.lite.python.interpreter import load_delegate

    if use_TPU:
        # If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
        if (GRAPH_NAME == 'detect.tflite'):
            GRAPH_NAME = 'edgetpu.tflite'

    # Get path to current working directory
    CWD_PATH = os.getcwd()

    # Path to .tflite file, which contains the model that is used for object detection
    PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)

    # Path to label map file
    PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)

    # Load the label map
    with open(PATH_TO_LABELS, 'r') as f:
        labels = [line.strip() for line in f.readlines()]

    if labels[0] == '???':
        del (labels[0])

    # Load the Tensorflow Lite model.
    if use_TPU:
        interpreter = Interpreter(
            model_path=PATH_TO_CKPT,
            experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
        print(PATH_TO_CKPT)
    else:
        interpreter = Interpreter(model_path=PATH_TO_CKPT)

    interpreter.allocate_tensors()

    # Get model details
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()
    height = input_details[0]['shape'][1]
    width = input_details[0]['shape'][2]

    floating_model = (input_details[0]['dtype'] == np.float32)

    input_mean = 127.5
    input_std = 127.5

    # Initialize frame rate calculation
    frame_rate_calc = 1
    freq = cv2.getTickFrequency()

    # Initialize video stream
    videostream = VideoStream(resolution=(imW, imH), framerate=30).start()
    time.sleep(1)

    #for frame1 in camera.capture_continuous(rawCapture, format="bgr",use_video_port=True):
    while True:
        flag = 0
        # Start timer (for calculating frame rate)
        t1 = cv2.getTickCount()

        # Grab frame from video stream
        frame1 = videostream.read()

        # Acquire frame and resize to expected shape [1xHxWx3]
        frame = frame1.copy()
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame_resized = cv2.resize(frame_rgb, (width, height))
        input_data = np.expand_dims(frame_resized, axis=0)

        # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
        if floating_model:
            input_data = (np.float32(input_data) - input_mean) / input_std

        # Perform the actual detection by running the model with the image as input
        interpreter.set_tensor(input_details[0]['index'], input_data)
        interpreter.invoke()

        # Retrieve detection results
        boxes = interpreter.get_tensor(output_details[0]['index'])[
            0]  # Bounding box coordinates of detected objects
        classes = interpreter.get_tensor(
            output_details[1]['index'])[0]  # Class index of detected objects
        scores = interpreter.get_tensor(
            output_details[2]['index'])[0]  # Confidence of detected objects
        #num = interpreter.get_tensor(output_details[3]['index'])[0]  # Total number of detected objects (inaccurate and not needed)

        # Loop over all detections and draw detection box if confidence is above minimum threshold
        for i in range(len(scores)):
            if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):

                # Get bounding box coordinates and draw box
                # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
                ymin = int(max(1, (boxes[i][0] * imH)))
                xmin = int(max(1, (boxes[i][1] * imW)))
                ymax = int(min(imH, (boxes[i][2] * imH)))
                xmax = int(min(imW, (boxes[i][3] * imW)))

                cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (10, 255, 0),
                              2)
                cv2.circle(frame, (xmin, ymin), 5, (255, 255, 0), cv2.FILLED)
                cv2.circle(frame, (xmax, ymax), 5, (0, 255, 255), cv2.FILLED)
                x_diff = xmax - xmin
                y_diff = ymax - ymin
                mid_x = x_diff / 2 + xmin
                mid_x = math.ceil(mid_x)
                mid_y = ymin + y_diff / 2
                mid_y = math.ceil(mid_y)
                cv2.circle(frame, (0, 0), 5, (0, 0, 255), cv2.FILLED)
                cv2.circle(frame, (mid_x, mid_y), 5, (255, 255, 255),
                           cv2.FILLED)

                # Draw label
                object_name = labels[int(
                    classes[i]
                )]  # Look up object name from "labels" array using class index
                label = '%s: %d%%' % (object_name, int(scores[i] * 100)
                                      )  # Example: 'person: 72%'
                labelSize, baseLine = cv2.getTextSize(label,
                                                      cv2.FONT_HERSHEY_SIMPLEX,
                                                      0.7, 2)  # Get font size
                label_ymin = max(
                    ymin, labelSize[1] + 10
                )  # Make sure not to draw label too close to top of window
                cv2.rectangle(
                    frame, (xmin, label_ymin - labelSize[1] - 10),
                    (xmin + labelSize[0], label_ymin + baseLine - 10),
                    (255, 255, 255),
                    cv2.FILLED)  # Draw white box to put label text in
                cv2.putText(frame, label, (xmin, label_ymin - 7),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0),
                            2)  # Draw label text
                label_out.append(label)
                mid_x_out.append(mid_x)
                mid_y_out.append(mid_y)
        # Draw framerate in corner of frame
        cv2.putText(frame, 'FPS: {0:.2f}'.format(frame_rate_calc), (30, 50),
                    cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 2, cv2.LINE_AA)

        # All the results have been drawn on the frame, so it's time to display it.
        #cv2.imshow('Object detector', frame)

        # Calculate framerate
        t2 = cv2.getTickCount()
        time1 = (t2 - t1) / freq
        frame_rate_calc = 1 / time1
        (h, w) = frame.shape[:2]
        cv2.waitKey(100)
        break
    # Clean up
    cv2.destroyAllWindows()
    videostream.stop()
    return (label_out, mid_x_out, mid_y_out, h / 2, w / 2)
Ejemplo n.º 14
0
def process_frame(frame):

    global entry, lime_count, marker_count, lime_sizes, found_list, total_marker_width, pixel_per_metric
    interpreter = Interpreter(model_path=PATH_TO_CKPT, num_threads=4)

    interpreter.allocate_tensors()

    # Get model details
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()
    height = input_details[0]['shape'][1]
    width = input_details[0]['shape'][2]

    floating_model = (input_details[0]['dtype'] == np.float32)

    input_mean = 127.5
    input_std = 127.5

    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    #frame_rgb = frame
    frame_resized = cv2.resize(frame_rgb, (width, height))
    #frame_resized = cv2.resize(frame_rgb, (480, 320))
    input_data = np.expand_dims(frame_resized, axis=0)
    # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
    if floating_model:
        input_data = (np.float32(input_data) - input_mean) / input_std

    # Perform the actual detection by running the model with the image as input
    try:
        start_time = time.time()
        interpreter.set_tensor(input_details[0]['index'], input_data)
        interpreter.invoke()
        elapsed_time.append(time.time() - start_time)
    except:
        print('Thread Error: interpreter not reference')
    # Retrieve detection results
    boxes = interpreter.get_tensor(output_details[0]['index'])[
        0]  # Bounding box coordinates of detected objects
    classes = interpreter.get_tensor(
        output_details[1]['index'])[0]  # Class index of detected objects
    scores = interpreter.get_tensor(
        output_details[2]['index'])[0]  # Confidence of detected objects
    #num = interpreter.get_tensor(output_details[3]['index'])[0]  # Total number of detected objects (inaccurate and not needed)

    # Loop over all detections and draw detection box if confidence is above minimum threshold
    for i in range(len(scores)):
        if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):

            # Get bounding box coordinates and draw box
            # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
            ymin = int(max(1, (boxes[i][0] * imH)))
            xmin = int(max(1, (boxes[i][1] * imW)))
            ymax = int(min(imH, (boxes[i][2] * imH)))
            xmax = int(min(imW, (boxes[i][3] * imW)))

            cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (10, 255, 0), 4)

            # Draw label
            object_name = labels[int(
                classes[i]
            )]  # Look up object name from "labels" array using class index
            label = '%s: %d%%' % (object_name, int(scores[i] * 100)
                                  )  # Example: 'person: 72%'
            labelSize, baseLine = cv2.getTextSize(label,
                                                  cv2.FONT_HERSHEY_SIMPLEX,
                                                  0.7, 2)  # Get font size
            label_ymin = max(
                ymin, labelSize[1] +
                10)  # Make sure not to draw label too close to top of window
            cv2.rectangle(frame, (xmin, label_ymin - labelSize[1] - 10),
                          (xmin + labelSize[0], label_ymin + baseLine - 10),
                          (255, 255, 255),
                          cv2.FILLED)  # Draw white box to put label text in
            cv2.putText(frame, label, (xmin, label_ymin - 7),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0),
                        2)  # Draw label text

            # counting objects and measure diameter of lime
            if xmin < LINE2 and xmax > LINE1 and not entry:
                entry = True

            if entry and xmax <= LINE1:
                entry = False

                if (int(classes[i]) + 1 == 1):
                    lime_found = time.time() - start_total_time

                    try:
                        lime_count += 1
                        lime_diameter = (
                            (xmax - xmin) +
                            (ymax - ymin)) / (2 * pixel_per_metric)
                        lime_sizes.append(lime_diameter)
                        found_list.append(lime_found)
                        print(
                            f'lime {lime_count} is found at {lime_found}, Diameter(size): {lime_diameter * 1000:.3f} mm'
                        )
                    except:
                        # marker must came first for calculating pixel/metric
                        lime_count -= 1
                        marker_count += 1
                        total_marker_width += ((xmax - xmin) +
                                               (ymax - ymin)) / 2
                        pixel_per_metric = (total_marker_width /
                                            marker_count) / MARKER_DIAMETER

                elif (int(classes[i]) + 1 == 2):
                    marker_count += 1
                    total_marker_width += ((xmax - xmin) + (ymax - ymin)) / 2

                    pixel_per_metric = (total_marker_width /
                                        marker_count) / MARKER_DIAMETER
    # insert Lime Count information text
    font = cv2.FONT_HERSHEY_SIMPLEX
    cv2.putText(
        frame,
        'Lime Count: ' + str(lime_count),
        (10, 35),
        font,
        0.8,
        (0, 0xFF, 0xFF),
        2,
        cv2.FONT_HERSHEY_SIMPLEX,
    )

    # insert Marker Count information text
    cv2.putText(
        frame,
        'Marker Count: ' + str(marker_count),
        (10, 55),
        font,
        0.8,
        (0, 0xFF, 0xFF),
        2,
        cv2.FONT_HERSHEY_SIMPLEX,
    )

    # overlay with line
    pt1 = (LINE1, 0)
    pt2 = (LINE1, int(sqsize))
    cv2.line(frame, pt1, pt2, (0, 0, 255), 2)

    pt1 = (LINE2, 0)
    pt2 = (LINE2, int(sqsize))
    cv2.line(frame, pt1, pt2, (0, 0, 255), 2)

    frame = cv2.resize(frame, (480, 320))

    return frame
Ejemplo n.º 15
0
class ButtDetector:
    def __init__(self):
        if TPU:
            self.interpreter = Interpreter(model_path=os.path.join(MODEL_DIR,
                                                                   'butt_detecter_quantized_edgetpu.tflite'),
                                           experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
        else:
            self.interpreter = Interpreter(model_path=os.path.join(MODEL_DIR, 'butt_detecter_quantized.tflite'))
        self.interpreter.allocate_tensors()
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()
        self.height = self.input_details[0]['shape'][1]
        self.width = self.input_details[0]['shape'][2]
        self.floating_model = (self.input_details[0]['dtype'] == np.float32)

        with open(os.path.join(MODEL_DIR, "labelmap.txt"), 'r') as f:
            self.labels = [line.strip() for line in f.readlines()]

        self.butt_nums = 0
        self.detected_status = False

    def detect_butts(self, frame, count_ret=False):
        if count_ret:
            # st_time = time.time()
            image_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            im_h, im_w, _ = frame.shape
            image_resized = cv2.resize(image_rgb, (self.width, self.height))
            input_data = np.expand_dims(image_resized, axis=0)

            # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
            if self.floating_model:
                input_data = (np.float32(input_data) - INPUT_MEAN) / INPUT_STD

            # Perform the actual detection by running the model with the image as input
            self.interpreter.set_tensor(self.input_details[0]['index'], input_data)
            self.interpreter.invoke()
            scores = self.interpreter.get_tensor(self.output_details[2]['index'])[0]

            # Loop over all detections and draw detection box if confidence is above minimum threshold
            detected_butts = 0
            for i in range(len(scores)):
                if (scores[i] > THRESHOLD) and (scores[i] <= 1.0):
                    detected_butts += 1

            if self.butt_nums == 0 and detected_butts > 0:
                self.butt_nums = detected_butts
                self.detected_status = True
            elif self.butt_nums != 0 and detected_butts != 0 and self.butt_nums != detected_butts:
                self.butt_nums = detected_butts
                self.detected_status = True
            elif self.butt_nums != 0 and detected_butts == 0:
                self.butt_nums = detected_butts
                self.detected_status = False
            elif self.butt_nums != 0 and self.butt_nums == detected_butts:
                self.detected_status = False
            # print(f"[INFO] Processing Time: {time.time() - st_time}")

        cv2.putText(frame, f"The number of Butts: {self.butt_nums}", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.7,
                    (0, 255, 0), 2)

        return frame
Ejemplo n.º 16
0
class ObjectDetectorLite:
    def __init__(self, model_path, label_path):
        """
            Builds Tensorflow graph, load model and labels
        """

        # Load label_map
        self._load_label(label_path)

        # Define lite graph and Load Tensorflow Lite model into memory
        self.interpreter = Interpreter(model_path=model_path)
        self.interpreter.allocate_tensors()
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()

        # Get input size
        input_shape = self.input_details[0]['shape']
        self.size = input_shape[:2] if len(input_shape) == 3 else input_shape[1:3]

    def get_input_size(self):
        return self.size

    def detect(self, image, threshold=0.1):
        """
            Predicts person in frame with threshold level of confidence
            Returns list with top-left, bottom-right coordinates and list with labels, confidence in %
        """
        # Add a batch dimension
        frame = np.expand_dims(image, axis=0)
        
        # run model
        self.interpreter.set_tensor(self.input_details[0]['index'], frame)
        self.interpreter.invoke()

        # get results
        boxes = self.interpreter.get_tensor(self.output_details[0]['index'])
        classes = self.interpreter.get_tensor(self.output_details[1]['index'])
        scores = self.interpreter.get_tensor(self.output_details[2]['index'])
        num = self.interpreter.get_tensor(self.output_details[3]['index'])

        # Find detected boxes coordinates
        return self._boxes_coordinates(image,
                                        np.squeeze(boxes[0]),
                                        np.squeeze(classes[0]+1).astype(np.int32),
                                        np.squeeze(scores[0]),
                                        min_score_thresh=threshold)

    def close(self):
        pass
      
    def _boxes_coordinates(self,
                        image,
                        boxes,
                        classes,
                        scores,
                        max_boxes_to_draw=20,
                        min_score_thresh=.5):
        """
        This function groups boxes that correspond to the same location
        and creates a display string for each detection 

        Args:
          image: uint8 numpy array with shape (img_height, img_width, 3)
          boxes: a numpy array of shape [N, 4]
          classes: a numpy array of shape [N]
          scores: a numpy array of shape [N] or None.  If scores=None, then
            this function assumes that the boxes to be plotted are groundtruth
            boxes and plot all boxes as black with no classes or scores.
          max_boxes_to_draw: maximum number of boxes to visualize.  If None, draw
            all boxes.
          min_score_thresh: minimum score threshold for a box to be visualized
        """

        if not max_boxes_to_draw:
            max_boxes_to_draw = boxes.shape[0]
        number_boxes = min(max_boxes_to_draw, boxes.shape[0])
        detected_boxes = []
        probabilities = []
        categories = []

        for i in range(number_boxes):
            if scores is None or scores[i] > min_score_thresh:
                box = tuple(boxes[i].tolist())
                detected_boxes.append(box)
                probabilities.append(scores[i])
                categories.append(self.category_index[classes[i]])
        return np.array(detected_boxes), probabilities, categories

    def _load_label(self, path):
        """
            Loads labels
        """
        categories = load_labelmap(path)
        self.category_index = create_category_index(categories)
Ejemplo n.º 17
0
def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--model',
                        help='File path of .tflite file.',
                        required=True)
    parser.add_argument('--labels',
                        help='File path of labels file.',
                        required=True)
    args = parser.parse_args()

    labels = load_labels(args.labels)

    interpreter = Interpreter(args.model)
    interpreter.allocate_tensors()
    _, height, width, _ = interpreter.get_input_details()[0]['shape']

    print((height, width))
    history = collections.deque(maxlen=10)

    tty = serial.Serial(
        port='/dev/ttyUSB0',
        baudrate=115200,
        #parity=serial.PARITY_ODD,
        #stopbits=serial.STOPBITS_TWO,
        #bytesize=serial.SEVENBITS
    )

    tty.timeout = 0.1
    tty.writeTimeout = 0
    tty.setDTR(True)

    tty.close()
    tty.open()
    assert tty.isOpen()

    with picamera.PiCamera(resolution=(800, 480), framerate=30) as camera:
        slides()
        camera.start_preview()
        time.sleep(2)
        try:
            stream = io.BytesIO()
            for _ in camera.capture_continuous(stream,
                                               format='jpeg',
                                               use_video_port=True):
                stream.seek(0)
                image = Image.open(stream).convert('RGB').resize(
                    (width, height), Image.ANTIALIAS)
                start_time = time.time()
                results = classify_image(interpreter, image)
                elapsed_ms = (time.time() - start_time) * 1000
                label_id, prob = results[0]
                if prob < 0.6:
                    label_id = 1
                history.append(label_id)

                if len(history) == 10 and len(set(history)) == 1:

                    def move(avalue):
                        tty.write(b"A0\n")
                        time.sleep(0.5)
                        tty.write(b"C105\n")
                        time.sleep(0.5)
                        tty.write(b"D55\n")
                        time.sleep(0.5)
                        tty.write(b"B150\n")
                        time.sleep(0.5)

                        tty.write(b"C140\n")
                        time.sleep(0.5)
                        tty.write(b"D0\n")
                        time.sleep(0.5)
                        tty.write(f"A{avalue}\n".encode())
                        time.sleep(0.5)
                        tty.write(b"B90\n")
                        time.sleep(2)
                        tty.write(b"A90\n")
                        time.sleep(0.5)

                    if history[0] == 0:  # nakretka
                        move(70)

                    if history[0] == 2:  # papier
                        move(110)
                    history = collections.deque(maxlen=10)

                stream.seek(0)
                stream.truncate()
                camera.annotate_text_size = 48
                camera.annotate_foreground = picamera.Color(y=0, u=0, v=0)
                camera.annotate_background = picamera.Color(y=1, u=0, v=0)
                camera.annotate_text = '%s' % (labels[label_id]
                                               )  #, prob, elapsed_ms)
        finally:
            camera.stop_preview()
            tty.close()
Ejemplo n.º 18
0
class ImageProcessing:

    
    import numpy as np
    import cv2
    import math
    from threading import Thread
    from multiprocessing import Process
    from threading import Timer
    import os

    def __init__(self, resolution, flag):
        
        self.height, self.width = resolution
        self.resolution = self.height, self.width
        
        self.index=0
        self.mode = 0
        self.flag = flag
        
        self.camera = self.cv2.VideoCapture(-1)
        self.camera.set(self.cv2.CAP_PROP_FRAME_HEIGHT, self.height)
        self.camera.set(self.cv2.CAP_PROP_FRAME_WIDTH, self.width)
        
        self._setROI()
        
        self._setCoral()
        
        self._getDashboard()
        
        self._getIpmMat()
        
        self._getMaskBG()
        
        
                
    def _setROI(self):
        self.ROI_W = int(self.width*0.2)
        self.ROI_H = int(self.height*0.35)
        self.ROI_far_pos = 0.65 # Set along your vehicle velocity and frane rate | Late: far, Fast: near
        self.ROI_far_rngH = slice(int(self.height*self.ROI_far_pos - self.ROI_H), int(self.height*self.ROI_far_pos))
        self.ROI_near_rngH = slice(int(self.height - self.ROI_H), int(self.height))
        self.ROI_rngW = slice(int(self.width/2 - self.ROI_W/2),int(self.width/2 + self.ROI_W/2))
        self.ROI_lane_rngH = slice(int(self.height*0.7 - self.ROI_H*0.7), int(self.height*0.7))
        self.ROI_lane_rngW = slice(int(self.width/2 - self.ROI_W*0.5),int(self.width/2 + self.ROI_W*0.5))
        
    def _getIpmMat(self):
        
        # Your camera inner & external parameters
        alpha = (7-90)*self.np.pi/180
        beta = 0
        gamma = 0
        dist = 300
        focal = 500
        
        # Calculating rotational transformation matrix
        A1 = self.np.array([[1,0,-self.width/2],[0,1,-self.height/2],[0,0,0],[0,0,1]],dtype='f')
        RX = self.np.array([[1,0,0,0],[0,self.math.cos(alpha), -self.math.sin(alpha),0],[0,self.math.sin(alpha),self.math.cos(alpha),0],[0,0,0,1]],dtype='f')
        RY = self.np.array([[self.math.cos(beta),0,-self.math.sin(beta),0],[0,1,0,0],[self.math.sin(beta),0,self.math.cos(beta),0],[0,0,0,1]],dtype='f')
        RZ = self.np.array([[self.math.cos(gamma),-self.math.sin(gamma),0,0],[self.math.sin(gamma),self.math.cos(gamma),0,0],[0,0,1,0],[0,0,0,1]],dtype='f')
        R = self.np.dot(RX,self.np.dot(RY,RZ))
        T = self.np.array([[1,0,0,0],[0,1,0,0],[0,0,1,dist],[0,0,0,1]],dtype='f')
        K = self.np.array([[focal,0,self.width/2,0],[0,focal,self.height/2,0],[0,0,1,0]],dtype='f')
        self.IpmMat = self.np.dot(K,self.np.dot(T,self.np.dot(R,A1)))
        
    
        
        
    def _getMaskBG(self):
        # Mask for blank area when images were ipm mapped 
        tmp_blank = self.np.full((self.height, self.width,3), 255, dtype='uint8')
        tmp_ipm = self.cv2.warpPerspective(tmp_blank, self.IpmMat, (self.width, self.height), flags=self.cv2.INTER_CUBIC|self.cv2.WARP_INVERSE_MAP)
        self.maskBG =  self.cv2.bitwise_not(tmp_ipm)
        
    def _getDashboard(self):
        self.board_size = 320
        size = self.board_size
        self.board = self.np.full((size*2, size*2,3), 255, dtype='uint8')
        tmp_stopline = self.cv2.imread('dashboard/stopline_red.jpg',self.cv2.IMREAD_COLOR)
        self.icon_stopline = self.cv2.resize(tmp_stopline, dsize=(size,size), interpolation=self.cv2.INTER_AREA)
        tmp_blindspot = self.cv2.imread('dashboard/blind_spot_red.jpg',self.cv2.IMREAD_COLOR)
        self.icon_blindspot = self.cv2.resize(tmp_blindspot, dsize=(size,size), interpolation=self.cv2.INTER_AREA)
        self.icon_blank = self.np.full((size,size*2,3),255,dtype='uint8')
        self.icon_schoolzone = self.np.full((size,size*2,3),0,dtype='uint8')
        self.icon_schoolzone[:,:,2] = 255
        self.icon_subs = self.np.full((size,size,3),0,dtype='uint8')
        self.icon_subs[:,:,2] = 255
        
               
        
        
    def processing(self):
        # Calibration parameters by experiments
        CamMat = self.np.array([[314.484, 0, 321.999],[0, 315.110, 259.722],[ 0, 0, 1]],dtype='f')
        DistMat = self.np.array([ -0.332015,    0.108453,    0.001100,    0.002183],dtype='f')
        
        # For inRange function in opencv
        # Modify value along your brightness condition
        lower_k = self.np.array([0,0,0])
        upper_k = self.np.array([180,255,100])
        lower_r1 = self.np.array([0,50,50])
        upper_r1 = self.np.array([30,255,255])
        lower_r2 = self.np.array([150,50,50])
        upper_r2 = self.np.array([180,255,255])
        
        
        ret, frame = self.camera.read(); del(ret) # Now take frame from camera
        calibration = self.cv2.undistort(frame, CamMat, DistMat, None, CamMat) # Calibration because of wide angle camera
        tmp_ipm1 = self.cv2.warpPerspective(calibration, self.IpmMat, (self.width,self.height), flags=self.cv2.INTER_CUBIC|self.cv2.WARP_INVERSE_MAP) # Geometrical transform image to Top view perspective
        tmp_ipm2 = self.cv2.add(tmp_ipm1, self.maskBG) # It just merges ipm image with white background
        ipm = self.cv2.bilateralFilter(tmp_ipm2,9,50,50) # Just Filter
        self.result = ipm.copy()
        hsv = self.cv2.cvtColor(ipm, self.cv2.COLOR_BGR2HSV)
        gray = self.cv2.cvtColor(ipm, self.cv2.COLOR_BGR2GRAY)
        
        #canny = self.cv2.Canny(gray, 100, 200, 3) # If you want to use canny edge algorithm, activate this line
        threshold_inv = self.cv2.adaptiveThreshold(gray, 255, self.cv2.ADAPTIVE_THRESH_MEAN_C, self.cv2.THRESH_BINARY, 21, 5)
        threshold = self.cv2.bitwise_not(threshold_inv)
        #mask_k = self.cv2.inRange(hsv, lower_k, upper_k)
        #mask_k = canny.copy()
        mask_k = threshold.copy()
        self.mask_k = mask_k[self.ROI_far_rngH, self.ROI_rngW]#[self.ROI_far_rngH, self.ROI_rngW]
        self.mask_lane = mask_k[self.ROI_lane_rngH,self.ROI_lane_rngW]
        
        # Now you can get red mask for schoolzone detecting
        mask_r1 = self.cv2.inRange(hsv, lower_r1, upper_r1)
        mask_r2 = self.cv2.inRange(hsv, lower_r2, upper_r2)
        mask_r = self.cv2.add(mask_r1, mask_r2)
        self.mask_r = mask_r[self.ROI_near_rngH, self.ROI_rngW]
        
        
    def detectingSchoolzone(self):
        # Just counting red dots
        if((self.np.sum(self.mask_r)/255) > ((self.ROI_H)*(self.ROI_W)*0.2)):
            self.flag.schoolzone = True
        else:
            self.flag.schoolzone = False
    
    def laneDetect(self):
        # By Jinwon, Lane detecting algorithm.
        # adaptive detecting method
        # It need to improve
        frame = self.cv2.flip(self.mask_lane.copy(),0)
        
        H,W = frame.shape[0:2]
        
        lane_base = self.np.array(range(0,W))
        
        
        lane = self.np.full((H,1), int(W/2), dtype='uint32')
        laneL = self.np.full((H,1), int(W/2), dtype='uint32')
        laneR = self.np.full((H,1), int(W/2), dtype='uint32')
        
        
        num0 = self.np.sum(frame[0,:] != False)
        if(num0 != 0): lane[0] = int(self.np.sum(lane_base*frame[0,:])/(255*num0))
        else: lane[0] = int(W/2)
        
        for j in range(1,H):
            rangeL = range(0, int(lane[j-1]))
            rangeR = range(int(lane[j-1]), int(W))
            numL = self.np.sum(frame[j,rangeL] !=  False)
            numR = self.np.sum(frame[j,rangeR] !=  False)
            if(numL == 0)|(numR == 0): lane[j] = lane[j-1]
            else:
                laneL[j] = self.np.sum(lane_base[rangeL]*frame[j,rangeL])/(255*numL)
                laneR[j] = self.np.sum(lane_base[rangeR]*frame[j,rangeR])/(255*numR)
                lane[j] = (laneR[j] + laneL[j])/2
            self.mask_lane[int(H - j),int(lane[j])] = 255
        
        self.flag.lane_err = ((self.np.mean(lane)*2/W) -1) # Return method is various. It just return mean value
        
        
    def _setCoral(self, modeldir="Model"): # By github.com/EdjeElectronics & coral.ai
        CWD_PATH =self.os.getcwd()
        MODEL_NAME = modeldir
        GRAPH_NAME = "edgetpu.tflite" #If you don't have coral, "detect.tflite"
        LABELMAP_NAME = "labelmap.txt"
        # path to 
        PATH_TO_CKPT = self.os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
        PATH_TO_LABELS = self.os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
        
        # Load the label map
        with open(PATH_TO_LABELS, 'r') as f:
            self.coral_labels = [line.strip() for line in f.readlines()]
        
        # Have to do a weird fix for label map if using the COCO "starter model" from
        # https://www.tensorflow.org/lite/models/object_detection/overview
        # First label is '???', which has to be removed.
        if self.coral_labels[0] == '???':
            del(self.coral_labels[0])
        
        # Load the Tensorflow Lite model.
        # If using Edge TPU, use special load_delegate argument
        self.coral_interpreter = Interpreter(model_path=PATH_TO_CKPT,
                                  experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
                                  
        self.coral_interpreter.allocate_tensors()
        
        # Get model details
        self.coral_input_details = self.coral_interpreter.get_input_details()
        self.coral_output_details = self.coral_interpreter.get_output_details()
        self.coral_height = self.coral_input_details[0]['shape'][1]
        self.coral_width = self.coral_input_details[0]['shape'][2]

        self.coral_input_mean = 127.5
        self.coral_input_std = 127.5
        
        
        
    def detectingStopline(self): # By github.com/EdjeElectronics & coral.ai
        
        image = self.mask_k.copy()
        imH,imW = image.shape[0:2]
        
        # Get image & general
        coral_frame = self.cv2.cvtColor(image, self.cv2.COLOR_GRAY2RGB)
        frame_rgb = self.cv2.cvtColor(coral_frame, self.cv2.COLOR_BGR2RGB)
        frame_resized = self.cv2.resize(frame_rgb, (self.coral_width, self.coral_height))
        input_data = self.np.expand_dims(frame_resized, axis=0)


        # Perform the actual detection by running the model with the image as input
        self.coral_interpreter.set_tensor(self.coral_input_details[0]['index'],input_data)
        self.coral_interpreter.invoke()

        # Retrieve detection results
        self.coral_boxes = self.coral_interpreter.get_tensor(self.coral_output_details[0]['index'])[0] # Bounding box coordinates of detected objects
        self.coral_classes = self.coral_interpreter.get_tensor(self.coral_output_details[1]['index'])[0] # Class index of detected objects
        self.coral_scores = self.coral_interpreter.get_tensor(self.coral_output_details[2]['index'])[0] # Confidence of detected objects
        #num = interpreter.get_tensor(output_details[3]['index'])[0]  # Total number of detected objects (inaccurate and not needed)
        
        # Threshold
        self.coral_min_conf_threshold = 0.90
        
        self.flag.stopline = False
        # Loop over all detections and draw detection box if confidence is above minimum threshold
        for i in range(len(self.coral_scores)):
            if ((self.coral_scores[i] > self.coral_min_conf_threshold) and (self.coral_scores[i] <= 1.0)):
                if(self.coral_labels[int(self.coral_classes[i])] == "stopline"): self.flag.stopline = True
                
                
                # Get bounding box coordinates and draw box
                # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
                ymin = int(max(1,(self.coral_boxes[i][0] * imH)) + self.ROI_far_rngH.start)
                xmin = int(max(1,(self.coral_boxes[i][1] * imW)) + self.ROI_rngW.start)
                ymax = int(min(imH,(self.coral_boxes[i][2] * imH)) + self.ROI_far_rngH.start)
                xmax = int(min(imW,(self.coral_boxes[i][3] * imW)) + self.ROI_rngW.start)
                
                self.cv2.rectangle(self.result, (xmin,ymin), (xmax,ymax), (10, 255, 0), 2)
    
                # Draw label
                object_name = self.coral_labels[int(self.coral_classes[i])] # Look up object name from "labels" array using class index
                label = '%s: %d%%' % (object_name, int(self.coral_scores[i]*100)) # Example: 'person: 72%'
                labelSize, baseLine = self.cv2.getTextSize(label, self.cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size
                label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window
                self.cv2.rectangle(self.result, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), self.cv2.FILLED) # Draw white box to put label text in
                self.cv2.putText(self.result, label, (xmin, label_ymin-7), self.cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # Draw label text
        



    




    # Thread   
    def task(self):
        while(1):
            t1 = self.cv2.getTickCount()
            self.processing()
            self.laneDetect()
            self.detectingSchoolzone()
            self.flag.schoolzone = False #tmp
            
            
            if(self.mode == 1):
                self.detectingStopline()
            
            
            self.board[160:640,:,:] = self.result.copy()
            self.board[0:self.board_size,:,:] = self.icon_blank.copy()


            # Now, Mode Selector
            if(self.mode == 0):
                if(self.flag.schoolzone == True):
                    self.board[0:self.board_size,:,:] = self.icon_schoolzone.copy()
                    self.mode = 1
       
            elif(self.mode == 1):
                self.board[0:self.board_size,:,:] = self.icon_schoolzone.copy()
                if(self.flag.stopline == True):
                    self.board[0:self.board_size,0:self.board_size,:] = self.icon_stopline.copy()
                    self.mode = 2
                    self.flag.stop = True
                    
                                                                                
            elif(self.mode == 2):
                self.board[0:self.board_size,:,:] = self.icon_schoolzone.copy()
                self.board[0:self.board_size,0:self.board_size,:] = self.icon_stopline.copy()
                
                if(self.flag.depart == True):
                    self.flag.depart = False
                    self.flag.powerHandle = True
                    self.mode = 3
                    
            elif(self.mode == 3):
                self.board[0:self.board_size,:,:] = self.icon_schoolzone.copy()
                
                if(self.flag.refresh == True):
                    self.flag.refresh = False
                    self.flag.lidar = True
                    self.mode = 4
                    
            elif(self.mode == 4):
                self.board[0:self.board_size,:,:] = self.icon_schoolzone.copy()
                if(self.flag.blindspot == True):
                    self.mode = 5
                    self.flag.slow = True
                    
            elif(self.mode == 5):
                self.board[0:self.board_size,:,:] = self.icon_schoolzone.copy()
                self.board[0:self.board_size,self.board_size:(self.board_size*2),:] = self.icon_blindspot.copy()
                if(self.flag.blindspot == False):
                    self.mode = 6
                    self.flag.slow = False
                    
            elif(self.mode == 6):
                self.board[0:self.board_size,:,:] = self.icon_schoolzone.copy()
                if(self.flag.schoolzone == False):
                    self.board[0:self.board_size,:,:] = self.icon_blank.copy()
                    self.mode = 7
                    
               
            self.cv2.imshow('Dashboard', self.board)
            t2 = self.cv2.getTickCount()
            freq = self.cv2.getTickFrequency()
            #print(freq/(t2-t1))

            self.cv2.waitKey(1)
            
            if(self.mode == 7):
                self.flag.end = True
                break
        self.cv2.destroyAllWindows()

        
    def start(self):
        self.thread = self.Thread(target=self.task)
        self.thread.start()
    def startLane(self):
        self.threadLane = self.Thread(target=self.taskLane)
        self.threadLane.start()
Ejemplo n.º 19
0
def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--model',
                        help='File path of .tflite file.',
                        required=True)
    parser.add_argument('--labels',
                        help='File path of labels file.',
                        required=True)
    parser.add_argument('--threshold',
                        help='Score threshold for detected objects.',
                        required=False,
                        type=float,
                        default=0.4)
    args = parser.parse_args()

    labels = load_labels(args.labels)
    model_file, *device = args.model.split('@')
    try:
        interpreter = Interpreter(
            model_file,
            experimental_delegates=[
                tflite.load_delegate(EDGETPU_SHARED_LIB,
                                     {'device': device[0]} if device else {})
            ])
    except (ValueError, OSError):
        interpreter = Interpreter(model_file)
    interpreter.allocate_tensors()
    _, input_height, input_width, _ = interpreter.get_input_details(
    )[0]['shape']

    cap = cv2.VideoCapture(0)
    while True:
        ret, frame = cap.read()
        (CAMERA_WIDTH, CAMERA_HEIGHT) = (frame.shape[1], frame.shape[0])
        image = cv2.resize(frame,
                           dsize=(input_width, input_height),
                           interpolation=cv2.INTER_NEAREST)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

        start_time = time.monotonic()
        results = detect_objects(interpreter, image, args.threshold)
        elapsed_ms = (time.monotonic() - start_time) * 1000
        for obj in results:
            ymin, xmin, ymax, xmax = obj['bounding_box']
            xmin = int(xmin * CAMERA_WIDTH)
            xmax = int(xmax * CAMERA_WIDTH)
            ymin = int(ymin * CAMERA_HEIGHT)
            ymax = int(ymax * CAMERA_HEIGHT)

            cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2)
            text = '{} {:.2f}'.format(labels[obj['class_id']], obj['score'])
            (text_width,
             text_height), baseline = cv2.getTextSize(text,
                                                      cv2.FONT_HERSHEY_SIMPLEX,
                                                      0.5, 1)
            cv2.rectangle(frame, (xmin, ymin),
                          (xmin + text_width, ymin - text_height - baseline),
                          (255, 0, 0),
                          thickness=cv2.FILLED)
            cv2.putText(frame, text, (xmin, ymin - baseline),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)

        cv2.putText(frame, '{:.1f}ms'.format(elapsed_ms), (10, 30),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)

        cv2.imshow('frame', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    cap.release()
    cv2.destroyAllWindows()
    return
Ejemplo n.º 20
0
    def __init__(self,
                 weights='yolov5s.pt',
                 device=None,
                 dnn=False,
                 data=None):
        # Usage:
        #   PyTorch:              weights = *.pt
        #   TorchScript:                    *.torchscript
        #   ONNX Runtime:                   *.onnx
        #   ONNX OpenCV DNN:                *.onnx with --dnn
        #   OpenVINO:                       *.xml
        #   CoreML:                         *.mlmodel
        #   TensorRT:                       *.engine
        #   TensorFlow SavedModel:          *_saved_model
        #   TensorFlow GraphDef:            *.pb
        #   TensorFlow Lite:                *.tflite
        #   TensorFlow Edge TPU:            *_edgetpu.tflite
        from models.experimental import attempt_download, attempt_load  # scoped to avoid circular import

        super().__init__()
        w = str(weights[0] if isinstance(weights, list) else weights)
        pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs = self.model_type(
            w)  # get backend
        stride, names = 64, [f'class{i}'
                             for i in range(1000)]  # assign defaults
        w = attempt_download(w)  # download if not local
        if data:  # data.yaml path (optional)
            with open(data, errors='ignore') as f:
                names = yaml.safe_load(f)['names']  # class names

        if pt:  # PyTorch
            model = attempt_load(weights if isinstance(weights, list) else w,
                                 map_location=device)
            stride = max(int(model.stride.max()), 32)  # model stride
            names = model.module.names if hasattr(
                model, 'module') else model.names  # get class names
            self.model = model  # explicitly assign for to(), cpu(), cuda(), half()
        elif jit:  # TorchScript
            LOGGER.info(f'Loading {w} for TorchScript inference...')
            extra_files = {'config.txt': ''}  # model metadata
            model = torch.jit.load(w, _extra_files=extra_files)
            if extra_files['config.txt']:
                d = json.loads(extra_files['config.txt'])  # extra_files dict
                stride, names = int(d['stride']), d['names']
        elif dnn:  # ONNX OpenCV DNN
            LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')
            check_requirements(('opencv-python>=4.5.4', ))
            net = cv2.dnn.readNetFromONNX(w)
        elif onnx:  # ONNX Runtime
            LOGGER.info(f'Loading {w} for ONNX Runtime inference...')
            cuda = torch.cuda.is_available()
            check_requirements(
                ('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime'))
            import onnxruntime
            providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'
                         ] if cuda else ['CPUExecutionProvider']
            session = onnxruntime.InferenceSession(w, providers=providers)
        elif xml:  # OpenVINO
            LOGGER.info(f'Loading {w} for OpenVINO inference...')
            check_requirements(
                ('openvino-dev', )
            )  # requires openvino-dev: https://pypi.org/project/openvino-dev/
            import openvino.inference_engine as ie
            core = ie.IECore()
            if not Path(w).is_file():  # if not *.xml
                w = next(Path(w).glob(
                    '*.xml'))  # get *.xml file from *_openvino_model dir
            network = core.read_network(
                model=w,
                weights=Path(w).with_suffix('.bin'))  # *.xml, *.bin paths
            executable_network = core.load_network(network,
                                                   device_name='CPU',
                                                   num_requests=1)
        elif engine:  # TensorRT
            LOGGER.info(f'Loading {w} for TensorRT inference...')
            import tensorrt as trt  # https://developer.nvidia.com/nvidia-tensorrt-download
            check_version(trt.__version__, '7.0.0',
                          hard=True)  # require tensorrt>=7.0.0
            Binding = namedtuple('Binding',
                                 ('name', 'dtype', 'shape', 'data', 'ptr'))
            trt_fp16_input = False
            logger = trt.Logger(trt.Logger.INFO)
            with open(w, 'rb') as f, trt.Runtime(logger) as runtime:
                model = runtime.deserialize_cuda_engine(f.read())
            bindings = OrderedDict()
            for index in range(model.num_bindings):
                name = model.get_binding_name(index)
                dtype = trt.nptype(model.get_binding_dtype(index))
                shape = tuple(model.get_binding_shape(index))
                data = torch.from_numpy(np.empty(
                    shape, dtype=np.dtype(dtype))).to(device)
                bindings[name] = Binding(name, dtype, shape, data,
                                         int(data.data_ptr()))
                if model.binding_is_input(index) and dtype == np.float16:
                    trt_fp16_input = True
            binding_addrs = OrderedDict(
                (n, d.ptr) for n, d in bindings.items())
            context = model.create_execution_context()
            batch_size = bindings['images'].shape[0]
        elif coreml:  # CoreML
            LOGGER.info(f'Loading {w} for CoreML inference...')
            import coremltools as ct
            model = ct.models.MLModel(w)
        else:  # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)
            if saved_model:  # SavedModel
                LOGGER.info(
                    f'Loading {w} for TensorFlow SavedModel inference...')
                import tensorflow as tf
                keras = False  # assume TF1 saved_model
                model = tf.keras.models.load_model(
                    w) if keras else tf.saved_model.load(w)
            elif pb:  # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt
                LOGGER.info(
                    f'Loading {w} for TensorFlow GraphDef inference...')
                import tensorflow as tf

                def wrap_frozen_graph(gd, inputs, outputs):
                    x = tf.compat.v1.wrap_function(
                        lambda: tf.compat.v1.import_graph_def(gd, name=""),
                        [])  # wrapped
                    ge = x.graph.as_graph_element
                    return x.prune(tf.nest.map_structure(ge, inputs),
                                   tf.nest.map_structure(ge, outputs))

                gd = tf.Graph().as_graph_def()  # graph_def
                gd.ParseFromString(open(w, 'rb').read())
                frozen_func = wrap_frozen_graph(gd,
                                                inputs="x:0",
                                                outputs="Identity:0")
            elif tflite or edgetpu:  # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python
                try:  # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu
                    from tflite_runtime.interpreter import Interpreter, load_delegate
                except ImportError:
                    import tensorflow as tf
                    Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate,
                if edgetpu:  # Edge TPU https://coral.ai/software/#edgetpu-runtime
                    LOGGER.info(
                        f'Loading {w} for TensorFlow Lite Edge TPU inference...'
                    )
                    delegate = {
                        'Linux': 'libedgetpu.so.1',
                        'Darwin': 'libedgetpu.1.dylib',
                        'Windows': 'edgetpu.dll'
                    }[platform.system()]
                    interpreter = Interpreter(
                        model_path=w,
                        experimental_delegates=[load_delegate(delegate)])
                else:  # Lite
                    LOGGER.info(
                        f'Loading {w} for TensorFlow Lite inference...')
                    interpreter = Interpreter(
                        model_path=w)  # load TFLite model
                interpreter.allocate_tensors()  # allocate
                input_details = interpreter.get_input_details()  # inputs
                output_details = interpreter.get_output_details()  # outputs
            elif tfjs:
                raise Exception(
                    'ERROR: YOLOv5 TF.js inference is not supported')
        self.__dict__.update(locals())  # assign all variables to self
Ejemplo n.º 21
0
def detection():

    # Define and parse input arguments
    #parser = argparse.ArgumentParser()
    #parser.add_argument('--modeldir', help='Folder the .tflite file is located in',
    #	            required=True)
    #parser.add_argument('--graph', help='Name of the .tflite file, if different than detect.tflite',
    #	            default='detect.tflite')
    #parser.add_argument('--labels', help='Name of the labelmap file, if different than labelmap.txt',
    #	            default='labelmap.txt')
    #parser.add_argument('--threshold', help='Minimum confidence threshold for displaying detected objects',
    #	            default=0.5)
    #parser.add_argument('--image', help='Name of the single image to perform detection on. To run detection on multiple images, use --imagedir',
    #	            default=None)
    #parser.add_argument('--imagedir', help='Name of the folder containing images to perform detection on. Folder must contain only images.',
    #	            default=None)
    #parser.add_argument('--edgetpu', help='Use Coral Edge TPU Accelerator to speed up detection',
    #	            action='store_true')

    #args = parser.parse_args()

    MODEL_NAME = 'Sample_TFLite_model'
    GRAPH_NAME = 'detect.tflite'
    LABELMAP_NAME = 'labelmap.txt'
    min_conf_threshold = float(0.5)
    use_TPU = None

    # Parse input image name and directory.
    IM_NAME = None
    IM_DIR = None

    # If both an image AND a folder are specified, throw an error
    if (IM_NAME and IM_DIR):
        print(
            'Error! Please only use the --image argument or the --imagedir argument, not both. Issue "python TFLite_detection_image.py -h" for help.'
        )
        sys.exit()

# If neither an image or a folder are specified, default to using 'test1.jpg' for image name
    if (not IM_NAME and not IM_DIR):
        IM_NAME = 'test1.jpg'

# Import TensorFlow libraries
# If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
    pkg = importlib.util.find_spec('tflite_runtime')
    if pkg:
        from tflite_runtime.interpreter import Interpreter
        if use_TPU:
            from tflite_runtime.interpreter import load_delegate
    else:
        from tensorflow.lite.python.interpreter import Interpreter
        if use_TPU:
            from tensorflow.lite.python.interpreter import load_delegate

# If using Edge TPU, assign filename for Edge TPU model
    if use_TPU:
        # If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
        if (GRAPH_NAME == 'detect.tflite'):
            GRAPH_NAME = 'edgetpu.tflite'

# Get path to current working directory
    CWD_PATH = os.getcwd()

    # Define path to images and grab all image filenames
    if IM_DIR:
        PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_DIR)
        images = glob.glob(PATH_TO_IMAGES + '/*')

    elif IM_NAME:
        PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_NAME)
        images = glob.glob(PATH_TO_IMAGES)

# Path to .tflite file, which contains the model that is used for object detection
    PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)

    # Path to label map file
    PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)

    # Load the label map
    with open(PATH_TO_LABELS, 'r') as f:
        labels = [line.strip() for line in f.readlines()]

# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
    if labels[0] == '???':
        del (labels[0])

# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
    if use_TPU:
        interpreter = Interpreter(
            model_path=PATH_TO_CKPT,
            experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
        print(PATH_TO_CKPT)
    else:
        interpreter = Interpreter(model_path=PATH_TO_CKPT)

    interpreter.allocate_tensors()

    # Get model details
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()
    height = input_details[0]['shape'][1]
    width = input_details[0]['shape'][2]

    floating_model = (input_details[0]['dtype'] == np.float32)

    input_mean = 127.5
    input_std = 127.5

    # Loop over every image and perform detection
    for image_path in images:

        # Load image and resize to expected shape [1xHxWx3]
        image = cv2.imread(image_path)
        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        imH, imW, _ = image.shape
        image_resized = cv2.resize(image_rgb, (width, height))
        input_data = np.expand_dims(image_resized, axis=0)

        # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
        if floating_model:
            input_data = (np.float32(input_data) - input_mean) / input_std

# Perform the actual detection by running the model with the image as input
        interpreter.set_tensor(input_details[0]['index'], input_data)
        interpreter.invoke()

        # Retrieve detection results
        boxes = interpreter.get_tensor(output_details[0]['index'])[
            0]  # Bounding box coordinates of detected objects
        classes = interpreter.get_tensor(
            output_details[1]['index'])[0]  # Class index of detected objects
        scores = interpreter.get_tensor(
            output_details[2]['index'])[0]  # Confidence of detected objects
        #num = interpreter.get_tensor(output_details[3]['index'])[0]  # Total number of detected objects (inaccurate and not needed)

        # Loop over all detections and draw detection box if confidence is above minimum threshold
        for i in range(len(scores)):
            if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):

                # Get bounding box coordinates and draw box
                # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
                ymin = int(max(1, (boxes[i][0] * imH)))
                xmin = int(max(1, (boxes[i][1] * imW)))
                ymax = int(min(imH, (boxes[i][2] * imH)))
                xmax = int(min(imW, (boxes[i][3] * imW)))

                cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (10, 255, 0),
                              2)

                # Draw label
                object_name = labels[int(
                    classes[i]
                )]  # Look up object name from "labels" array using class index
                label = '%s: %d%%' % (object_name, int(scores[i] * 100)
                                      )  # Example: 'person: 72%'
                score_detection = int(scores[i] * 100)
                #print("Your object is a ",object_name, " by : ",test_reCup, "%")
                labelSize, baseLine = cv2.getTextSize(label,
                                                      cv2.FONT_HERSHEY_SIMPLEX,
                                                      0.7, 2)  # Get font size
                label_ymin = max(
                    ymin, labelSize[1] + 10
                )  # Make sure not to draw label too close to top of window
                cv2.rectangle(
                    image, (xmin, label_ymin - labelSize[1] - 10),
                    (xmin + labelSize[0], label_ymin + baseLine - 10),
                    (255, 255, 255),
                    cv2.FILLED)  # Draw white box to put label text in
                cv2.putText(image, label, (xmin, label_ymin - 7),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0),
                            2)  # Draw label text
                return (object_name == 'cup' and score_detection >= 55)

# All the results have been drawn on the image, now display the image
        cv2.imshow('Object detector', image)

        # Press any key to continue to next image, or press 'q' to quit
        if cv2.waitKey(0) == ord('q'):
            break

# Clean up
    cv2.destroyAllWindows()
Ejemplo n.º 22
0
  def __init__(
      self,
      model_path: str,
      options: ObjectDetectorOptions = ObjectDetectorOptions()
  ) -> None:
    """Initialize a TFLite object detection model.

    Args:
        model_path: Path to the TFLite model.
        options: The config to initialize an object detector. (Optional)

    Raises:
        ValueError: If the TFLite model is invalid.
        OSError: If the current OS isn't supported by EdgeTPU.
    """

    # Load label list from metadata.
    try:
      with zipfile.ZipFile(model_path) as model_with_metadata:
        if not model_with_metadata.namelist():
          raise ValueError('Invalid TFLite model: no label file found.')

        file_name = model_with_metadata.namelist()[0]
        with model_with_metadata.open(file_name) as label_file:
          label_list = label_file.read().splitlines()
          self._label_list = [label.decode('ascii') for label in label_list]
    except zipfile.BadZipFile:
      print(
          'ERROR: Please use models trained with Model Maker or downloaded from TensorFlow Hub.'
      )
      raise ValueError('Invalid TFLite model: no metadata found.')

    # Initialize TFLite model.
    if options.enable_edgetpu:
      if edgetpu_lib_name() is None:
        raise OSError("The current OS isn't supported by Coral EdgeTPU.")
      interpreter = Interpreter(
          model_path=model_path,
          experimental_delegates=[load_delegate(edgetpu_lib_name())],
          num_threads=options.num_threads)
    else:
      interpreter = Interpreter(
          model_path=model_path, num_threads=options.num_threads)

    interpreter.allocate_tensors()
    input_detail = interpreter.get_input_details()[0]

    # From TensorFlow 2.6, the order of the outputs become undefined.
    # Therefore we need to sort the tensor indices of TFLite outputs and to know
    # exactly the meaning of each output tensor. For example, if
    # output indices are [601, 599, 598, 600], tensor names and indices aligned
    # are:
    #   - location: 598
    #   - category: 599
    #   - score: 600
    #   - detection_count: 601
    # because of the op's ports of TFLITE_DETECTION_POST_PROCESS
    # (https://github.com/tensorflow/tensorflow/blob/a4fe268ea084e7d323133ed7b986e0ae259a2bc7/tensorflow/lite/kernels/detection_postprocess.cc#L47-L50).
    sorted_output_indices = sorted(
        [output['index'] for output in interpreter.get_output_details()])
    self._output_indices = {
        self._OUTPUT_LOCATION_NAME: sorted_output_indices[0],
        self._OUTPUT_CATEGORY_NAME: sorted_output_indices[1],
        self._OUTPUT_SCORE_NAME: sorted_output_indices[2],
        self._OUTPUT_NUMBER_NAME: sorted_output_indices[3],
    }

    self._input_size = input_detail['shape'][2], input_detail['shape'][1]
    self._is_quantized_input = input_detail['dtype'] == np.uint8
    self._interpreter = interpreter
    self._options = options
def main():
  parser = argparse.ArgumentParser(
      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  parser.add_argument(
      '--model', help='File path of .tflite file.', required=True)
  parser.add_argument(
      '--labels', help='File path of labels file.', required=True)
  args = parser.parse_args()

  labels = load_labels(args.labels)

  interpreter = Interpreter(args.model)
  interpreter.allocate_tensors()
  _, height, width, _ = interpreter.get_input_details()[0]['shape']
  cap = cv2.VideoCapture(gstreamer_pipeline(flip_method=0), cv2.CAP_GSTREAMER)

  if cap.isOpened():
    window_handle = cv2.namedWindow("CSI Camera", cv2.WINDOW_AUTOSIZE)
    # Window
    while cv2.getWindowProperty("CSI Camera", 0) >= 0:
      # cv2.imshow function should use "img";classify_image function should use "image". Because of the file type.
      ret_val, img = cap.read()
      cv2.imshow("CSI Camera", img)
      image = Image.fromarray(cv2.cvtColor(img,cv2.COLOR_BGR2RGB))
      start_time = time.time()
      results = classify_image(interpreter, image)
      elapsed_ms = (time.time() - start_time) * 1000
      label_id, prob = results[0]
      seconds = time.time()
      local_time=time.ctime(seconds)

      #font = cv2.FONT_HERSHEY_SIMPLEX
      #color = (0, 0, 255)
      if prob > 0.7:
        print("LabelName:",labels[label_id])
        print("ScoreValue:",prob)
        print("Time:",local_time)
        #img=cv2.putText(img, labels[label_id], (10, 10), font,1, color, 1) #LabelName
        #img=cv2.putText(img, prob, (10, 20), font,1, color, 1) #ScoreValue
        #img=cv2.putText(img, local_time, (10, 30), font, 1, color, 1) #Time

        # fileName: use label name + random number as our file name.
        fileName =labels[label_id] + str(random.randint(1,99999)) + '.jpg'

        # Write image to labelName.jpg
        cv2.imwrite(fileName,img)

        # Upload data to Firebase
        os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="firebase_key.json   #This file is downloaded from FireBaseProject -> Settings -> serviceaccounts -> click the button to get your CREDENTIALS
        db_url='https://test-e7b86.firebaseio.com'    
        fdb=firebase.FirebaseApplication(db_url,None)

        # Upload image to Firebase
        client = storage.Client()
        bucket = client.get_bucket('test-e7b86.appspot.com')

        imagePath = "/home/e96031413/" + fileName    # Replace "/home/e96031413" with your own path
        imageBlob = bucket.blob(fileName)

        imageBlob.upload_from_filename(imagePath)    # Upload image to firebase
        imageBlob.make_public()
        publicURL = imageBlob.public_url

        firebase_data =[{'label':labels[label_id],'Score':prob,'Time':local_time,'fileName':fileName,'image_url':publicURL},]
        for data in firebase_data:
          fdb.post('bird-data',data)
          time.sleep(3)

        # Save to CSV file with pandas
        pandas_data = {'Label':labels[label_id],'Score':prob,'Time':local_time,'fileName':fileName,'image_url':publicURL}
        df = pd.DataFrame(data=pandas_data,index=[0])
        df.to_csv('bird_data.csv',mode='a',encoding='utf8')  # Use append mode so that the csv file won't be replaced by new files

      # keyCode detects which key you press
      keyCode = cv2.waitKey(30) & 0xFF
      # Stop the program on the ESC key
      if keyCode == 27:
        cap.release()
        cv2.destroyAllWindows()
Ejemplo n.º 24
0
    def humancheck(self, msg):
        #    if data.msg == 'stop':exit()
        human = False
        #define a name for the snapshot by giving a number
        img = '/home/pi/Images' + str(self.pic_name) + ".jpg"
        self.camera.capture(img)
        self.pic_name = self.pic_name + 1
        # Define and parse input argumets
        MODEL_NAME = '/home/pi/Sample_TFLite_model'
        GRAPH_NAME = 'detect.tflite'
        LABELMAP_NAME = 'labelmap.txt'
        min_conf_threshold = 0.5

        # Import TensorFlow libraries
        # If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
        pkg = importlib.util.find_spec('tflite_runtime')
        if pkg:
            from tflite_runtime.interpreter import Interpreter

        else:
            from tensorflow.lite.python.interpreter import Interpreter

            # Get path to current working directory
        CWD_PATH = os.getcwd()

        # Path to .tflite file, which contains the model that is used for object detection
        PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)

        # Path to label map file
        PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)

        # Load the label map
        with open(PATH_TO_LABELS, 'r') as f:
            labels = [line.strip() for line in f.readlines()]

            # Have to do a weird fix for label map if using the COCO "starter model" from
            # https://www.tensorflow.org/lite/models/object_detection/overview
            # First label is '???', which has to be removed.
        del (labels[0])

        # Load the Tensorflow Lite model.
        # If using Edge TPU, use special load_delegate argument

        interpreter = Interpreter(model_path=PATH_TO_CKPT)

        interpreter.allocate_tensors()

        # Get model details
        input_details = interpreter.get_input_details()
        output_details = interpreter.get_output_details()
        height = input_details[0]['shape'][1]
        width = input_details[0]['shape'][2]

        floating_model = (input_details[0]['dtype'] == np.float32)

        input_mean = 127.5
        input_std = 127.5

        # Load image and resize to expected shape [1xHxWx3]
        image = cv2.imread(img)
        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        imH, imW, _ = image.shape
        image_resized = cv2.resize(image_rgb, (height, width))
        input_data = np.expand_dims(image_resized, axis=0)

        # Perform the actual detection by running the model with the image as input
        interpreter.set_tensor(input_details[0]['index'], input_data)
        interpreter.invoke()

        # Retrieve detection results
        boxes = interpreter.get_tensor(output_details[0]['index'])[
            0]  # Bounding box coordinates of detected objects
        classes = interpreter.get_tensor(
            output_details[1]['index'])[0]  # Class index of detected objects
        scores = interpreter.get_tensor(
            output_details[2]['index'])[0]  # Confidence of detected objects

        # Loop over all detections and draw detection box if confidence is above minimum threshold
        for i in range(len(scores)):
            if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
                # Draw label
                object_name = labels[int(
                    classes[i]
                )]  # Look up object name from "labels" array using class index
                if object_name == 'person':  #if label is person stop looking and label it as a human
                    human = True
                    ymin = int(max(1, (boxes[i][0] * imH)))
                    xmin = int(max(1, (boxes[i][1] * imW)))
                    ymax = int(min(imH, (boxes[i][2] * imH)))
                    xmax = int(min(imW, (boxes[i][3] * imW)))
                    x = str(xmin + ((xmax - xmin) / 2))
                    y = str(ymin + ((ymax - ymin) / 2))
                    info = '(' + x + ',' + y + ')'
                    break
        if human == True:
            human_folder = '/home/pi/Images/humans/' + img[15:]
            rospy.Publisher("human", String, queue_size=10).publish(info)
            #shutil.move(human_folder)
            rospy.loginfo('Human Found!')

        else:
            rospy.Publisher("human", String,
                            queue_size=10).publish('No humans found')
            rospy.loginfo('No Human found')
Ejemplo n.º 25
0
    def frames():

        # configure arguments -- begin
        MODEL_NAME = 'Sample_TFlite_model'
        GRAPH_NAME = 'objectdetect.tflite'
        LABELMAP_NAME = 'labelmapobjectdetect.txt'
        min_conf_threshold = float(0.7)
        imW, imH = 640, 320
        # configure arguments --end

        # Import TensorFlow libraries
        # If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
        pkg = importlib.util.find_spec('tflite_runtime')
        if pkg:
            from tflite_runtime.interpreter import Interpreter
        else:
            from tensorflow.lite.python.interpreter import Interpreter

# Get path to current working directory
        CWD_PATH = os.getcwd()

        # Path to .tflite file, which contains the model that is used for object detection
        PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)

        # Path to label map file
        PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)

        # Load the label map
        with open(PATH_TO_LABELS, 'r') as f:
            labels = [line.strip() for line in f.readlines()]

# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
        if labels[0] == '???':
            del (labels[0])


# Load the Tensorflow Lite model.
        interpreter = Interpreter(model_path=PATH_TO_CKPT)

        interpreter.allocate_tensors()

        # Get model details
        input_details = interpreter.get_input_details()
        output_details = interpreter.get_output_details()
        height = input_details[0]['shape'][1]
        width = input_details[0]['shape'][2]

        floating_model = (input_details[0]['dtype'] == np.float32)

        input_mean = 127.5
        input_std = 127.5

        frame_rate_calc = 1
        freq = cv2.getTickFrequency()

        # Initialize video stream
        videostream = VideoStream(resolution=(imW, imH), framerate=30).start()
        time.sleep(1)

        #for frame1 in camera.capture_continuous(rawCapture, format="bgr",use_video_port=True):
        while True:
            t1 = cv2.getTickCount()
            # Grab frame from video stream
            frame1 = videostream.read()

            # Acquire frame and resize to expected shape [1xHxWx3]
            frame = frame1.copy()
            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            frame_resized = cv2.resize(frame_rgb, (width, height))
            input_data = np.expand_dims(frame_resized, axis=0)

            # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
            if floating_model:
                input_data = (np.float32(input_data) - input_mean) / input_std

            # Perform the actual detection by running the model with the image as input
            interpreter.set_tensor(input_details[0]['index'], input_data)
            interpreter.invoke()

            # Retrieve detection results
            boxes = interpreter.get_tensor(output_details[0]['index'])[
                0]  # Bounding box coordinates of detected objects
            classes = interpreter.get_tensor(output_details[1]['index'])[
                0]  # Class index of detected objects
            scores = interpreter.get_tensor(output_details[2]['index'])[
                0]  # Confidence of detected objects

            # boxes: [ymin, xmin, ymax, xmax]

            # Loop over all detections and draw detection box if confidence is above minimum threshold
            for i in range(len(scores)):
                if (scores[i] > min_conf_threshold) and (scores[i] < 1):
                    # Get bounding box coordinates and draw box
                    # Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
                    ymin = int(max(1, (boxes[i][0] * imH)))
                    xmin = int(max(1, (boxes[i][1] * imW)))
                    ymax = int(min(imH, (boxes[i][2] * imH)))
                    xmax = int(min(imW, (boxes[i][3] * imW)))

                    cv2.rectangle(frame, (xmin, ymin), (xmax, ymax),
                                  (10, 255, 0), 2)

                    # Draw label
                    object_name = labels[int(
                        classes[i]
                    )]  # Look up object name from "labels" array using class index
                    label = '%s: %d%%' % (object_name, int(scores[i] * 100)
                                          )  # Example: 'person: 72%'
                    labelSize, baseLine = cv2.getTextSize(
                        label, cv2.FONT_HERSHEY_SIMPLEX, 0.7,
                        2)  # Get font size
                    label_ymin = max(
                        ymin, labelSize[1] + 10
                    )  # Make sure not to draw label too close to top of window
                    cv2.rectangle(
                        frame, (xmin, label_ymin - labelSize[1] - 10),
                        (xmin + labelSize[0], label_ymin + baseLine - 10),
                        (255, 255, 255),
                        cv2.FILLED)  # Draw white box to put label text in
                    cv2.putText(frame, label, (xmin, label_ymin - 7),
                                cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0),
                                2)  # Draw label text

            # Draw framerate in corner of frame
            cv2.putText(frame, 'FPS: {0:.2f}'.format(frame_rate_calc),
                        (30, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0),
                        2, cv2.LINE_AA)

            # Calculate framerate
            t2 = cv2.getTickCount()
            time1 = (t2 - t1) / freq
            frame_rate_calc = 1 / time1

            # encode as a jpeg image and return it
            yield cv2.imencode('.jpg', frame)[1].tobytes()
Ejemplo n.º 26
0

def load_labels(filename):
    with open(filename, 'r') as f:
        return [line.strip() for line in f.readlines()]


t2 = time()
# print("Libraries loaded in ",t2-t1)
# cap=cv2.VideoCapture(0)
print(
    "1"
)  #Start camera instance.0 denotes the webcam (if present) else the first cam atteched
interpreter = Interpreter(model_path="detect.tflite")  #loading the model
interpreter.allocate_tensors()
input_details = interpreter.get_input_details(
)  #to help find out input format to be given(to model)
output_details = interpreter.get_output_details(
)  #to help find out output format of model
floating_model = input_details[0][
    'dtype'] == np.float32  #to check model type i.e Quantized model or Floating Point model
#print(input_details)
#print(output_details)
t1 = time()
# print("Initialized in ",t1-t2)
print("oki1")

while (True):

    t1 = time()
    # ret, frame=cap.read()
    frame = cv2.imread(
Ejemplo n.º 27
0
class Detector:
    """
    Perform object detection with the given model. The model is a quantized tflite
    file which if the detector can not find it at the path it will download it
    from neuralet repository automatically.

    :param config: Is a ConfigEngine instance which provides necessary parameters.
    """
    def __init__(self, config):
        self.config = config
        # Get the model name from the config
        self.model_name = self.config.get_section_dict('Detector')['Name']
        # Frames Per Second
        self.fps = None
        self.model_file = 'ped_ssd_mobilenet_v2_quantized_edgetpu.tflite'
        self.model_path = '/repo/data/edgetpu/' + self.model_file

        # Get the model .tflite file path from the config.
        # If there is no .tflite file in the path it will be downloaded automatically from base_url
        user_model_path = self.config.get_section_dict('Detector')['ModelPath']
        if len(user_model_path) > 0:
            print('using %s as model' % user_model_path)
            self.model_path = user_model_path
        else:
            base_url = 'https://media.githubusercontent.com/media/neuralet/neuralet-models/master/edge-tpu/'
            url = base_url + self.model_name + '/' + self.model_file

            if not os.path.isfile(self.model_path):
                print('model does not exist under: ', self.model_path,
                      'downloading from ', url)
                wget.download(url, self.model_path)

        # Load TFLite model and allocate tensors
        self.interpreter = Interpreter(
            self.model_path,
            experimental_delegates=[load_delegate("libedgetpu.so.1")])
        self.interpreter.allocate_tensors()
        # Get the model input and output tensor details
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()

        # Get class id from config
        self.class_id = int(
            self.config.get_section_dict('Detector')['ClassID'])
        self.score_threshold = float(
            self.config.get_section_dict('Detector')['MinScore'])

    def inference(self, resized_rgb_image):
        """
        inference function sets input tensor to input image and gets the output.
        The interpreter instance provides corresponding detection output which is used for creating result
        Args:
            resized_rgb_image: uint8 numpy array with shape (img_height, img_width, channels)

        Returns:
            result: a dictionary contains of [{"id": 0, "bbox": [x1, y1, x2, y2], "score":s%}, {...}, {...}, ...]
        """
        input_image = np.expand_dims(resized_rgb_image, axis=0)
        # Fill input tensor with input_image
        self.interpreter.set_tensor(self.input_details[0]["index"],
                                    input_image)
        t_begin = time.perf_counter()
        self.interpreter.invoke()
        inference_time = time.perf_counter() - t_begin  # Second
        self.fps = convert_infr_time_to_fps(inference_time)
        # The function `get_tensor()` returns a copy of the tensor data.
        # Use `tensor()` in order to get a pointer to the tensor.
        boxes = self.interpreter.get_tensor(self.output_details[0]['index'])
        labels = self.interpreter.get_tensor(self.output_details[1]['index'])
        scores = self.interpreter.get_tensor(self.output_details[2]['index'])
        # TODO: will be used for getting number of objects
        # num = self.interpreter.get_tensor(self.output_details[3]['index'])

        result = []
        for i in range(boxes.shape[1]):  # number of boxes
            if labels[0,
                      i] == self.class_id and scores[0,
                                                     i] > self.score_threshold:
                result.append({
                    "id": str(self.class_id) + '-' + str(i),
                    "bbox": boxes[0, i, :],
                    "score": scores[0, i]
                })

        return result
Ejemplo n.º 28
0
class PoseEngine():
    """Engine used for pose tasks."""
    def __init__(self, model_path, mirror=False):
        """Creates a PoseEngine with given model.

        Args:
          model_path: String, path to TF-Lite Flatbuffer file.
          mirror: Flip keypoints horizontally.

        Raises:
          ValueError: An error occurred when model output is invalid.
        """
        edgetpu_delegate = load_delegate(EDGETPU_SHARED_LIB)
        posenet_decoder_delegate = load_delegate(POSENET_SHARED_LIB)
        self._interpreter = Interpreter(model_path,
                                        experimental_delegates=[
                                            edgetpu_delegate,
                                            posenet_decoder_delegate
                                        ])
        self._interpreter.allocate_tensors()

        self._mirror = mirror

        self._input_tensor_shape = self.get_input_tensor_shape()
        if (self._input_tensor_shape.size != 4
                or self._input_tensor_shape[3] != 3
                or self._input_tensor_shape[0] != 1):
            raise ValueError(
                ('Image model should have input shape [1, height, width, 3]!'
                 ' This model has {}.'.format(self._input_tensor_shape)))
        _, self._input_height, self._input_width, self._input_depth = self.get_input_tensor_shape(
        )
        self._input_type = self._interpreter.get_input_details()[0]['dtype']
        self._inf_time = 0

    def run_inference(self, input_data):
        """Run inference using the zero copy feature from pycoral and returns inference time in ms.
        """
        start = time.monotonic()
        edgetpu.run_inference(self._interpreter, input_data)
        self._inf_time = time.monotonic() - start
        return (self._inf_time * 1000)

    def DetectPosesInImage(self, img):
        """Detects poses in a given image.

           For ideal results make sure the image fed to this function is close to the
           expected input size - it is the caller's responsibility to resize the
           image accordingly.

        Args:
          img: numpy array containing image
        """
        input_details = self._interpreter.get_input_details()
        image_width, image_height = img.size
        resized_image = img.resize((self._input_width, self._input_height),
                                   Image.NEAREST)
        input_data = np.expand_dims(resized_image, axis=0)
        if self._input_type is np.float32:
            # Floating point versions of posenet take image data in [-1,1] range.
            input_data = np.float32(resized_image) / 128.0 - 1.0
        else:
            # Assuming to be uint8
            input_data = np.asarray(resized_image)
        self.run_inference(input_data.flatten())
        return self.ParseOutput()

    def get_input_tensor_shape(self):
        """Returns input tensor shape."""
        return self._interpreter.get_input_details()[0]['shape']

    def get_output_tensor(self, idx):
        """Returns output tensor view."""
        return np.squeeze(
            self._interpreter.tensor(
                self._interpreter.get_output_details()[idx]['index'])())

    def ParseOutput(self):
        """Parses interpreter output tensors and returns decoded poses."""
        keypoints = self.get_output_tensor(0)
        keypoint_scores = self.get_output_tensor(1)
        pose_scores = self.get_output_tensor(2)
        num_poses = self.get_output_tensor(3)
        poses = []
        for i in range(int(num_poses)):
            pose_score = pose_scores[i]
            pose_keypoints = {}
            for j, point in enumerate(keypoints[i]):
                y, x = point
                if self._mirror:
                    y = self._input_width - y
                pose_keypoints[KeypointType(j)] = Keypoint(
                    Point(x, y), keypoint_scores[i, j])
            poses.append(Pose(pose_keypoints, pose_score))
        return poses, self._inf_time
Ejemplo n.º 29
0
#-----initialise the Model and Load into interpreter-------------------------

#specify the path of Model and Label file
model_path = "mobilenet_v1_1.0_224_quant.tflite"
label_path = "labels_mobilenet_quant_v1_224.txt"
top_k_results = 3

with open(label_path, 'r') as f:
    labels = list(map(str.strip, f.readlines()))

# Load TFLite model and allocate tensors
interpreter = Interpreter(model_path=model_path)
interpreter.allocate_tensors()

# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

## Get input size
input_shape = input_details[0]['shape']
#print(input_shape)
size = input_shape[:2] if len(input_shape) == 3 else input_shape[1:3]
#print(size)

#prediction threshold for triggering actions
threshold = 0.5
#-----------------------------------------------------------

#-------Window to display camera view---------------------
plt.ion()
plt.tight_layout()
Ejemplo n.º 30
0
class camera_interface():
    """
    The main interface for using the camera and determining the grip we need to be in.
    https://www.hackster.io/gatoninja236/scan-qr-codes-in-real-time-with-raspberry-pi-a5268b
      
    Attributes:
        count (int): Count of saved screenshots. File titles are frame'count'.jpg.
        cap (cv2 VideoCapture): The VideoCapture object.
        detector (QRCodeDetector): The QR Code detecting object.
    """

    def __init__(self,resolution=(640,480),framerate=30):
        self.count = 0
        # self.cap = cv2.VideoCapture(0)
        self.vs = VideoStream(resolution=(1280,720),framerate=30).start()
        # self.stream = cv2.VideoCapture(0)
        # ret = self.stream.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
        # ret = self.stream.set(3,resolution[0])
        # ret = self.stream.set(4,resolution[1])

        #Wait for the camera to startup for one seconds
        time.sleep(1)
        print("[INFO] Created video capture object")
        print("[INFO] loading model...")

        #Load the tflite model and labelmap
        # Get path to current working directory
        GRAPH_NAME = "detect.tflite"
        MODEL_NAME = "Camera_Interpreter/Coco"
        LABELMAP_NAME = "labelmap.txt"
        CWD_PATH = os.getcwd()

        # Path to .tflite file, which contains the model that is used for object detection
        PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,GRAPH_NAME)

        # Path to label map file
        PATH_TO_LABELS = os.path.join(CWD_PATH,MODEL_NAME,LABELMAP_NAME)

        # Load the label map
        with open(PATH_TO_LABELS, 'r') as f:
            self.labels = [line.strip() for line in f.readlines()]

        # Have to do a weird fix for label map if using the COCO "starter model" from
        # https://www.tensorflow.org/lite/models/object_detection/overview
        # First label is '???', which has to be removed.
        if self.labels[0] == '???':
            del(self.labels[0])

        # Load the Tensorflow Lite model.
        # If using Edge TPU, use special load_delegate argument
        use_TPU = False
        if use_TPU:
            self.interpreter = Interpreter(model_path=PATH_TO_CKPT,
                                    experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
            print(PATH_TO_CKPT)
        else:
            self.interpreter = Interpreter(model_path=PATH_TO_CKPT)

        self.interpreter.allocate_tensors()

        # Get model details
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()
        self.height = self.input_details[0]['shape'][1]
        self.width = self.input_details[0]['shape'][2]

        self.floating_model = (self.input_details[0]['dtype'] == np.float32)

        self.input_mean = 127.5
        self.input_std = 127.5
        
        # QR code detection object
        # self.detector = cv2.QRCodeDetector()
        self.cam_data = ""
        self.object_spotted = False
        self.test_count = 0
        self.killed_thread = False
        self.cam_image = None
        self.cam_image_index = 0
        self.object_spotted_T0 = 0
        self.object_not_spotted_delta_req = 3

        #Initialize the paused flag to false
        self.temp_pause = False

    def camera_read_threader(self):
        #Start the read cam thread
        read_cam = threading.Thread(target=self.read_cam_thread, args=())
        read_cam.start()
        while(self.cam_image_index == 0):
            time.sleep(0.05)
        #Start the image decode thread
        decoder = threading.Thread(target=self.decode_image_thread, args=())
        decoder.start()
        while not self.killed_thread and read_cam.is_alive() and decoder.is_alive():
            time.sleep(0.25)
        #Flag is thrown or error, so ensure flag is thrown and wait for threads to join
        self.killed_thread = True
        read_cam.join()
        decoder.join()

    def decode_image_thread(self):
        previous_index = None
        while not self.killed_thread:
            #Detect and decode the stored image if it's ready
            # t = time.time()
            if(previous_index != self.cam_image_index and (not self.temp_pause)):
                previous_index = self.cam_image_index
                # data, _, _ = self.detector.detectAndDecode(self.cam_image) Deprecated QR Code reader
                data, score = self.detect_main_object(self.cam_image)
                # print("[INFO] Camera objects: " + data)
                # if(data not in grips._value2member_map_):
                #     data = grips.openGrip.value

                #If the camera sees an object, skip the time requirement
                if(data != ""):
                    self.cam_data = data
                    self.object_spotted_T0 = time.time()
                    self.object_spotted = True
                #If the camera doesn't see an object, require a delay before reporting nothing
                else:
                    if((time.time() - self.object_spotted_T0) > self.object_not_spotted_delta_req):
                        # print("[DEBUG] Delta Req passed; reporting no object now")
                        self.cam_data = data
                        self.object_spotted = False
                
                #####No sleep since detecting/decoding takes significant time, just do it as fast as possible
            # print("[INFO] Time to decode image: " + (str(time.time() - t)))
            time.sleep(0.01)

    def detect_main_object(self, frame1):
        min_conf_threshold = 0.35

        # Acquire frame and resize to expected shape [1xHxWx3]
        frame = frame1.copy()
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame_resized = cv2.resize(frame_rgb, (self.width, self.height))
        input_data = np.expand_dims(frame_resized, axis=0)

        # Normalize pixel values if using a floating model (i.e. if model is non-quantized)
        if self.floating_model:
            input_data = (np.float32(input_data) - self.input_mean) / self.input_std

        # Perform the actual detection by running the model with the image as input
        self.interpreter.set_tensor(self.input_details[0]['index'],input_data)
        self.interpreter.invoke()

        # Retrieve detection results
        # boxes = self.interpreter.get_tensor(self.output_details[0]['index'])[0] # Bounding box coordinates of detected objects
        classes = self.interpreter.get_tensor(self.output_details[1]['index'])[0] # Class index of detected objects
        scores = self.interpreter.get_tensor(self.output_details[2]['index'])[0] # Confidence of detected objects

        highest_scoring_label = ""
        highest_score = 0
        for i in range(len(scores)):
            object_name = self.labels[int(classes[i])] # Look up object name from "labels" array using class index
            if((scores[i] > min_conf_threshold) and (scores[i] <= 1.0) and (scores[i] > highest_score) and (object_name in grips._value2member_map_)):
                # Draw label
                highest_scoring_label = object_name
                highest_score = scores[i]

        return (highest_scoring_label, highest_score)

    def read_cam_thread(self):
        while not self.killed_thread:
            if(not self.temp_pause):
                # t = time.time()
                #Get camera image, rescale, and store in class variable
                frame = self.vs.read()
                self.cam_image = imutils.resize(frame, width=400)
                
                #Increase index by 1
                self.cam_image_index += 1
                #Pause temply
                time.sleep(0.2)
                # print("Time to save/resize new image: " + (str(time.time() - t)))

    # def read_cam(self):
    #     # get the image
    #     _, img = self.cap.read() #TODO: #14 Downscale the resolution for faster processing
    #     # get bounding box coords and data
    #     data, bbox, _ = self.detector.detectAndDecode(img)
    #     #Define a parameter we can easily read later if anything is detected
    #     is_object = False
    #     #Update parameter/output the data we found, if any
    #     if data:
    #         #print("data found: ", data)
    #         is_object = True
    #     #return the information we got from the camera
    #     # cv2.imwrite("frame1.jpg", img)     # save frame as JPEG file
    #     return data, bbox, img, is_object

    # def read_cam_display_out(self):
    #     #Call the standard method to get the qr data / bounding box
    #     data, bbox, img, _ = self.read_cam()
    #     # if there is a bounding box, draw one, along with the data
    #     if(bbox is not None):
    #         for i in range(len(bbox)):
    #             cv2.line(img, tuple(bbox[i][0]), tuple(bbox[(i+1) % len(bbox)][0]), color=(255,
    #                     0, 255), thickness=2)
    #         cv2.putText(img, data, (int(bbox[0][0][0]), int(bbox[0][0][1]) - 10), cv2.FONT_HERSHEY_SIMPLEX,
    #                     0.5, (0, 255, 0), 2)
    #         #if data:
    #             #print("data found: ", data)
    #     # display the image preview
    #     cv2.imshow("code detector", img)

    #     # save the image
    #     cv2.imwrite("frame1.jpg", img)     # save frame as JPEG file
    #     #self.count += 1

    def end_camera_session(self):
        #Stop the camera thread 
        self.killed_thread = True
        time.sleep(0.1)
        #Release the camera object
        self.vs.stop()