def GetMediaStreamMessageResponse(self, predictions, imgShape, confidenceThreshold=0.1):
        try:
            msg = extension_pb2.MediaStreamMessage()

            ih, iw, _ = imgShape

            for prediction in predictions:
                confidenceScore = prediction['probability']
                if confidenceScore >= confidenceThreshold:
                    objectLabel = prediction['tagName']

                    inference = msg.media_sample.inferences.add()
                    inference.type = inferencing_pb2.Inference.InferenceType.ENTITY
                    inference.entity.CopyFrom(inferencing_pb2.Entity(
                        tag=inferencing_pb2.Tag(
                            value=objectLabel,
                            confidence=confidenceScore
                        ),
                        box=inferencing_pb2.Rectangle(
                            l=prediction['boundingBox']['left'],
                            t=prediction['boundingBox']['top'],
                            w=prediction['boundingBox']['width'],
                            h=prediction['boundingBox']['height'],
                        )
                    )
                    )
            return msg
        except:
            PrintGetExceptionDetails()
            raise
Beispiel #2
0
    def GetMediaStreamMessageResponse(self, boxes, scores, indices, imgShape, confidenceThreshold=0.1):
        try:
            msg = extension_pb2.MediaStreamMessage()

            ih, iw, _ = imgShape

            for idx in indices[0]:
                confidenceScore = scores[tuple(idx)].tolist()
                if confidenceScore >= confidenceThreshold:
                    objectLabel = self._tYoloV3._labelList[idx[1].tolist()]

                    idxTuple = (idx[0], idx[2])
                    ymin, xmin, ymax, xmax = boxes[idxTuple].tolist()

                    inference = msg.media_sample.inferences.add()
                    inference.type = inferencing_pb2.Inference.InferenceType.ENTITY
                    inference.entity.CopyFrom(  inferencing_pb2.Entity(
                                                    tag = inferencing_pb2.Tag(
                                                        value = objectLabel,
                                                        confidence = confidenceScore
                                                    ),
                                                    box = inferencing_pb2.Rectangle(
                                                        l = xmin / iw,
                                                        t = ymin / ih,
                                                        w = (xmax - xmin) / iw,
                                                        h = (ymax - ymin) / ih,
                                                    )
                                                )
                                            )
            return msg
        except:
            PrintGetExceptionDetails()
            raise
Beispiel #3
0
    def process_images(self, mediaStreamMessage, rawBytes, size):
        # Read image raw bytes
        im = Image.frombytes('RGB', size, rawBytes.tobytes())
        draw = ImageDraw.Draw(im)
        imgBuf = io.BytesIO()
        im.save(imgBuf, format='JPEG')
        imgBytes = np.frombuffer(imgBuf.getvalue(), dtype=np.uint8)

        # Convert to grayscale
        cvGrayImage = cv.imdecode(imgBytes, cv.COLOR_BGR2RGB)
        grayBytes = cvGrayImage.tobytes()

        # Calculate intensity
        totalColor = cvGrayImage.sum()
        avgColor = totalColor / len(grayBytes)

        colorIntensity = 'dark' if avgColor < 127 else 'light'

        logging.info('Color intensity: {}'.format(colorIntensity))

        inference = mediaStreamMessage.media_sample.inferences.add()
        inference.subtype = 'colorIntensity'
        classification = inferencing_pb2.Classification(
            tag=inferencing_pb2.Tag(value=colorIntensity, confidence=1.0))
        inference.classification.CopyFrom(classification)

        return mediaStreamMessage
    def GetMediaStreamMessageResponse(self, bboxes, originalImageSize):
        try:
            msg = extension_pb2.MediaStreamMessage()
            ih, iw = originalImageSize

            for i, bbox in enumerate(bboxes):
                confidenceScore = bbox[4].tolist()
                objectLabel = self._YoloV4._labelList[int(bbox[5])]
                xmin, ymin, xmax, ymax = np.array(bbox[:4], dtype=np.int32)

                inference = msg.media_sample.inferences.add()
                inference.type = inferencing_pb2.Inference.InferenceType.ENTITY
                inference.entity.CopyFrom(
                    inferencing_pb2.Entity(tag=inferencing_pb2.Tag(
                        value=objectLabel, confidence=confidenceScore),
                                           box=inferencing_pb2.Rectangle(
                                               l=xmin / iw,
                                               t=ymin / ih,
                                               w=(xmax - xmin) / iw,
                                               h=(ymax - ymin) / ih,
                                           )))
            return msg
        except:
            PrintGetExceptionDetails()
            raise
    def GetDummyMediaStreamMessageResponse(self, dummyValue):
        try:
            ih = 480
            iw = 640

            msg = extension_pb2.MediaStreamMessage()
            inference = msg.media_sample.inferences.add()
            inference.type = inferencing_pb2.Inference.InferenceType.ENTITY
            inference.entity.CopyFrom(
                inferencing_pb2.Entity(tag=inferencing_pb2.Tag(
                    value=dummyValue, confidence=0.0),
                                       box=inferencing_pb2.Rectangle(l=0,
                                                                     t=0,
                                                                     w=iw,
                                                                     h=ih)))

            return msg

        except:
            PrintGetExceptionDetails()
            raise
    def ProcessMediaStream(self, requestIterator, context):
        # Below logic can be extended into multi-process (per CPU cores, i.e. in case using CPU inferencing)
        # For simplicity below, we use single process to handle gRPC clients

        # Auto increment counter. Increases per client requests
        responseSeqNum = 1

        # First message from the client is (must be) MediaStreamDescriptor
        mediaStreamMessageRequest = next(requestIterator)

        # Extract message IDs
        requestSeqNum = mediaStreamMessageRequest.sequence_number
        requestAckSeqNum = mediaStreamMessageRequest.ack_sequence_number

        # State object per client       
        clientState = State(mediaStreamMessageRequest.media_stream_descriptor)

        logging.info('Connection created with peer {0}.\nMediaStreamDescriptor:\n{1}'.format(context.peer(), clientState._mediaStreamDescriptor))
        logging.debug('[Received] SeqNum: {0:07d} | AckNum: {1}'.format(requestSeqNum, requestAckSeqNum))

        # First message response ...
        mediaStreamMessage = extension_pb2.MediaStreamMessage(
                                    sequence_number = responseSeqNum,
                                    ack_sequence_number = requestSeqNum,
                                    media_stream_descriptor = extension_pb2.MediaStreamDescriptor(
                                        media_descriptor = media_pb2.MediaDescriptor(
                                            timescale = clientState._mediaStreamDescriptor.media_descriptor.timescale
                                        )
                                    )
                                )
        yield mediaStreamMessage

        # Process rest of the MediaStream message sequence
        for mediaStreamMessageRequest in requestIterator:
            try:
                # Increment response counter, will be sent to client
                responseSeqNum += 1
                
                # Read request id, sent by client
                requestSeqNum = mediaStreamMessageRequest.sequence_number

                logging.debug('[Received] SeqNum: {0:07d}'.format(requestSeqNum))

                # Get media content bytes. (bytes sent over shared memory buffer, segment or inline to message)
                cvImage = self.GetCvImageFromRawBytes(clientState, mediaStreamMessageRequest.media_sample)

                if cvImage is None:
                    message = "Can't decode received bytes."
                    logging.info(message)
                    context.set_details(message)
                    context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
                    return

                if list(cvImage.shape[:2]) != self._tYoloV3.image_shape:
                    message = "Received an image of size {0}, but expected one of size {1}".format(cvImage.shape[:2], self._tYoloV3.image_shape)
                    context.set_details(message)
                    logging.info(message)
                    context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
                    return

                mediaStreamMessage = extension_pb2.MediaStreamMessage()

                # Check confidence
                allConfidenceReached = True
                for inference in mediaStreamMessageRequest.media_sample.inferences:
                    confidence = inference.entity.tag.confidence
                    if(confidence < self.inferenceConfidence):
                        allConfidenceReached = False
                        break
                
                if(len(mediaStreamMessageRequest.media_sample.inferences) > 0 and allConfidenceReached):
                    # Return acknowledge message
                    mediaStreamMessage.sequence_number = responseSeqNum
                    mediaStreamMessage.ack_sequence_number = requestSeqNum
                    mediaStreamMessage.media_sample.timestamp = mediaStreamMessageRequest.media_sample.timestamp
                    
                    for tinyYoloInference in mediaStreamMessageRequest.media_sample.inferences:
                        objectLabel = tinyYoloInference.entity.tag.value

                        if(self.objectTag is None):
                            self.objectTag = objectLabel

                        if(self.objectTag == objectLabel):
                            inference = mediaStreamMessage.media_sample.inferences.add()
                            inference.type = inferencing_pb2.Inference.InferenceType.ENTITY
                            inference.subtype = 'From upstream'
                            inference.entity.CopyFrom(inferencing_pb2.Entity(
                                                            tag = inferencing_pb2.Tag(
                                                                value = objectLabel,
                                                                confidence = tinyYoloInference.entity.tag.confidence
                                                            ),
                                                            box = inferencing_pb2.Rectangle(
                                                                l = tinyYoloInference.entity.box.l,
                                                                t = tinyYoloInference.entity.box.t,
                                                                w = tinyYoloInference.entity.box.w,
                                                                h = tinyYoloInference.entity.box.h,
                                                            )
                                                        )
                                                    )
                else:
                    # run inference
                    boxes, scores, indices = self._tYoloV3.Score(cvImage)

                    logging.debug('Detected {0} inferences'.format(len(indices)))

                    if DEBUG is not None:
                        self.CreateDebugOutput(requestSeqNum, cvImage, boxes, scores, indices)

                    # return inference result as MediaStreamMessage
                    mediaStreamMessage = self.GetMediaStreamMessageResponse(mediaStreamMessage, boxes, scores, indices, cvImage.shape, mediaStreamMessageRequest.media_sample.timestamp)

                    mediaStreamMessage.sequence_number = responseSeqNum
                    mediaStreamMessage.ack_sequence_number = requestSeqNum
                    mediaStreamMessage.media_sample.timestamp = mediaStreamMessageRequest.media_sample.timestamp

                # Check client connection state
                if context.is_active():
                    # yield response
                    yield mediaStreamMessage
                else:
                    break
            except:
                PrintGetExceptionDetails()
                
        logging.info('Connection closed with peer {0}.'.format(context.peer()))
Beispiel #7
0
    def get_lva_MediaStreamMessage(self, buffer, gst_lva_message, ih, iw):

        msg = extension_pb2.MediaStreamMessage()
        msg.ack_sequence_number = gst_lva_message.sequence_number
        msg.media_sample.timestamp = gst_lva_message.timestamp

        # # Retrieve batch metadata from the gst_buffer
        # # Note that pyds.gst_buffer_get_nvds_batch_meta() expects the
        # # C address of gst_buffer as input, which is obtained with hash(gst_buffer)
        batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(buffer))
        frame = batch_meta.frame_meta_list
        while frame is not None:
            try:
                # Note that frame.data needs a cast to pyds.NvDsFrameMeta
                # The casting is done by pyds.NvDsFrameMeta.cast()
                # The casting also keeps ownership of the underlying memory
                # in the C code, so the Python garbage collector will leave
                # it alone.
                frame_meta = pyds.NvDsFrameMeta.cast(frame.data)
                objInference = frame_meta.obj_meta_list
                frameWidth = frame_meta.source_frame_width
                frameHeight = frame_meta.source_frame_height

                inference = msg.media_sample.inferences.add()

                attributes = []
                obj_label = None
                obj_confidence = 0
                obj_left = 0
                obj_width = 0
                obj_top = 0
                obj_width = 0

                # iterate through objects
                while objInference is not None:
                    try:
                        # Casting objInference.data to pyds.NvDsObjectMeta
                        obj_meta = pyds.NvDsObjectMeta.cast(objInference.data)
                    except StopIteration:
                        break

                    rect_params = obj_meta.rect_params
                    top = int(rect_params.top)
                    left = int(rect_params.left)
                    width = int(rect_params.width)
                    height = int(rect_params.height)
                    obj_confidence = obj_meta.confidence
                    objLabel = obj_meta.obj_label
                    obj_label = objLabel

                    obj_left = left / iw
                    obj_top = top / ih
                    obj_width = width / iw
                    obj_height = height / ih

                    inference.type = inferencing_pb2.Inference.InferenceType.ENTITY
                    try:
                        objInference = objInference.next
                    except StopIteration:
                        break

                if obj_label is not None:
                    try:
                        entity = inferencing_pb2.Entity(
                            tag=inferencing_pb2.Tag(value=obj_label,
                                                    confidence=obj_confidence),
                            box=inferencing_pb2.Rectangle(l=obj_left,
                                                          t=obj_top,
                                                          w=obj_width,
                                                          h=obj_height))

                        for attr in attributes:
                            attribute = inferencing_pb2.Attribute(
                                name=attr[0],
                                value=attr[1],
                                confidence=attr[2])

                            entity.attributes.append(attribute)
                    except:
                        PrintGetExceptionDetails()

                    inference.entity.CopyFrom(entity)
            except StopIteration:
                break

            try:
                frame = frame.next
            except StopIteration:
                break

        return msg
Beispiel #8
0
	def get_lva_MediaStreamMessage(self, buffer, gst_lva_message, ih, iw):

		msg = extension_pb2.MediaStreamMessage()		
		msg.ack_sequence_number = gst_lva_message.sequence_number
		msg.media_sample.timestamp = gst_lva_message.timestamp
			
		# # Retrieve batch metadata from the gst_buffer
		# # Note that pyds.gst_buffer_get_nvds_batch_meta() expects the
		# # C address of gst_buffer as input, which is obtained with hash(gst_buffer)
		batch_meta = pyds.gst_buffer_get_nvds_batch_meta(hash(buffer))

		frame = batch_meta.frame_meta_list
		
		while frame is not None:
			try:
				# Note that frame.data needs a cast to pyds.NvDsFrameMeta
				# The casting is done by pyds.NvDsFrameMeta.cast()
				# The casting also keeps ownership of the underlying memory
				# in the C code, so the Python garbage collector will leave
				# it alone.
				frame_meta = pyds.NvDsFrameMeta.cast(frame.data)
				objInference = frame_meta.obj_meta_list
				frameWidth = frame_meta.source_frame_width
				frameHeight = frame_meta.source_frame_height
				# iterate through objects 
				while objInference is not None:
					try: 
						# Casting objInference.data to pyds.NvDsObjectMeta
						obj_meta=pyds.NvDsObjectMeta.cast(objInference.data)
					except StopIteration:
						break

					inference = msg.media_sample.inferences.add()	

					attributes = []
					obj_label = None
					obj_confidence = 0
					obj_left = 0
					obj_width = 0
					obj_top = 0
					obj_width = 0

					color = ''
					# Classification 
					attribute = None
					if(obj_meta.class_id == 0 and obj_meta.classifier_meta_list is not None):
						classifier_meta = obj_meta.classifier_meta_list
						while classifier_meta is not None:
							classifierItem = pyds.NvDsClassifierMeta.cast(classifier_meta.data)
							if(classifierItem is not None):
								label_meta = classifierItem.label_info_list
								while label_meta is not None:
									labelItem = pyds.NvDsLabelInfo.cast(label_meta.data)
									prob = round(labelItem.result_prob, 2)
									attrValue = labelItem.result_label

									attrName = 'unknown'
									if(classifierItem.unique_component_id == PGIE_CLASS_ID_VEHICLE_COLOR):
										attrName = 'color'
									else:
										attrName = 'type'

									attributes.append([attrName, attrValue, prob])
									
									try: 
										label_meta=label_meta.next
									except StopIteration:
										break

							try: 
								classifier_meta=classifier_meta.next
							except StopIteration:
								break
							
					rect_params=obj_meta.rect_params
					top=int(rect_params.top)
					left=int(rect_params.left)
					width=int(rect_params.width)
					height=int(rect_params.height)
					obj_confidence = obj_meta.confidence
					obj_label = obj_meta.obj_label
					
					obj_left = left / iw
					obj_top = top / ih
					obj_width = width/ iw
					obj_height = height / ih
					obj_id = None

					# Tracking: Active tracking bbox information
					if(self.trackinEnabled):
						obj_id = obj_meta.object_id
						obj_active_tracking = obj_meta.tracker_bbox_info
						tracking_coord = obj_active_tracking.org_bbox_coords
						if(tracking_coord is not None and tracking_coord.left > 0 and tracking_coord.width > 0 and tracking_coord.top > 0 and tracking_coord.height > 0):
							obj_left = tracking_coord.left / iw
							obj_top = tracking_coord.top / ih
							obj_width = tracking_coord.width/ iw
							obj_height = tracking_coord.height / ih

					inference.type = inferencing_pb2.Inference.InferenceType.ENTITY

					if obj_label is not None:
						try:
							entity = inferencing_pb2.Entity(
													tag = inferencing_pb2.Tag(
														value = obj_label,
														confidence = obj_confidence
													),
													box = inferencing_pb2.Rectangle(
														l = obj_left,
														t = obj_top,
														w = obj_width,
														h = obj_height
													)												
												)

							if(self.trackinEnabled and obj_id is not None):
								entity.id = str(obj_id)

							for attr in attributes:
								attribute = inferencing_pb2.Attribute(
									name = attr[0],
									value = attr[1],
									confidence = attr[2]
								)

								entity.attributes.append(attribute)
						except:
							PrintGetExceptionDetails()
										
						inference.entity.CopyFrom(entity)

					try: 
						objInference=objInference.next
					except StopIteration:
						break

			except StopIteration:
				break

			try:
				frame = frame.next
			except StopIteration:
				break

		return msg