def process_frame(self, frame: VideoFrame, _: float = DETECT_THRESHOLD) -> bool: while self.json_objects: metadata_pts = self.json_objects[0][ "timestamp"] + self.offset_timestamp timestamp_difference = abs(frame.video_meta().buffer.pts - metadata_pts) # A margin of error of 1000 nanoseconds # If the difference is greater than the margin of error: # If frame has a higher pts then the timestamp at the head of the list, # pop the head of the list for being outdated # If frame has a lower pts then the timestamp at the head of the list, # its still possible for the timestamp to come up, so break # Otherwise, assume this timestamp at the head of the list is accurate to that frame if timestamp_difference > 1000: if (frame.video_meta().buffer.pts - metadata_pts) > 0: self.json_objects.pop(0) continue break detected_objects = self.json_objects[0]["objects"] for indv_object in detected_objects: frame.add_region( indv_object["detection"]["bounding_box"]["x_min"], indv_object["detection"]["bounding_box"]["y_min"], indv_object["detection"]["bounding_box"]["x_max"] - \ indv_object["detection"]["bounding_box"]["x_min"], indv_object["detection"]["bounding_box"]["y_max"] - \ indv_object["detection"]["bounding_box"]["y_min"], indv_object["detection"]["label"], indv_object["detection"]["confidence"], True) self.json_objects.pop(0) break return True
def process_frame(frame: VideoFrame, threshold: float = DETECT_THRESHOLD) -> bool: width = frame.video_info().width height = frame.video_info().height for tensor in frame.tensors(): dims = tensor.dims() data = tensor.data() object_size = dims[-1] for i in range(dims[-2]): image_id = data[i * object_size + 0] label_id = data[i * object_size + 1] confidence = data[i * object_size + 2] x_min = int(data[i * object_size + 3] * width + 0.5) y_min = int(data[i * object_size + 4] * height + 0.5) x_max = int(data[i * object_size + 5] * width + 0.5) y_max = int(data[i * object_size + 6] * height + 0.5) if image_id != 0: break if confidence < threshold: continue roi = frame.add_region(x_min, y_min, x_max - x_min, y_max - y_min, str(label_id), confidence) return True
def frame_callback(frame: VideoFrame): with frame.data() as mat: for roi in frame.regions(): labels = [] rect = roi.rect() for tensor in roi.tensors(): data = tensor.data() if "align_fc3" == tensor.layer_name(): lm_color = (255, 0, 0) for i in range(0, len(data), 2): x = int(rect.x + rect.w * data[i]) y = int(rect.y + rect.h * data[i + 1]) cv2.circle(mat, (x, y), int(1 + 0.02 * rect.w), lm_color, -1) if "prob" == tensor.layer_name(): if data[1] > 0.5: labels.append("M") else: labels.append("F") if "age_conv3" == tensor.layer_name(): labels.append(str(int(data[0] * 100))) if "prob_emotion" == tensor.layer_name(): emotions = ["neutral", "happy", "sad", "surprise", "anger"] index = numpy.argmax(data) labels.append(emotions[index]) if labels: label = " ".join(labels) cv2.putText(mat, label, (rect.x, rect.y + rect.h + 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
def frame_callback(frame: VideoFrame): with frame.data() as mat: for roi in frame.regions(): labels = [] for tensor in roi: data = tensor.data() if "landmarks" in tensor.model_name(): lm_color = (255, 0, 0) for i in range(0, len(data), 2): x = int(roi.meta().x + roi.meta().w * data[i]) y = int(roi.meta().y + roi.meta().h * data[i + 1]) cv2.circle(mat, (x, y), int(1 + 0.02 * roi.meta().w), lm_color, -1) if "gender" in tensor.model_name() and "prob" in tensor.layer_name(): if data[1] > 0.5: labels.append("M") else: labels.append("F") elif "age" in tensor.layer_name(): labels.append(str(int(data[0] * 100))) elif "EmoNet" in tensor.model_name(): emotions = ["neutral", "happy", "sad", "surprise", "anger"] index = numpy.argmax(data) labels.append(emotions[index]) if labels: label = " ".join(labels) cv2.putText(mat, label, (roi.meta().x, roi.meta().y + roi.meta().h + 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
def draw_conf(frame: VideoFrame): with frame.data() as img: for roi in frame.regions(): rect = roi.rect() conf = roi.confidence() if rect: cv2.putText(img, f'{conf:.2f}', (rect.x, rect.y + rect.h + 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
def frame_callback(frame: VideoFrame): event_list = None if True: for i, detection in enumerate(frame.regions()): #print("number of tensors: ", len(detection.tensors()), " label: ", detection.label(), " roi_type: ", detection.meta().get_roi_type()) # , " object_id : ", detection.object_id() for j, tensor in enumerate(detection.tensors()): if "detection" in tensor.name(): # bbbox = (tensor["x_min"], tensor["y_min"], tensor["x_max"], tensor["y_max"]) # print(i, detection.meta().get_roi_type(), bbbox, tensor["confidence"]) pass elif "object_id" in tensor.name(): # print(i, tensor["id"]) pass elif "ocr" in tensor.name(): #print(i, "MONOTOSH: ", tensor) pass else: print(tensor.name()) if event_list is not None: #event publish logic #with frame.data() as mat: # cv2.imwrite("dump.jpg", mat) pass
def pad_probe_callback(pad, info): with util.GST_PAD_PROBE_INFO_BUFFER(info) as buffer: caps = pad.get_current_caps() frame = VideoFrame(buffer, caps=caps) frame_callback(frame) return Gst.PadProbeReturn.OK
def process_frame(self, frame: VideoFrame) -> bool: try: logger.debug('Python Bridge.process_frame() method invoked!') for message in frame.messages(): frame.remove_message(message) msg_dict = { "name": self._edgex_device, "cmd": self._edgex_command, "method": "get", self._edgex_resource: message } msg_dict = json.dumps(msg_dict) frame.add_message(msg_dict) logger.debug('Done!') except Exception: print_message("Error processing frame: {}".format( traceback.print_exc())) return True
def log_age(self, frame: VideoFrame) -> bool: for roi in frame.regions(): for tensor in roi.tensors(): if tensor.name() == 'detection': continue layer_name = tensor.layer_name() if 'age_conv3' == layer_name: self.log_file.write(tensor.label() + "\n") continue return True
def process_frame(frame: VideoFrame, threshold: float = DETECT_THRESHOLD) -> bool: global input_height, input_width, prob_threshold, iou_threshold width = frame.video_info().width height = frame.video_info().height objects = list() for tensor in frame.tensors(): dims = tensor.dims() data = tensor.data() layer_params = YoloParams(dims[2]) data = data.reshape(dims[0], dims[1], dims[2], dims[3]) objects += parse_yolo_region(data, (input_height, input_width), (height, width), layer_params, prob_threshold, is_proportional=False) objects = filter_objects(objects, iou_threshold, prob_threshold) with frame.data() as frame: for obj in objects: # Validation bbox of detected object obj['xmax'] = min(obj['xmax'], width) obj['ymax'] = min(obj['ymax'], height) obj['xmin'] = max(obj['xmin'], 0) obj['ymin'] = max(obj['ymin'], 0) color = (min(obj['class_id'] * 12.5, 255), min(obj['class_id'] * 7, 255), min(obj['class_id'] * 5, 255)) det_label = labels_map[obj['class_id']] if labels_map and len( labels_map) >= obj['class_id'] else str(obj['class_id']) cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2) cv2.putText( frame, "#" + det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + ' %', (obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) return True
def process_frame(frame: VideoFrame) -> bool: for roi in frame.regions(): for tensor in roi: if tensor.name() == 'detection': continue layer_name = tensor.layer_name() data = tensor.data() if 'age' in layer_name: tensor.set_label(str(int(data[0] * 100))) if 'gender' in tensor.model_name() and 'prob' in layer_name: tensor.set_label(" M " if data[1] > 0.5 else " F ") if 'EmoNet' in layer_name: emotions = ["neutral", "happy", "sad", "surprise", "anger"] tensor.set_label(emotions[data.index(max(data))]) return True
def process_frame(frame: VideoFrame) -> bool: for roi in frame.regions(): for tensor in roi.tensors(): if tensor.name() == 'detection': continue layer_name = tensor.layer_name() data = tensor.data() if 'age_conv3' == layer_name: tensor.set_label(str(int(data[0] * 100))) continue if 'prob' == layer_name: tensor.set_label(" M " if data[1] > 0.5 else " F ") continue if 'prob_emotion' == layer_name: emotions = ["neutral", "happy", "sad", "surprise", "anger"] tensor.set_label(emotions[data.index(max(data))]) continue return True
def frame_callback(frame: VideoFrame): color = (255, 0, 0) with frame.data() as mat: poses = process(frame) x_values = poses[0] y_values = poses[1] for person in range(len(x_values)): for i in range(len(x_values[0])): cv2.circle( mat, (int(x_values[person][i]), int(y_values[person][i])), 3, color, -1) for j in range(len(mapping)): start_idx = mapping[j][0] end_idx = mapping[j][1] cv2.line(mat, (int(x_values[person][start_idx]), int(y_values[person][start_idx])), (int(x_values[person][end_idx]), int(y_values[person][end_idx])), color, 3) return True
def draw_ts(frame: VideoFrame) -> bool: font = cv2.FONT_HERSHEY_SIMPLEX msgs = frame.messages() for m in msgs: data = json.loads(m) if start_ts is not None: data["ts"] = int(start_ts) + int(data["timestamp"] / 1000000000) else: data["ts"] = int(time.time()) frame_ts = data["ts"] frame.remove_message(m) frame.add_message(json.dumps(data)) with frame.data() as mat: cv2.putText(mat, str(frame_ts), (20, 20), font, 1, (255, 255, 255), 2) return True
def process_frame(self, frame: VideoFrame) -> bool: msgs = frame.messages() for m in msgs: data = json.loads(m) if self.ts is not None: ts = int(ciso8601.parse_datetime(self.ts).timestamp()) data["ts"] = int(ts) + int(data["timestamp"] / 1000000000) else: data["ts"] = int(time.time()) self.fs = data["ts"] frame.remove_message(m) frame.add_message(json.dumps(data)) # print(data) with frame.data() as mat: cv2.putText(mat, str(self.fs), (20, 20), font, 0.5, (255, 255, 255), 2) return True
# ============================================================================== import sys import json import time from collections import Counter import gi gi.require_version('Gst', '1.0') from gi.repository import Gst, GObject from gstgva import VideoFrame Gst.init(sys.argv) REGION_TENSOR = VideoFrame.create_labels_structure( ["LOW COUNT", "PERSON DETECTED FRAME DELETED"]) class BottleCount: def __init__(self, redact_person: bool = True): self.redact_person = redact_person self.item_count = Counter() def remove_regions(self, frame: VideoFrame): for _ in range(len(frame.regions())): frame.pop_region() def process_frame(self, frame: VideoFrame) -> bool: timestamp = int(round(time.time()*1000)) events = []
def process_frame(self, frame: VideoFrame) -> bool: timestamp = int(round(time.time()*1000)) events = [] new_counts = Counter() for detection in frame.regions(): new_counts[detection.meta().get_roi_type()] += 1 for key, count in new_counts.items(): if key in self.item_count: if count > self.item_count[key]: for x in range(0, count-self.item_count[key]): events.append({'event_time': timestamp, 'roi_action': 'ENTER', 'object_id': key}) elif count < self.item_count[key]: for x in range(0, self.item_count[key]-count): events.append({'event_time': timestamp, 'roi_action': 'DEPART', 'object_id': key}) else: for x in range(0, count): events.append({'event_time': timestamp, 'roi_action': 'ENTER', 'object_id': key}) for key, count in self.item_count.items(): if key not in new_counts: for x in range(0, count): events.append({'event_time': timestamp, 'roi_action': 'DEPART', 'object_id': key}) if events: frame.add_message(json.dumps(events)) self.item_count = new_counts if self.item_count['bottle'] <= 1: frame.add_region(0, 0, 0, 0, 0, region_tensor=REGION_TENSOR) else: frame.add_region(0, 0, 0, 0, 0, region_tensor=VideoFrame.create_labels_structure( ["Bottle Count: " + str(self.item_count['bottle'])])) if (self.item_count['person'] > 0) and self.redact_person: with frame.data() as contents: contents[:] = 0 self.remove_regions(frame) frame.add_region(0, 0, 0, 0, 1, region_tensor=REGION_TENSOR) return True
def detect_postproc_callback(pad, info): with util.GST_PAD_PROBE_INFO_BUFFER(info) as buffer: caps = pad.get_current_caps() frame = VideoFrame(buffer, caps=caps) status = ssd_object_detection.process_frame(frame) return Gst.PadProbeReturn.OK if status else Gst.PadProbeReturn.DROP
def frame_callback(self, frame: VideoFrame): for message in frame.messages(): m = json.loads(message) #if iot_hub_manager is not None: iot_hub_manager.send_message_to_upstream(json.dumps(message)) print(m)
# # SPDX-License-Identifier: MIT # ============================================================================== import sys import gi gi.require_version('Gst', '1.0') from gi.repository import Gst, GObject from gstgva import VideoFrame DETECT_THRESHOLD = 0.5 Gst.init(sys.argv) REGION_TENSOR = VideoFrame.create_labels_structure( ["background", "face", "my_label_2", "etc."]) def process_frame(frame: VideoFrame, threshold: float = DETECT_THRESHOLD) -> bool: width = frame.video_info().width height = frame.video_info().height for tensor in frame.tensors(): dims = tensor.dims() data = tensor.data() object_size = dims[-1] for i in range(dims[-2]): image_id = data[i * object_size + 0] confidence = data[i * object_size + 2] x_min = int(data[i * object_size + 3] * width + 0.5)
def classify_postproc_callback(pad, info): with util.GST_PAD_PROBE_INFO_BUFFER(info) as buffer: caps = pad.get_current_caps() frame = VideoFrame(buffer, caps=caps) status = age_gend_class_process_frame(frame) return Gst.PadProbeReturn.OK if status else Gst.PadProbeReturn.DROP
def remove_regions(self, frame: VideoFrame): for _ in range(len(frame.regions())): frame.pop_region()