def process_frame(self,
frame: VideoFrame,
_: float = DETECT_THRESHOLD) -> bool:
while self.json_objects:
metadata_pts = self.json_objects[0][
"timestamp"] + self.offset_timestamp
timestamp_difference = abs(frame.video_meta().buffer.pts -
metadata_pts)
# A margin of error of 1000 nanoseconds
# If the difference is greater than the margin of error:
# If frame has a higher pts then the timestamp at the head of the list,
# pop the head of the list for being outdated
# If frame has a lower pts then the timestamp at the head of the list,
# its still possible for the timestamp to come up, so break
# Otherwise, assume this timestamp at the head of the list is accurate to that frame
if timestamp_difference > 1000:
if (frame.video_meta().buffer.pts - metadata_pts) > 0:
self.json_objects.pop(0)
continue
break
detected_objects = self.json_objects[0]["objects"]
for indv_object in detected_objects:
frame.add_region(
indv_object["detection"]["bounding_box"]["x_min"],
indv_object["detection"]["bounding_box"]["y_min"],
indv_object["detection"]["bounding_box"]["x_max"] - \
indv_object["detection"]["bounding_box"]["x_min"],
indv_object["detection"]["bounding_box"]["y_max"] - \
indv_object["detection"]["bounding_box"]["y_min"],
indv_object["detection"]["label"],
indv_object["detection"]["confidence"],
True)
self.json_objects.pop(0)
break
return True
def process_frame(frame: VideoFrame, threshold: float = DETECT_THRESHOLD) -> bool:
width = frame.video_info().width
height = frame.video_info().height
for tensor in frame.tensors():
dims = tensor.dims()
data = tensor.data()
object_size = dims[-1]
for i in range(dims[-2]):
image_id = data[i * object_size + 0]
label_id = data[i * object_size + 1]
confidence = data[i * object_size + 2]
x_min = int(data[i * object_size + 3] * width + 0.5)
y_min = int(data[i * object_size + 4] * height + 0.5)
x_max = int(data[i * object_size + 5] * width + 0.5)
y_max = int(data[i * object_size + 6] * height + 0.5)
if image_id != 0:
break
if confidence < threshold:
continue
roi = frame.add_region(x_min, y_min, x_max - x_min, y_max - y_min, str(label_id), confidence)
return True
def frame_callback(frame: VideoFrame):
with frame.data() as mat:
for roi in frame.regions():
labels = []
rect = roi.rect()
for tensor in roi.tensors():
data = tensor.data()
if "align_fc3" == tensor.layer_name():
lm_color = (255, 0, 0)
for i in range(0, len(data), 2):
x = int(rect.x + rect.w * data[i])
y = int(rect.y + rect.h * data[i + 1])
cv2.circle(mat, (x, y), int(1 + 0.02 * rect.w),
lm_color, -1)
if "prob" == tensor.layer_name():
if data[1] > 0.5:
labels.append("M")
else:
labels.append("F")
if "age_conv3" == tensor.layer_name():
labels.append(str(int(data[0] * 100)))
if "prob_emotion" == tensor.layer_name():
emotions = ["neutral", "happy", "sad", "surprise", "anger"]
index = numpy.argmax(data)
labels.append(emotions[index])
if labels:
label = " ".join(labels)
cv2.putText(mat, label, (rect.x, rect.y + rect.h + 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
def frame_callback(frame: VideoFrame):
with frame.data() as mat:
for roi in frame.regions():
labels = []
for tensor in roi:
data = tensor.data()
if "landmarks" in tensor.model_name():
lm_color = (255, 0, 0)
for i in range(0, len(data), 2):
x = int(roi.meta().x + roi.meta().w * data[i])
y = int(roi.meta().y + roi.meta().h * data[i + 1])
cv2.circle(mat, (x, y), int(1 + 0.02 * roi.meta().w), lm_color, -1)
if "gender" in tensor.model_name() and "prob" in tensor.layer_name():
if data[1] > 0.5:
labels.append("M")
else:
labels.append("F")
elif "age" in tensor.layer_name():
labels.append(str(int(data[0] * 100)))
elif "EmoNet" in tensor.model_name():
emotions = ["neutral", "happy", "sad", "surprise", "anger"]
index = numpy.argmax(data)
labels.append(emotions[index])
if labels:
label = " ".join(labels)
cv2.putText(mat, label, (roi.meta().x, roi.meta().y + roi.meta().h + 30), cv2.FONT_HERSHEY_SIMPLEX,
1, (0, 0, 255), 2)
def draw_conf(frame: VideoFrame):
with frame.data() as img:
for roi in frame.regions():
rect = roi.rect()
conf = roi.confidence()
if rect:
cv2.putText(img, f'{conf:.2f}', (rect.x, rect.y + rect.h + 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
def frame_callback(frame: VideoFrame):
event_list = None
if True:
for i, detection in enumerate(frame.regions()):
#print("number of tensors: ", len(detection.tensors()), " label: ", detection.label(), " roi_type: ", detection.meta().get_roi_type())
# , " object_id : ", detection.object_id()
for j, tensor in enumerate(detection.tensors()):
if "detection" in tensor.name():
# bbbox = (tensor["x_min"], tensor["y_min"], tensor["x_max"], tensor["y_max"])
# print(i, detection.meta().get_roi_type(), bbbox, tensor["confidence"])
pass
elif "object_id" in tensor.name():
# print(i, tensor["id"])
pass
elif "ocr" in tensor.name():
#print(i, "MONOTOSH: ", tensor)
pass
else:
print(tensor.name())
if event_list is not None:
#event publish logic
#with frame.data() as mat:
# cv2.imwrite("dump.jpg", mat)
pass
def pad_probe_callback(pad, info):
with util.GST_PAD_PROBE_INFO_BUFFER(info) as buffer:
caps = pad.get_current_caps()
frame = VideoFrame(buffer, caps=caps)
frame_callback(frame)
return Gst.PadProbeReturn.OK
def process_frame(self, frame: VideoFrame) -> bool:
try:
logger.debug('Python Bridge.process_frame() method invoked!')
for message in frame.messages():
frame.remove_message(message)
msg_dict = {
"name": self._edgex_device,
"cmd": self._edgex_command,
"method": "get",
self._edgex_resource: message
}
msg_dict = json.dumps(msg_dict)
frame.add_message(msg_dict)
logger.debug('Done!')
except Exception:
print_message("Error processing frame: {}".format(
traceback.print_exc()))
return True
def log_age(self, frame: VideoFrame) -> bool:
for roi in frame.regions():
for tensor in roi.tensors():
if tensor.name() == 'detection':
continue
layer_name = tensor.layer_name()
if 'age_conv3' == layer_name:
self.log_file.write(tensor.label() + "\n")
continue
return True
def process_frame(frame: VideoFrame,
threshold: float = DETECT_THRESHOLD) -> bool:
global input_height, input_width, prob_threshold, iou_threshold
width = frame.video_info().width
height = frame.video_info().height
objects = list()
for tensor in frame.tensors():
dims = tensor.dims()
data = tensor.data()
layer_params = YoloParams(dims[2])
data = data.reshape(dims[0], dims[1], dims[2], dims[3])
objects += parse_yolo_region(data, (input_height, input_width),
(height, width),
layer_params,
prob_threshold,
is_proportional=False)
objects = filter_objects(objects, iou_threshold, prob_threshold)
with frame.data() as frame:
for obj in objects:
# Validation bbox of detected object
obj['xmax'] = min(obj['xmax'], width)
obj['ymax'] = min(obj['ymax'], height)
obj['xmin'] = max(obj['xmin'], 0)
obj['ymin'] = max(obj['ymin'], 0)
color = (min(obj['class_id'] * 12.5,
255), min(obj['class_id'] * 7,
255), min(obj['class_id'] * 5, 255))
det_label = labels_map[obj['class_id']] if labels_map and len(
labels_map) >= obj['class_id'] else str(obj['class_id'])
cv2.rectangle(frame, (obj['xmin'], obj['ymin']),
(obj['xmax'], obj['ymax']), color, 2)
cv2.putText(
frame, "#" + det_label + ' ' +
str(round(obj['confidence'] * 100, 1)) + ' %',
(obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6,
color, 1)
return True
def process_frame(frame: VideoFrame) -> bool:
for roi in frame.regions():
for tensor in roi:
if tensor.name() == 'detection':
continue
layer_name = tensor.layer_name()
data = tensor.data()
if 'age' in layer_name:
tensor.set_label(str(int(data[0] * 100)))
if 'gender' in tensor.model_name() and 'prob' in layer_name:
tensor.set_label(" M " if data[1] > 0.5 else " F ")
if 'EmoNet' in layer_name:
emotions = ["neutral", "happy", "sad", "surprise", "anger"]
tensor.set_label(emotions[data.index(max(data))])
return True
def process_frame(frame: VideoFrame) -> bool:
for roi in frame.regions():
for tensor in roi.tensors():
if tensor.name() == 'detection':
continue
layer_name = tensor.layer_name()
data = tensor.data()
if 'age_conv3' == layer_name:
tensor.set_label(str(int(data[0] * 100)))
continue
if 'prob' == layer_name:
tensor.set_label(" M " if data[1] > 0.5 else " F ")
continue
if 'prob_emotion' == layer_name:
emotions = ["neutral", "happy", "sad", "surprise", "anger"]
tensor.set_label(emotions[data.index(max(data))])
continue
return True
def frame_callback(frame: VideoFrame):
color = (255, 0, 0)
with frame.data() as mat:
poses = process(frame)
x_values = poses[0]
y_values = poses[1]
for person in range(len(x_values)):
for i in range(len(x_values[0])):
cv2.circle(
mat, (int(x_values[person][i]), int(y_values[person][i])),
3, color, -1)
for j in range(len(mapping)):
start_idx = mapping[j][0]
end_idx = mapping[j][1]
cv2.line(mat, (int(x_values[person][start_idx]),
int(y_values[person][start_idx])),
(int(x_values[person][end_idx]),
int(y_values[person][end_idx])), color, 3)
return True
def draw_ts(frame: VideoFrame) -> bool:
font = cv2.FONT_HERSHEY_SIMPLEX
msgs = frame.messages()
for m in msgs:
data = json.loads(m)
if start_ts is not None:
data["ts"] = int(start_ts) + int(data["timestamp"] / 1000000000)
else:
data["ts"] = int(time.time())
frame_ts = data["ts"]
frame.remove_message(m)
frame.add_message(json.dumps(data))
with frame.data() as mat:
cv2.putText(mat, str(frame_ts), (20, 20), font, 1, (255, 255, 255), 2)
return True
def process_frame(self, frame: VideoFrame) -> bool:
msgs = frame.messages()
for m in msgs:
data = json.loads(m)
if self.ts is not None:
ts = int(ciso8601.parse_datetime(self.ts).timestamp())
data["ts"] = int(ts) + int(data["timestamp"] / 1000000000)
else:
data["ts"] = int(time.time())
self.fs = data["ts"]
frame.remove_message(m)
frame.add_message(json.dumps(data))
# print(data)
with frame.data() as mat:
cv2.putText(mat, str(self.fs), (20, 20), font, 0.5,
(255, 255, 255), 2)
return True
# ==============================================================================
import sys
import json
import time
from collections import Counter
import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst, GObject
from gstgva import VideoFrame
Gst.init(sys.argv)
REGION_TENSOR = VideoFrame.create_labels_structure(
["LOW COUNT", "PERSON DETECTED FRAME DELETED"])
class BottleCount:
def __init__(self, redact_person: bool = True):
self.redact_person = redact_person
self.item_count = Counter()
def remove_regions(self, frame: VideoFrame):
for _ in range(len(frame.regions())):
frame.pop_region()
def process_frame(self, frame: VideoFrame) -> bool:
timestamp = int(round(time.time()*1000))
events = []
def process_frame(self, frame: VideoFrame) -> bool:
timestamp = int(round(time.time()*1000))
events = []
new_counts = Counter()
for detection in frame.regions():
new_counts[detection.meta().get_roi_type()] += 1
for key, count in new_counts.items():
if key in self.item_count:
if count > self.item_count[key]:
for x in range(0, count-self.item_count[key]):
events.append({'event_time': timestamp,
'roi_action': 'ENTER',
'object_id': key})
elif count < self.item_count[key]:
for x in range(0, self.item_count[key]-count):
events.append({'event_time': timestamp,
'roi_action': 'DEPART',
'object_id': key})
else:
for x in range(0, count):
events.append({'event_time': timestamp,
'roi_action': 'ENTER',
'object_id': key})
for key, count in self.item_count.items():
if key not in new_counts:
for x in range(0, count):
events.append({'event_time': timestamp,
'roi_action': 'DEPART',
'object_id': key})
if events:
frame.add_message(json.dumps(events))
self.item_count = new_counts
if self.item_count['bottle'] <= 1:
frame.add_region(0, 0, 0, 0, 0, region_tensor=REGION_TENSOR)
else:
frame.add_region(0, 0, 0, 0, 0, region_tensor=VideoFrame.create_labels_structure(
["Bottle Count: " + str(self.item_count['bottle'])]))
if (self.item_count['person'] > 0) and self.redact_person:
with frame.data() as contents:
contents[:] = 0
self.remove_regions(frame)
frame.add_region(0, 0, 0, 0, 1, region_tensor=REGION_TENSOR)
return True
def detect_postproc_callback(pad, info):
with util.GST_PAD_PROBE_INFO_BUFFER(info) as buffer:
caps = pad.get_current_caps()
frame = VideoFrame(buffer, caps=caps)
status = ssd_object_detection.process_frame(frame)
return Gst.PadProbeReturn.OK if status else Gst.PadProbeReturn.DROP
def frame_callback(self, frame: VideoFrame):
for message in frame.messages():
m = json.loads(message)
#if iot_hub_manager is not None:
iot_hub_manager.send_message_to_upstream(json.dumps(message))
print(m)
#
# SPDX-License-Identifier: MIT
# ==============================================================================
import sys
import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst, GObject
from gstgva import VideoFrame
DETECT_THRESHOLD = 0.5
Gst.init(sys.argv)
REGION_TENSOR = VideoFrame.create_labels_structure(
["background", "face", "my_label_2", "etc."])
def process_frame(frame: VideoFrame,
threshold: float = DETECT_THRESHOLD) -> bool:
width = frame.video_info().width
height = frame.video_info().height
for tensor in frame.tensors():
dims = tensor.dims()
data = tensor.data()
object_size = dims[-1]
for i in range(dims[-2]):
image_id = data[i * object_size + 0]
confidence = data[i * object_size + 2]
x_min = int(data[i * object_size + 3] * width + 0.5)
def classify_postproc_callback(pad, info):
with util.GST_PAD_PROBE_INFO_BUFFER(info) as buffer:
caps = pad.get_current_caps()
frame = VideoFrame(buffer, caps=caps)
status = age_gend_class_process_frame(frame)
return Gst.PadProbeReturn.OK if status else Gst.PadProbeReturn.DROP
def remove_regions(self, frame: VideoFrame):
for _ in range(len(frame.regions())):
frame.pop_region()
