def get_current_frame(self, draw=False):
with self.current_frame_lock:
frame_copy = np.copy(self._current_frame)
frame_time = self.current_frame_time
tracked_objects = copy.deepcopy(self.tracked_objects)
frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_YUV2BGR_I420)
# draw on the frame
if draw:
# draw the bounding boxes on the frame
for obj in tracked_objects.values():
thickness = 2
color = COLOR_MAP[obj['label']]
if obj['frame_time'] != frame_time:
thickness = 1
color = (255,0,0)
# draw the bounding boxes on the frame
box = obj['box']
draw_box_with_label(frame_copy, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
# draw the regions on the frame
region = obj['region']
cv2.rectangle(frame_copy, (region[0], region[1]), (region[2], region[3]), (0,255,0), 1)
if self.config['snapshots']['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(frame_copy, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
if self.config['snapshots']['draw_zones']:
for name, zone in self.config['zones'].items():
thickness = 8 if any([name in obj['zones'] for obj in tracked_objects.values()]) else 2
cv2.drawContours(frame_copy, [zone['contour']], -1, zone['color'], thickness)
return frame_copy
def snapshot(camera, obj):
if not 'frame' in obj:
return
best_frame = cv2.cvtColor(obj['frame'], cv2.COLOR_YUV2BGR_I420)
if self.camera_config[camera]['snapshots']['draw_bounding_boxes']:
thickness = 2
color = COLOR_MAP[obj['label']]
box = obj['box']
draw_box_with_label(best_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
mqtt_config = self.camera_config[camera].get('mqtt', {'crop_to_region': False})
if mqtt_config.get('crop_to_region'):
region = obj['region']
best_frame = best_frame[region[1]:region[3], region[0]:region[2]]
if 'snapshot_height' in mqtt_config:
height = int(mqtt_config['snapshot_height'])
width = int(height*best_frame.shape[1]/best_frame.shape[0])
best_frame = cv2.resize(best_frame, dsize=(width, height), interpolation=cv2.INTER_AREA)
if self.camera_config[camera]['snapshots']['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
size = cv2.getTextSize(time_to_show, cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, thickness=2)
text_width = size[0][0]
text_height = size[0][1]
desired_size = max(200, 0.33*best_frame.shape[1])
font_scale = desired_size/text_width
cv2.putText(best_frame, time_to_show, (5, best_frame.shape[0]-7), cv2.FONT_HERSHEY_SIMPLEX, fontScale=font_scale, color=(255, 255, 255), thickness=2)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(f"{self.topic_prefix}/{camera}/{obj['label']}/snapshot", jpg_bytes, retain=True)
def save_debug_frame(self, debug_path, frame_time, tracked_objects):
current_frame = cv2.cvtColor(
self.frame_manager.get(f"{self.camera_name}{frame_time}",
self.camera_config.frame_shape_yuv),
cv2.COLOR_YUV2BGR_I420,
)
# draw the bounding boxes on the frame
for obj in tracked_objects:
thickness = 2
color = (0, 0, 175)
if obj["frame_time"] != frame_time:
thickness = 1
color = (255, 0, 0)
else:
color = (255, 255, 0)
# draw the bounding boxes on the frame
box = obj["box"]
draw_box_with_label(
current_frame,
box[0],
box[1],
box[2],
box[3],
obj["id"],
f"{int(obj['score']*100)}% {int(obj['area'])}",
thickness=thickness,
color=color,
)
# draw the regions on the frame
region = obj["region"]
draw_box_with_label(
current_frame,
region[0],
region[1],
region[2],
region[3],
"region",
"",
thickness=1,
color=(0, 255, 0),
)
cv2.imwrite(
f"{os.path.join(debug_path, os.path.basename(self.clip_path))}.{int(frame_time*1000000)}.jpg",
current_frame,
)
def save_debug_frame(self, debug_path, frame_time, tracked_objects):
current_frame = self.frame_manager.get(
f"{self.camera_name}{frame_time}", self.frame_shape)
# draw the bounding boxes on the frame
for obj in tracked_objects:
thickness = 2
color = (0, 0, 175)
if obj['frame_time'] != frame_time:
thickness = 1
color = (255, 0, 0)
else:
color = (255, 255, 0)
# draw the bounding boxes on the frame
box = obj['box']
draw_box_with_label(current_frame,
box[0],
box[1],
box[2],
box[3],
obj['label'],
f"{int(obj['score']*100)}% {int(obj['area'])}",
thickness=thickness,
color=color)
# draw the regions on the frame
region = obj['region']
draw_box_with_label(current_frame,
region[0],
region[1],
region[2],
region[3],
'region',
"",
thickness=1,
color=(0, 255, 0))
cv2.imwrite(
f"{os.path.join(debug_path, os.path.basename(self.clip_path))}.{int(frame_time*1000000)}.jpg",
cv2.cvtColor(current_frame, cv2.COLOR_RGB2BGR))
def update(self, frame_time, tracked_objects):
self.current_frame_time = frame_time
# get the new frame and delete the old frame
frame_id = f"{self.name}{frame_time}"
self.current_frame = self.frame_manager.get(frame_id)
if not self.previous_frame_id is None:
self.frame_manager.delete(self.previous_frame_id)
self.previous_frame_id = frame_id
current_ids = tracked_objects.keys()
previous_ids = self.tracked_objects.keys()
removed_ids = list(set(previous_ids).difference(current_ids))
new_ids = list(set(current_ids).difference(previous_ids))
updated_ids = list(set(current_ids).intersection(previous_ids))
for id in new_ids:
self.tracked_objects[id] = tracked_objects[id]
self.tracked_objects[id]['zones'] = []
# start the score history
self.tracked_objects[id]['score_history'] = [
self.tracked_objects[id]['score']
]
# calculate if this is a false positive
self.tracked_objects[id]['computed_score'] = self.compute_score(
self.tracked_objects[id])
self.tracked_objects[id]['false_positive'] = self.false_positive(
self.tracked_objects[id])
# call event handlers
for c in self.callbacks['start']:
c(self.name, tracked_objects[id])
for id in updated_ids:
self.tracked_objects[id].update(tracked_objects[id])
# if the object is not in the current frame, add a 0.0 to the score history
if self.tracked_objects[id][
'frame_time'] != self.current_frame_time:
self.tracked_objects[id]['score_history'].append(0.0)
else:
self.tracked_objects[id]['score_history'].append(
self.tracked_objects[id]['score'])
# only keep the last 10 scores
if len(self.tracked_objects[id]['score_history']) > 10:
self.tracked_objects[id][
'score_history'] = self.tracked_objects[id][
'score_history'][-10:]
# calculate if this is a false positive
self.tracked_objects[id]['computed_score'] = self.compute_score(
self.tracked_objects[id])
self.tracked_objects[id]['false_positive'] = self.false_positive(
self.tracked_objects[id])
# call event handlers
for c in self.callbacks['update']:
c(self.name, self.tracked_objects[id])
for id in removed_ids:
# publish events to mqtt
self.tracked_objects[id]['end_time'] = frame_time
for c in self.callbacks['end']:
c(self.name, self.tracked_objects[id])
del self.tracked_objects[id]
# check to see if the objects are in any zones
for obj in self.tracked_objects.values():
current_zones = []
bottom_center = (obj['centroid'][0], obj['box'][3])
# check each zone
for name, zone in self.config['zones'].items():
contour = zone['contour']
# check if the object is in the zone and not filtered
if (cv2.pointPolygonTest(contour, bottom_center, False) >= 0
and not zone_filtered(obj, zone.get('filters', {}))):
current_zones.append(name)
obj['zones'] = current_zones
# draw on the frame
if not self.current_frame is None:
# draw the bounding boxes on the frame
for obj in self.tracked_objects.values():
thickness = 2
color = COLOR_MAP[obj['label']]
if obj['frame_time'] != frame_time:
thickness = 1
color = (255, 0, 0)
# draw the bounding boxes on the frame
box = obj['box']
draw_box_with_label(
self.current_frame,
box[0],
box[1],
box[2],
box[3],
obj['label'],
f"{int(obj['score']*100)}% {int(obj['area'])}",
thickness=thickness,
color=color)
# draw the regions on the frame
region = obj['region']
cv2.rectangle(self.current_frame, (region[0], region[1]),
(region[2], region[3]), (0, 255, 0), 1)
if self.config['snapshots']['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(
frame_time).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(self.current_frame,
time_to_show, (10, 30),
cv2.FONT_HERSHEY_SIMPLEX,
fontScale=.8,
color=(255, 255, 255),
thickness=2)
if self.config['snapshots']['draw_zones']:
for name, zone in self.config['zones'].items():
thickness = 8 if any([
name in obj['zones']
for obj in self.tracked_objects.values()
]) else 2
cv2.drawContours(self.current_frame, [zone['contour']], -1,
zone['color'], thickness)
# maintain best objects
for obj in self.tracked_objects.values():
object_type = obj['label']
# if the object wasn't seen on the current frame, skip it
if obj['frame_time'] != self.current_frame_time or obj[
'false_positive']:
continue
obj_copy = copy.deepcopy(obj)
if object_type in self.best_objects:
current_best = self.best_objects[object_type]
now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score
# or the current object is older than desired, use the new object
if obj_copy['score'] > current_best['score'] or (
now - current_best['frame_time']) > self.config.get(
'best_image_timeout', 60):
obj_copy['frame'] = np.copy(self.current_frame)
self.best_objects[object_type] = obj_copy
for c in self.callbacks['snapshot']:
c(self.name, self.best_objects[object_type])
else:
obj_copy['frame'] = np.copy(self.current_frame)
self.best_objects[object_type] = obj_copy
for c in self.callbacks['snapshot']:
c(self.name, self.best_objects[object_type])
# update overall camera state for each object type
obj_counter = Counter()
for obj in self.tracked_objects.values():
if not obj['false_positive']:
obj_counter[obj['label']] += 1
# report on detected objects
for obj_name, count in obj_counter.items():
new_status = 'ON' if count > 0 else 'OFF'
if new_status != self.object_status[obj_name]:
self.object_status[obj_name] = new_status
for c in self.callbacks['object_status']:
c(self.name, obj_name, new_status)
# expire any objects that are ON and no longer detected
expired_objects = [
obj_name for obj_name, status in self.object_status.items()
if status == 'ON' and not obj_name in obj_counter
]
for obj_name in expired_objects:
self.object_status[obj_name] = 'OFF'
for c in self.callbacks['object_status']:
c(self.name, obj_name, 'OFF')
for c in self.callbacks['snapshot']:
c(self.name, self.best_objects[obj_name])
def run(self):
while True:
if self.stop_event.is_set():
print(f"Exiting object processor...")
break
try:
camera, frame_time, current_tracked_objects = self.tracked_objects_queue.get(
True, 10)
except queue.Empty:
continue
camera_config = self.camera_config[camera]
best_objects = self.camera_data[camera]['best_objects']
current_object_status = self.camera_data[camera]['object_status']
tracked_objects = self.camera_data[camera]['tracked_objects']
current_ids = current_tracked_objects.keys()
previous_ids = tracked_objects.keys()
removed_ids = list(set(previous_ids).difference(current_ids))
new_ids = list(set(current_ids).difference(previous_ids))
updated_ids = list(set(current_ids).intersection(previous_ids))
for id in new_ids:
# only register the object here if we are sure it isnt a false positive
if not filter_false_positives(current_tracked_objects[id]):
tracked_objects[id] = current_tracked_objects[id]
# publish events to mqtt
self.client.publish(
f"{self.topic_prefix}/{camera}/events/start",
json.dumps(tracked_objects[id]),
retain=False)
self.event_queue.put(
('start', camera, tracked_objects[id]))
for id in updated_ids:
tracked_objects[id] = current_tracked_objects[id]
for id in removed_ids:
# publish events to mqtt
tracked_objects[id]['end_time'] = frame_time
self.client.publish(f"{self.topic_prefix}/{camera}/events/end",
json.dumps(tracked_objects[id]),
retain=False)
self.event_queue.put(('end', camera, tracked_objects[id]))
del tracked_objects[id]
self.camera_data[camera]['current_frame_time'] = frame_time
# build a dict of objects in each zone for current camera
current_objects_in_zones = defaultdict(lambda: [])
for obj in tracked_objects.values():
bottom_center = (obj['centroid'][0], obj['box'][3])
# check each zone
for name, zone in self.zone_data.items():
current_contour = zone['contours'].get(camera, None)
# if the current camera does not have a contour for this zone, skip
if current_contour is None:
continue
# check if the object is in the zone and not filtered
if (cv2.pointPolygonTest(current_contour, bottom_center,
False) >= 0
and not zone_filtered(
obj, self.zone_config[name][camera].get(
'filters', {}))):
current_objects_in_zones[name].append(obj['label'])
###
# Draw tracked objects on the frame
###
current_frame = self.plasma_client.get(f"{camera}{frame_time}")
if not current_frame is plasma.ObjectNotAvailable:
# draw the bounding boxes on the frame
for obj in tracked_objects.values():
thickness = 2
color = COLOR_MAP[obj['label']]
if obj['frame_time'] != frame_time:
thickness = 1
color = (255, 0, 0)
# draw the bounding boxes on the frame
box = obj['box']
draw_box_with_label(
current_frame,
box[0],
box[1],
box[2],
box[3],
obj['label'],
f"{int(obj['score']*100)}% {int(obj['area'])}",
thickness=thickness,
color=color)
# draw the regions on the frame
region = obj['region']
cv2.rectangle(current_frame, (region[0], region[1]),
(region[2], region[3]), (0, 255, 0), 1)
if camera_config['snapshots']['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(
frame_time).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(current_frame,
time_to_show, (10, 30),
cv2.FONT_HERSHEY_SIMPLEX,
fontScale=.8,
color=(255, 255, 255),
thickness=2)
if camera_config['snapshots']['draw_zones']:
for name, zone in self.zone_data.items():
thickness = 2 if len(
current_objects_in_zones[name]) == 0 else 8
if camera in zone['contours']:
cv2.drawContours(current_frame,
[zone['contours'][camera]], -1,
zone['color'], thickness)
###
# Set the current frame
###
self.camera_data[camera]['current_frame'] = current_frame
# delete the previous frame from the plasma store and update the object id
if not self.camera_data[camera]['object_id'] is None:
self.plasma_client.delete(
self.camera_data[camera]['object_id'])
self.camera_data[camera]['object_id'] = f"{camera}{frame_time}"
###
# Maintain the highest scoring recent object and frame for each label
###
for obj in tracked_objects.values():
# if the object wasn't seen on the current frame, skip it
if obj['frame_time'] != frame_time:
continue
if obj['label'] in best_objects:
now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score
# or the current object is more than 1 minute old, use the new object
if obj['score'] > best_objects[obj['label']]['score'] or (
now -
best_objects[obj['label']]['frame_time']) > 60:
obj['frame'] = np.copy(
self.camera_data[camera]['current_frame'])
best_objects[obj['label']] = obj
# send updated snapshot over mqtt
best_frame = cv2.cvtColor(obj['frame'],
cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj['label']}/snapshot",
jpg_bytes,
retain=True)
else:
obj['frame'] = np.copy(
self.camera_data[camera]['current_frame'])
best_objects[obj['label']] = obj
###
# Report over MQTT
###
# get the zones that are relevant for this camera
relevant_zones = [
zone for zone, config in self.zone_config.items()
if camera in config
]
for zone in relevant_zones:
# create the set of labels in the current frame and previously reported
labels_for_zone = set(
current_objects_in_zones[zone] +
list(self.zone_data[zone]['object_status'][camera].keys()))
# for each label
for label in labels_for_zone:
# compute the current 'ON' vs 'OFF' status by checking if any camera sees the object in the zone
previous_state = any([
c[label] == 'ON' for c in self.zone_data[zone]
['object_status'].values()
])
self.zone_data[zone]['object_status'][camera][
label] = 'ON' if label in current_objects_in_zones[
zone] else 'OFF'
new_state = any([
c[label] == 'ON' for c in self.zone_data[zone]
['object_status'].values()
])
# if the value is changing, send over MQTT
if previous_state == False and new_state == True:
self.client.publish(
f"{self.topic_prefix}/{zone}/{label}",
'ON',
retain=False)
elif previous_state == True and new_state == False:
self.client.publish(
f"{self.topic_prefix}/{zone}/{label}",
'OFF',
retain=False)
# count by type
obj_counter = Counter()
for obj in tracked_objects.values():
obj_counter[obj['label']] += 1
# report on detected objects
for obj_name, count in obj_counter.items():
new_status = 'ON' if count > 0 else 'OFF'
if new_status != current_object_status[obj_name]:
current_object_status[obj_name] = new_status
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}",
new_status,
retain=False)
# send the best snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'],
cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}/snapshot",
jpg_bytes,
retain=True)
# expire any objects that are ON and no longer detected
expired_objects = [
obj_name for obj_name, status in current_object_status.items()
if status == 'ON' and not obj_name in obj_counter
]
for obj_name in expired_objects:
current_object_status[obj_name] = 'OFF'
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}",
'OFF',
retain=False)
# send updated snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'],
cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}/snapshot",
jpg_bytes,
retain=True)
def run(self):
while True:
camera, frame_time, tracked_objects = self.tracked_objects_queue.get(
)
debug("{} {} {}".format(camera, frame_time, tracked_objects))
# debug(self.config)
# delete by wang jin liang
# config = self.config[camera]
best_objects = self.camera_data[camera]['best_objects']
current_object_status = self.camera_data[camera]['object_status']
self.camera_data[camera]['tracked_objects'] = tracked_objects
###
# Draw tracked objects on the frame
###
object_id_hash = hashlib.sha1(str.encode(f"{camera}{frame_time}"))
object_id_bytes = object_id_hash.digest()
object_id = plasma.ObjectID(object_id_bytes)
current_frame = self.plasma_client.get(object_id, timeout_ms=0)
if not current_frame is plasma.ObjectNotAvailable:
# draw the bounding boxes on the frame
for obj in tracked_objects.values():
thickness = 2
color = COLOR_MAP[obj['label']]
if obj['frame_time'] != frame_time:
thickness = 1
color = (255, 0, 0)
# draw the bounding boxes on the frame
box = obj['box']
draw_box_with_label(
current_frame,
box[0],
box[1],
box[2],
box[3],
obj['label'],
f"{int(obj['score']*100)}% {int(obj['area'])}",
thickness=thickness,
color=color)
# draw the regions on the frame
region = obj['region']
cv2.rectangle(current_frame, (region[0], region[1]),
(region[2], region[3]), (0, 255, 0), 1)
# delete by wangjinliang
# if config['snapshots']['show_timestamp']:
# time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
# cv2.putText(current_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
###
# Set the current frame as ready
###
self.camera_data[camera]['current_frame'] = current_frame
# store the object id, so you can delete it at the next loop
previous_object_id = self.camera_data[camera]['object_id']
if not previous_object_id is None:
self.plasma_client.delete([previous_object_id])
self.camera_data[camera]['object_id'] = object_id
###
# Maintain the highest scoring recent object and frame for each label
###
for obj in tracked_objects.values():
# if the object wasn't seen on the current frame, skip it
if obj['frame_time'] != frame_time:
continue
if obj['label'] in best_objects:
now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score
# or the current object is more than 1 minute old, use the new object
if obj['score'] > best_objects[obj['label']]['score'] or (
now -
best_objects[obj['label']]['frame_time']) > 60:
obj['frame'] = np.copy(
self.camera_data[camera]['current_frame'])
best_objects[obj['label']] = obj
else:
obj['frame'] = np.copy(
self.camera_data[camera]['current_frame'])
best_objects[obj['label']] = obj
###
# Report over MQTT
###
# count objects with more than 2 entries in history by type
obj_counter = Counter()
for obj in tracked_objects.values():
if len(obj['history']) > 1:
obj_counter[obj['label']] += 1
# report on detected objects
for obj_name, count in obj_counter.items():
new_status = 'ON' if count > 0 else 'OFF'
if new_status != current_object_status[obj_name]:
current_object_status[obj_name] = new_status
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}",
new_status,
retain=False)
# send the best snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'],
cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}/snapshot",
jpg_bytes,
retain=True)
# expire any objects that are ON and no longer detected
expired_objects = [
obj_name for obj_name, status in current_object_status.items()
if status == 'ON' and not obj_name in obj_counter
]
for obj_name in expired_objects:
current_object_status[obj_name] = 'OFF'
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}",
'OFF',
retain=False)
# send updated snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'],
cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}/snapshot",
jpg_bytes,
retain=True)
def get_current_frame(self, draw_options={}):
with self.current_frame_lock:
frame_copy = np.copy(self._current_frame)
frame_time = self.current_frame_time
tracked_objects = {
k: v.to_dict()
for k, v in self.tracked_objects.items()
}
motion_boxes = self.motion_boxes.copy()
regions = self.regions.copy()
frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_YUV2BGR_I420)
# draw on the frame
if draw_options.get("bounding_boxes"):
# draw the bounding boxes on the frame
for obj in tracked_objects.values():
if obj["frame_time"] == frame_time:
thickness = 2
color = COLOR_MAP[obj["label"]]
else:
thickness = 1
color = (255, 0, 0)
# draw the bounding boxes on the frame
box = obj["box"]
draw_box_with_label(
frame_copy,
box[0],
box[1],
box[2],
box[3],
obj["label"],
f"{obj['score']:.0%} {int(obj['area'])}",
thickness=thickness,
color=color,
)
if draw_options.get("regions"):
for region in regions:
cv2.rectangle(
frame_copy,
(region[0], region[1]),
(region[2], region[3]),
(0, 255, 0),
2,
)
if draw_options.get("zones"):
for name, zone in self.camera_config.zones.items():
thickness = (8 if any(
name in obj["current_zones"]
for obj in tracked_objects.values()) else 2)
cv2.drawContours(frame_copy, [zone.contour], -1, zone.color,
thickness)
if draw_options.get("mask"):
mask_overlay = np.where(self.camera_config.motion.mask == [0])
frame_copy[mask_overlay] = [0, 0, 0]
if draw_options.get("motion_boxes"):
for m_box in motion_boxes:
cv2.rectangle(
frame_copy,
(m_box[0], m_box[1]),
(m_box[2], m_box[3]),
(0, 0, 255),
2,
)
if draw_options.get("timestamp"):
time_to_show = datetime.datetime.fromtimestamp(
frame_time).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(
frame_copy,
time_to_show,
(10, 30),
cv2.FONT_HERSHEY_SIMPLEX,
fontScale=0.8,
color=(255, 255, 255),
thickness=2,
)
return frame_copy
def get_jpg_bytes(self,
timestamp=False,
bounding_box=False,
crop=False,
height=None):
if self.thumbnail_data is None:
return None
try:
best_frame = cv2.cvtColor(
self.frame_cache[self.thumbnail_data["frame_time"]],
cv2.COLOR_YUV2BGR_I420,
)
except KeyError:
logger.warning(
f"Unable to create jpg because frame {self.thumbnail_data['frame_time']} is not in the cache"
)
return None
if bounding_box:
thickness = 2
color = COLOR_MAP[self.obj_data["label"]]
# draw the bounding boxes on the frame
box = self.thumbnail_data["box"]
draw_box_with_label(
best_frame,
box[0],
box[1],
box[2],
box[3],
self.obj_data["label"],
f"{int(self.thumbnail_data['score']*100)}% {int(self.thumbnail_data['area'])}",
thickness=thickness,
color=color,
)
if crop:
box = self.thumbnail_data["box"]
region = calculate_region(best_frame.shape, box[0], box[1], box[2],
box[3], 1.1)
best_frame = best_frame[region[1]:region[3], region[0]:region[2]]
if height:
width = int(height * best_frame.shape[1] / best_frame.shape[0])
best_frame = cv2.resize(best_frame,
dsize=(width, height),
interpolation=cv2.INTER_AREA)
if timestamp:
time_to_show = datetime.datetime.fromtimestamp(
self.thumbnail_data["frame_time"]).strftime(
"%m/%d/%Y %H:%M:%S")
size = cv2.getTextSize(time_to_show,
cv2.FONT_HERSHEY_SIMPLEX,
fontScale=1,
thickness=2)
text_width = size[0][0]
desired_size = max(150, 0.33 * best_frame.shape[1])
font_scale = desired_size / text_width
cv2.putText(
best_frame,
time_to_show,
(5, best_frame.shape[0] - 7),
cv2.FONT_HERSHEY_SIMPLEX,
fontScale=font_scale,
color=(255, 255, 255),
thickness=2,
)
ret, jpg = cv2.imencode(".jpg", best_frame,
[int(cv2.IMWRITE_JPEG_QUALITY), 70])
if ret:
return jpg.tobytes()
else:
return None
def run(self):
while True:
camera, frame_time, current_tracked_objects = self.tracked_objects_queue.get(
)
config = self.config[camera]
best_objects = self.camera_data[camera]['best_objects']
current_object_status = self.camera_data[camera]['object_status']
tracked_objects = self.camera_data[camera]['tracked_objects']
current_ids = current_tracked_objects.keys()
previous_ids = tracked_objects.keys()
removed_ids = list(set(previous_ids).difference(current_ids))
new_ids = list(set(current_ids).difference(previous_ids))
updated_ids = list(set(current_ids).intersection(previous_ids))
for id in new_ids:
tracked_objects[id] = current_tracked_objects[id]
# publish events to mqtt
self.client.publish(
f"{self.topic_prefix}/{camera}/events/start",
json.dumps(tracked_objects[id]),
retain=False)
self.event_queue.put(('start', camera, tracked_objects[id]))
for id in updated_ids:
tracked_objects[id] = current_tracked_objects[id]
for id in removed_ids:
# publish events to mqtt
tracked_objects[id]['end_time'] = frame_time
self.client.publish(f"{self.topic_prefix}/{camera}/events/end",
json.dumps(tracked_objects[id]),
retain=False)
self.event_queue.put(('end', camera, tracked_objects[id]))
del tracked_objects[id]
self.camera_data[camera]['current_frame_time'] = frame_time
###
# Update room tracker if enabled
###
room_tracker_conf = config.get("room_tracker", None)
if room_tracker_conf is not None and room_tracker_conf.get(
"enabled", False):
if self.room_tracker is None:
self.room_tracker = RoomTracker(room_tracker_conf)
self.room_tracker.on_change(frame_time, tracked_objects)
###
# Draw tracked objects on the frame
###
current_frame = self.plasma_client.get(f"{camera}{frame_time}")
if not current_frame is plasma.ObjectNotAvailable:
# draw the bounding boxes on the frame
for obj in tracked_objects.values():
thickness = 2
color = COLOR_MAP[obj['label']]
if obj['frame_time'] != frame_time:
thickness = 1
color = (255, 0, 0)
# draw the bounding boxes on the frame
box = obj['box']
draw_box_with_label(
current_frame,
box[0],
box[1],
box[2],
box[3],
obj['label'],
f"{int(obj['score']*100)}% {int(obj['area'])}",
thickness=thickness,
color=color)
# draw the regions on the frame
region = obj['region']
cv2.rectangle(current_frame, (region[0], region[1]),
(region[2], region[3]), (0, 255, 0), 1)
if config['snapshots']['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(
frame_time).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(current_frame,
time_to_show, (10, 30),
cv2.FONT_HERSHEY_SIMPLEX,
fontScale=.8,
color=(255, 255, 255),
thickness=2)
# Draw room tracker area points
if self.room_tracker is not None:
for room_name, c in self.room_tracker.rooms_conf.items():
p = (c["point_x"], c["point_y"])
cv2.rectangle(current_frame, (p[0] - 10, p[1] - 10),
(p[0] + 10, p[1] + 10), (255, 0, 0), 3)
###
# Set the current frame
###
self.camera_data[camera]['current_frame'] = current_frame
# delete the previous frame from the plasma store and update the object id
if not self.camera_data[camera]['object_id'] is None:
self.plasma_client.delete(
self.camera_data[camera]['object_id'])
self.camera_data[camera]['object_id'] = f"{camera}{frame_time}"
###
# Maintain the highest scoring recent object and frame for each label
###
for obj in tracked_objects.values():
# if the object wasn't seen on the current frame, skip it
if obj['frame_time'] != frame_time:
continue
if obj['label'] in best_objects:
now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score
# or the current object is more than 1 minute old, use the new object
if obj['score'] > best_objects[obj['label']]['score'] or (
now -
best_objects[obj['label']]['frame_time']) > 60:
obj['frame'] = np.copy(
self.camera_data[camera]['current_frame'])
best_objects[obj['label']] = obj
# send updated snapshot over mqtt
best_frame = cv2.cvtColor(obj['frame'],
cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj['label']}/snapshot",
jpg_bytes,
retain=True)
else:
obj['frame'] = np.copy(
self.camera_data[camera]['current_frame'])
best_objects[obj['label']] = obj
###
# Report over MQTT
###
# count objects by type
obj_counter = Counter()
for obj in tracked_objects.values():
obj_counter[obj['label']] += 1
# report on detected objects
for obj_name, count in obj_counter.items():
new_status = 'ON' if count > 0 else 'OFF'
if new_status != current_object_status[obj_name]:
current_object_status[obj_name] = new_status
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}",
new_status,
retain=False)
# send the best snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'],
cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}/snapshot",
jpg_bytes,
retain=True)
# expire any objects that are ON and no longer detected
expired_objects = [
obj_name for obj_name, status in current_object_status.items()
if status == 'ON' and not obj_name in obj_counter
]
for obj_name in expired_objects:
current_object_status[obj_name] = 'OFF'
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}",
'OFF',
retain=False)
# send updated snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'],
cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(
f"{self.topic_prefix}/{camera}/{obj_name}/snapshot",
jpg_bytes,
retain=True)
# report area tracking
if self.room_tracker is not None:
for room_name, _ in self.room_tracker.rooms_conf.items():
ppl_count = self.room_tracker.get_area_count(room_name)
status = "ON" if ppl_count > 0 else "OFF"
timestamp = self.room_tracker.get_latest_change_timestamp(
room_name)
r = {
"status": status,
"count": ppl_count,
"timestamp": timestamp,
}
if room_name in self.room_tracker_mqtt_state and self.room_tracker_mqtt_state[
room_name] == r:
continue
else:
self.client.publish(
f"{self.topic_prefix}/{camera}/area/{room_name}",
json.dumps(r),
retain=False)
self.room_tracker_mqtt_state[room_name] = r
def get_current_frame(self, draw_options={}):
with self.current_frame_lock:
frame_copy = np.copy(self._current_frame)
frame_time = self.current_frame_time
tracked_objects = {k: v.to_dict() for k, v in self.tracked_objects.items()}
motion_boxes = self.motion_boxes.copy()
regions = self.regions.copy()
frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_YUV2BGR_I420)
# draw on the frame
if draw_options.get("bounding_boxes"):
# draw the bounding boxes on the frame
for obj in tracked_objects.values():
if obj["frame_time"] == frame_time:
thickness = 2
color = self.config.model.colormap[obj["label"]]
else:
thickness = 1
color = (255, 0, 0)
# draw the bounding boxes on the frame
box = obj["box"]
draw_box_with_label(
frame_copy,
box[0],
box[1],
box[2],
box[3],
obj["label"],
f"{obj['score']:.0%} {int(obj['area'])}",
thickness=thickness,
color=color,
)
if draw_options.get("regions"):
for region in regions:
cv2.rectangle(
frame_copy,
(region[0], region[1]),
(region[2], region[3]),
(0, 255, 0),
2,
)
if draw_options.get("zones"):
for name, zone in self.camera_config.zones.items():
thickness = (
8
if any(
name in obj["current_zones"] for obj in tracked_objects.values()
)
else 2
)
cv2.drawContours(frame_copy, [zone.contour], -1, zone.color, thickness)
if draw_options.get("mask"):
mask_overlay = np.where(self.camera_config.motion.mask == [0])
frame_copy[mask_overlay] = [0, 0, 0]
if draw_options.get("motion_boxes"):
for m_box in motion_boxes:
cv2.rectangle(
frame_copy,
(m_box[0], m_box[1]),
(m_box[2], m_box[3]),
(0, 0, 255),
2,
)
if draw_options.get("timestamp"):
color = self.camera_config.timestamp_style.color
draw_timestamp(
frame_copy,
frame_time,
self.camera_config.timestamp_style.format,
font_effect=self.camera_config.timestamp_style.effect,
font_thickness=self.camera_config.timestamp_style.thickness,
font_color=(color.blue, color.green, color.red),
position=self.camera_config.timestamp_style.position,
)
return frame_copy
def get_jpg_bytes(
self, timestamp=False, bounding_box=False, crop=False, height=None, quality=70
):
if self.thumbnail_data is None:
return None
try:
best_frame = cv2.cvtColor(
self.frame_cache[self.thumbnail_data["frame_time"]],
cv2.COLOR_YUV2BGR_I420,
)
except KeyError:
logger.warning(
f"Unable to create jpg because frame {self.thumbnail_data['frame_time']} is not in the cache"
)
return None
if bounding_box:
thickness = 2
color = self.colormap[self.obj_data["label"]]
# draw the bounding boxes on the frame
box = self.thumbnail_data["box"]
draw_box_with_label(
best_frame,
box[0],
box[1],
box[2],
box[3],
self.obj_data["label"],
f"{int(self.thumbnail_data['score']*100)}% {int(self.thumbnail_data['area'])}",
thickness=thickness,
color=color,
)
if crop:
box = self.thumbnail_data["box"]
region = calculate_region(
best_frame.shape, box[0], box[1], box[2], box[3], 1.1
)
best_frame = best_frame[region[1] : region[3], region[0] : region[2]]
if height:
width = int(height * best_frame.shape[1] / best_frame.shape[0])
best_frame = cv2.resize(
best_frame, dsize=(width, height), interpolation=cv2.INTER_AREA
)
if timestamp:
color = self.camera_config.timestamp_style.color
draw_timestamp(
best_frame,
self.thumbnail_data["frame_time"],
self.camera_config.timestamp_style.format,
font_effect=self.camera_config.timestamp_style.effect,
font_thickness=self.camera_config.timestamp_style.thickness,
font_color=(color.blue, color.green, color.red),
position=self.camera_config.timestamp_style.position,
)
ret, jpg = cv2.imencode(
".jpg", best_frame, [int(cv2.IMWRITE_JPEG_QUALITY), quality]
)
if ret:
return jpg.tobytes()
else:
return None
