def load_model(self):
self.logger.Debug (1, '|--------- Loading Yolo model from disk ---------|')
t = Timer()
self.net = cv2.dnn.readNet(self.options.get('object_weights'),
self.options.get('object_config'))
#self.net = cv2.dnn.readNetFromDarknet(config_file_abs_path, weights_file_abs_path)
diff_time = t.stop_and_get_ms()
self.logger.Debug(
1,'perf: processor:{} Yolo initialization (loading {} model from disk) took: {}'
.format(self.processor, self.options.get('object_weights'), diff_time))
if self.processor == 'gpu':
(maj, minor, patch) = cv2.__version__.split('.')
min_ver = int(maj + minor)
if min_ver < 42:
self.logger.Error('Not setting CUDA backend for OpenCV DNN')
self.logger.Error(
'You are using OpenCV version {} which does not support CUDA for DNNs. A minimum of 4.2 is required. See https://www.pyimagesearch.com/2020/02/03/how-to-use-opencvs-dnn-module-with-nvidia-gpus-cuda-and-cudnn/ on how to compile and install openCV 4.2'
.format(cv2.__version__))
self.processor = 'cpu'
else:
self.logger.Debug (1, 'Using CPU for detection')
if self.processor == 'gpu':
self.logger.Debug( 2,'Setting CUDA backend for OpenCV')
self.logger.Debug( 3,'If you did not set your CUDA_ARCH_BIN correctly during OpenCV compilation, you will get errors during detection related to invalid device/make_policy')
self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)
self.populate_class_labels()
def detect(self, image=None):
Height, Width = image.shape[:2]
img = image.copy()
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = Image.fromarray(img)
if self.options.get('auto_lock', True):
self.acquire_lock()
try:
if not self.model:
self.load_model()
g.logger.Debug(
1, '|---------- TPU (input image: {}w*{}h) ----------|'.format(
Width, Height))
t = Timer()
_, scale = common.set_resized_input(
self.model, img.size,
lambda size: img.resize(size, Image.ANTIALIAS))
self.model.invoke()
objs = detect.get_objects(
self.model, float(self.options.get('object_min_confidence')),
scale)
#outs = self.model.detect_with_image(img, threshold=int(self.options.get('object_min_confidence')),
# keep_aspect_ratio=True, relative_coord=False)
diff_time = t.stop_and_get_ms()
if self.options.get('auto_lock', True):
self.release_lock()
except:
if self.options.get('auto_lock', True):
self.release_lock()
raise
diff_time = t.stop_and_get_ms()
g.logger.Debug(
1, 'perf: processor:{} Coral TPU detection took: {}'.format(
self.processor, diff_time))
bbox = []
labels = []
conf = []
for obj in objs:
# box = obj.bbox.flatten().astype("int")
bbox.append([
int(round(obj.bbox.xmin)),
int(round(obj.bbox.ymin)),
int(round(obj.bbox.xmax)),
int(round(obj.bbox.ymax))
])
labels.append(self.classes.get(obj.id))
conf.append(float(obj.score))
g.logger.Debug(
3, 'Coral object returning: {},{},{}'.format(bbox, labels, conf))
return bbox, labels, conf, ['coral'] * len(labels)
def load_model(self):
name = self.options.get('name') or 'TPU'
g.logger.Debug(
1, '|--------- Loading "{}" model from disk -------------|'.format(
name))
t = Timer()
self.model = make_interpreter(self.options.get('face_weights'))
self.model.allocate_tensors()
diff_time = t.stop_and_get_ms()
g.logger.Debug(
1,
'perf: processor:{} TPU initialization (loading {} from disk) took: {}'
.format(self.processor, self.options.get('face_weights'),
diff_time))
def load_model(self):
self.logger.Debug(
1, '|--------- Loading TPU model from disk -------------|')
#self.model = DetectionEngine(self.options.get('object_weights'))
# Initialize the TF interpreter
t = Timer()
self.model = make_interpreter(self.options.get('object_weights'))
self.model.allocate_tensors()
diff_time = t.stop_and_get_ms()
self.logger.Debug(
1,
'perf: processor:{} TPU initialization (loading {} from disk) took: {}'
.format(self.processor, self.options.get('object_weights'),
diff_time))
def detect(self, image):
height, width = image.shape[:2]
g.logger.Debug(
2, f'detect extracted image dimensions as: {width}x{height}')
# Scale to 300x300 (that's what MobileNetSSD expects)
image_resized = cv2.resize(image,
(self.MODEL_HEIGHT, self.MODEL_WIDTH))
height_factor = image.shape[0] / self.MODEL_HEIGHT
width_factor = image.shape[1] / self.MODEL_WIDTH
if self.options.get('auto_lock', True):
self.acquire_lock()
try:
if not self.net:
self.load_model()
t = Timer()
blob = cv2.dnn.blobFromImage(image_resized, 0.007843,
(self.MODEL_HEIGHT, self.MODEL_WIDTH),
(127.5, 127.5, 127.5), False)
self.net.setInput(blob)
detections = self.net.forward()
if self.options.get('auto_lock', True):
self.release_lock()
except:
if self.options.get('auto_lock', True):
self.release_lock()
raise
diff_time = t.stop_and_get_ms()
g.logger.Debug(
1,
f'perf: processor:{self.processor} MobileNetSSD detection took: {diff_time}'
)
conf_threshold = float(self.options.get('object_min_confidence', 0.2))
bboxes = []
labels = []
confs = []
for i in range(detections.shape[2]):
confidence = float(detections[0, 0, i, 2])
label_id = int(detections[0, 0, i, 1])
bbox_rel = list(detections[0, 0, i, 3:7])
try:
label = self.classes[label_id]
except ValueError as e:
g.logger.Warning("LabelID={label_id} is outside of the range")
continue
if confidence < conf_threshold:
if confidence > 0:
g.logger.Debug(
3, f"Ignored Detection: {label}: {confidence:.2f}")
continue
g.logger.Debug(1, f"Detection: {label}: {confidence:.2f}")
bbox = (
round(width * bbox_rel[0]),
round(height * bbox_rel[1]),
round(width * bbox_rel[2]),
round(height * bbox_rel[3]),
)
bboxes.append(bbox)
labels.append(label)
confs.append(confidence)
return bboxes, labels, confs, ['mobnet'] * len(label)
def detect(self, image):
Height, Width = image.shape[:2]
self.logger.Debug(
1,'|---------- Face recognition (input image: {}w*{}h) ----------|'.
format(Width, Height))
downscaled = False
upsize_xfactor = None
upsize_yfactor = None
max_size = self.options.get('max_size', Width)
old_image = None
self.logger.Debug(5, 'Face options={}'.format(self.options))
if Width > max_size:
downscaled = True
self.logger.Debug (2, 'Scaling image down to max size: {}'.format(max_size))
old_image = image.copy()
image = imutils.resize(image,width=max_size)
newHeight, newWidth = image.shape[:2]
upsize_xfactor = Width/newWidth
upsize_yfactor = Height/newHeight
labels = []
classes = []
conf = []
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
rgb_image = image[:, :, ::-1]
#rgb_image = image
# Find all the faces and face encodings in the target image
#prin (self.options)
if self.options.get('auto_lock',True):
self.acquire_lock()
t = Timer()
face_locations = face_recognition.face_locations(
rgb_image,
model=self.face_model,
number_of_times_to_upsample=self.upsample_times)
diff_time = t.stop_and_get_ms()
self.logger.Debug(1,'perf: processor:{} Finding faces took {}'.format(self.processor, diff_time))
t = Timer()
face_encodings = face_recognition.face_encodings(
rgb_image,
known_face_locations=face_locations,
num_jitters=self.num_jitters)
if self.options.get('auto_lock',True):
self.release_lock()
diff_time = t.stop_and_get_ms()
self.logger.Debug(
1,'perf: processor:{} Computing face recognition distances took {}'.format(
self.processor, diff_time))
if not len(face_encodings):
return [], [], []
# Use the KNN model to find the best matches for the test face
self.logger.Debug(3,'Comparing to known faces...')
t = Timer()
if self.knn:
#self.logger.Debug(5, 'FACE ENCODINGS={}'.format(face_encodings))
closest_distances = self.knn.kneighbors(face_encodings, n_neighbors=1)
self.logger.Debug(5, 'Closest knn match indexes (lesser is better): {}'.format(closest_distances))
are_matches = [
closest_distances[0][i][0] <= float(self.options.get('face_recog_dist_threshold',0.6))
for i in range(len(face_locations))
]
prediction_labels = self.knn.predict(face_encodings)
#self.logger.Debug(5, 'KNN predictions: {} are_matches: {}'.format(prediction_labels, are_matches))
else:
# There were no faces to compare
# create a set of non matches for each face found
are_matches = [False] * len(face_locations)
prediction_labels = [''] * len(face_locations)
self.logger.Debug (1,'No faces to match, so creating empty set')
diff_time = t.stop_and_get_ms()
self.logger.Debug(
1,'perf: processor:{} Matching recognized faces to known faces took {}'.
format(self.processor, diff_time))
matched_face_names = []
matched_face_rects = []
if downscaled:
self.logger.Debug (2,'Scaling image back up to {}'.format(Width))
image = old_image
new_face_locations = []
for loc in face_locations:
a,b,c,d=loc
a = round(a * upsize_yfactor)
b = round(b * upsize_xfactor)
c = round(c * upsize_yfactor)
d = round(d * upsize_xfactor)
new_face_locations.append((a,b,c,d))
face_locations = new_face_locations
for pred, loc, rec in zip(prediction_labels,
face_locations, are_matches):
label = pred if rec else self.options.get('unknown_face_name', 'unknown')
if not rec and self.options.get('save_unknown_faces') == 'yes':
h, w, c = image.shape
x1 = max(loc[3] - int(self.options.get('save_unknown_faces_leeway_pixels',0)),0)
y1 = max(loc[0] - int(self.options.get('save_unknown_faces_leeway_pixels',0)),0)
x2 = min(loc[1] + int(self.options.get('save_unknown_faces_leeway_pixels',0)), w)
y2 = min(loc[2] + int(self.options.get('save_unknown_faces_leeway_pixels',0)),h)
#print (image)
crop_img = image[y1:y2, x1:x2]
# crop_img = image
timestr = time.strftime("%b%d-%Hh%Mm%Ss-")
unf = self.options.get('unknown_images_path') + '/' + timestr + str(
uuid.uuid4()) + '.jpg'
self.logger.Info(
'Saving cropped unknown face at [{},{},{},{} - includes leeway of {}px] to {}'
.format(x1, y1, x2, y2,
self.options.get('save_unknown_faces_leeway_pixels'), unf))
cv2.imwrite(unf, crop_img)
matched_face_rects.append([loc[3], loc[0], loc[1], loc[2]])
matched_face_names.append(label)
#matched_face_names.append('face:{}'.format(label))
conf.append(1)
self.logger.Debug(3,f'FACE:Returning: {matched_face_rects}, {matched_face_names}, {conf}')
return matched_face_rects, matched_face_names, conf
import pickle
from sklearn import neighbors
import imutils
import math
import uuid
import time
import datetime
from pyzm.helpers.Base import Base
# Class to handle face recognition
import portalocker
import re
from pyzm.helpers.Media import MediaStream
import imutils
from pyzm.helpers.utils import Timer
g_start =Timer()
import face_recognition
g_diff_time = g_start.stop_and_get_ms()
class Face(Base):
def __init__(self, logger=None, options={},upsample_times=1, num_jitters=0, model='hog'):
super().__init__(logger)
global g_diff_time
#self.logger.Debug (4, 'Face init params: {}'.format(options))
if dlib.DLIB_USE_CUDA and dlib.cuda.get_num_devices() >=1 :
self.processor = 'gpu'
else:
self.processor = 'cpu'
self.logger.Debug(
def train(self,size=None):
t = Timer()
known_images_path = self.options.get('known_images_path')
train_model = self.options.get('face_train_model')
knn_algo = self.options.get('face_recog_knn_algo', 'ball_tree')
upsample_times = int(self.options.get('face_upsample_times',1))
num_jitters = int(self.options.get('face_num_jitters',0))
encoding_file_name = known_images_path + '/faces.dat'
try:
if (os.path.isfile(known_images_path + '/faces.pickle')):
# old version, we no longer want it. begone
g.logger.Debug(
2,'removing old faces.pickle, we have moved to clustering')
os.remove(known_images_path + '/faces.pickle')
except Exception as e:
g.logger.Error('Error deleting old pickle file: {}'.format(e))
directory = known_images_path
ext = ['.jpg', '.jpeg', '.png', '.gif']
known_face_encodings = []
known_face_names = []
try:
for entry in os.listdir(directory):
if os.path.isdir(directory + '/' + entry):
# multiple images for this person,
# so we need to iterate that subdir
g.logger.Debug(
1,'{} is a directory. Processing all images inside it'.
format(entry))
person_dir = os.listdir(directory + '/' + entry)
for person in person_dir:
if person.endswith(tuple(ext)):
g.logger.Debug(1,'loading face from {}/{}'.format(
entry, person))
# imread seems to do a better job of color space conversion and orientation
known_face = cv2.imread('{}/{}/{}'.format(
directory, entry, person))
if known_face is None or known_face.size == 0:
g.logger.Error('Error reading file, skipping')
continue
#known_face = face_recognition.load_image_file('{}/{}/{}'.format(directory,entry, person))
if not size:
size = int(self.options.get('resize',800))
g.logger.Debug (1,'resizing to {}'.format(size))
known_face = imutils.resize(known_face,width=size)
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
#g.logger.Debug(1,'Converting from BGR to RGB')
known_face = known_face[:, :, ::-1]
face_locations = face_recognition.face_locations(
known_face,
model=train_model,
number_of_times_to_upsample=upsample_times)
if len(face_locations) != 1:
g.logger.Error(
'File {} has {} faces, cannot use for training. We need exactly 1 face. If you think you have only 1 face try using "cnn" for training mode. Ignoring...'
.format(person, len(face_locations)))
else:
face_encodings = face_recognition.face_encodings(
known_face,
known_face_locations=face_locations,
num_jitters=num_jitters)
known_face_encodings.append(face_encodings[0])
known_face_names.append(entry)
#g.logger.Debug ('Adding image:{} as known person: {}'.format(person, person_dir))
elif entry.endswith(tuple(ext)):
# this was old style. Lets still support it. The image is a single file with no directory
g.logger.Debug(1,'loading face from {}'.format(entry))
#known_face = cv2.imread('{}/{}/{}'.format(directory,entry, person))
known_face = cv2.imread('{}/{}'.format(directory, entry))
if not size:
size = int(self.options.get('resize',800))
g.logger.Debug (1,'resizing to {}'.format(size))
known_face = imutils.resize(known_face,width=size)
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
known_face = known_face[:, :, ::-1]
face_locations = face_recognition.face_locations(
known_face,
model=train_model,
number_of_times_to_upsample=upsample_times)
if len(face_locations) != 1:
g.logger.Error(
'File {} has {} faces, cannot use for training. We need exactly 1 face. If you think you have only 1 face try using "cnn" for training mode. Ignoring...'
.format(person), len(face_locations))
else:
face_encodings = face_recognition.face_encodings(
known_face,
known_face_locations=face_locations,
num_jitters=num_jitters)
known_face_encodings.append(face_encodings[0])
known_face_names.append(os.path.splitext(entry)[0])
except Exception as e:
g.logger.Error('Error initializing face recognition: {}'.format(e))
raise ValueError(
'Error opening known faces directory. Is the path correct?')
# Now we've finished iterating all files/dirs
# lets create the svm
if not len(known_face_names):
g.logger.Error(
'No known faces found to train, encoding file not created')
else:
n_neighbors = int(round(math.sqrt(len(known_face_names))))
g.logger.Debug(2,'Using algo:{} n_neighbors to be: {}'.format(knn_algo, n_neighbors))
knn = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors,
algorithm=knn_algo,
weights='distance')
g.logger.Debug(1,'Training model ...')
knn.fit(known_face_encodings, known_face_names)
f = open(encoding_file_name, "wb")
pickle.dump(knn, f)
f.close()
g.logger.Debug(1,'wrote encoding file: {}'.format(encoding_file_name))
diff_time = t.stop_and_get_ms()
g.logger.Debug(
1,'perf: Face Recognition training took: {}'.format(diff_time))
def detect(self, image=None):
Height, Width = image.shape[:2]
downscaled = False
upsize_xfactor = None
upsize_yfactor = None
max_size = self.options.get('max_size', Width)
old_image = None
if Width > max_size:
downscaled = True
self.logger.Debug (2, 'Scaling image down to max size: {}'.format(max_size))
old_image = image.copy()
image = imutils.resize(image,width=max_size)
newHeight, newWidth = image.shape[:2]
upsize_xfactor = Width/newWidth
upsize_yfactor = Height/newHeight
if self.options.get('auto_lock',True):
self.acquire_lock()
try:
if not self.net:
self.load_model()
self.logger.Debug(
1,'|---------- YOLO (input image: {}w*{}h, model resize dimensions: {}w*{}h) ----------|'
.format(Width, Height, self.model_width, self.model_height))
scale = 0.00392 # 1/255, really. Normalize inputs.
t = Timer()
ln = self.net.getLayerNames()
ln = [ln[i[0] - 1] for i in self.net.getUnconnectedOutLayers()]
blob = cv2.dnn.blobFromImage(image,
scale, (self.model_width, self.model_height), (0, 0, 0),
True,
crop=False)
self.net.setInput(blob)
outs = self.net.forward(ln)
if self.options.get('auto_lock',True):
self.release_lock()
except:
if self.options.get('auto_lock',True):
self.release_lock()
raise
diff_time = t.stop_and_get_ms()
self.logger.Debug(
1,'perf: processor:{} Yolo detection took: {}'.format(self.processor, diff_time))
class_ids = []
confidences = []
boxes = []
nms_threshold = 0.4
conf_threshold = 0.2
# first nms filter out with a yolo confidence of 0.2 (or less)
if float(self.options.get('object_min_confidence')) < conf_threshold:
conf_threshold = float(self.options.get('object_min_confidence'))
for out in outs:
for detection in out:
scores = detection[5:]
class_id = np.argmax(scores)
confidence = scores[class_id]
center_x = int(detection[0] * Width)
center_y = int(detection[1] * Height)
w = int(detection[2] * Width)
h = int(detection[3] * Height)
x = center_x - w / 2
y = center_y - h / 2
class_ids.append(class_id)
confidences.append(float(confidence))
boxes.append([x, y, w, h])
t = Timer()
indices = cv2.dnn.NMSBoxes(boxes, confidences, conf_threshold,
nms_threshold)
diff_time = t.stop_and_get_ms()
self.logger.Debug(
2,'perf: processor:{} Yolo NMS filtering took: {}'.format(self.processor, diff_time))
bbox = []
label = []
conf = []
prefix = '(yolo) ' if self.options.get('show_models')=='yes' else ''
# now filter out with configured yolo confidence, so we can see rejections in log
for i in indices:
i = i[0]
box = boxes[i]
x = box[0]
y = box[1]
w = box[2]
h = box[3]
bbox.append([
int(round(x)),
int(round(y)),
int(round(x + w)),
int(round(y + h))
])
label.append(prefix+str(self.classes[class_ids[i]]))
conf.append(confidences[i])
if downscaled:
self.logger.Debug (2,'Scaling image back up to {}'.format(Width))
image = old_image
for box in bbox:
box[0] = round (box[0] * upsize_xfactor)
box[1] = round (box[1] * upsize_yfactor)
box[2] = round (box[2] * upsize_xfactor)
box[3] = round (box[3] * upsize_yfactor)
return bbox, label, conf
def detect_stream(self, stream, options={}, ml_overrides={}):
"""Implements detection on a video stream
Args:
stream (string): location of media (file, url or event ID)
ml_overrides(string): Ignore it. You will almost never need it. zm_detect uses it for ugly foo
options (dict, optional): Various options that control the detection process. Defaults to {}:
- delay (int): Delay in seconds before starting media stream
- download (boolean): if True, will download video before analysis. Defaults to False
- download_dir (string): directory where downloads will be kept (only applies to videos). Default is /tmp
- start_frame (int): Which frame to start analysis. Default 1.
- frame_skip: (int): Number of frames to skip in video (example, 3 means process every 3rd frame)
- max_frames (int): Total number of frames to process before stopping
- pattern (string): regexp for objects that will be matched. 'frame_strategy' key below will be applied to only objects that match this pattern
- frame_set (string): comma separated frames to read. Example 'alarm,21,31,41,snapshot'
Note that if you are specifying frame IDs and using ZM, remember that ZM has a frame buffer
Default is 20, I think. So you may want to start at frame 21.
- contig_frames_before_error (int): How many contiguous frames should fail before we give up on reading this stream. Default 5
- max_attempts (int): Only for ZM indirection. How many times to retry a failed frame get. Default 1
- sleep_between_attempts (int): Only for ZM indirection. Time to wait before re-trying a failed frame
- disable_ssl_cert_check (bool): If True (default) will allow self-signed certs to work
- save_frames (boolean): If True, will save frames used in analysis. Default False
- save_analyzed_frames (boolean): If True, will save analyzed frames (with boxes). Default False
- save_frames_dir (string): Directory to save analyzed frames. Default /tmp
- frame_strategy: (string): various conditions to stop matching as below
- 'most_models': Match the frame that has matched most models (does not include same model alternatives) (Default)
- 'first': Stop at first match
- 'most': Match the frame that has the highest number of detected objects
- 'most_unique' Match the frame that has the highest number of unique detected objects
- resize (int): Width to resize image, default 800
- polygons(object): object # set of polygons that the detected image needs to intersect
Returns:
- object: representing matched frame, consists of:
- box (array): list of bounding boxes for matched frame
- label (array): list of labels for matched frame
- confidence (array): list of confidences for matched frame
- id (int): frame id of matched frame
- img (cv2 image): image grab of matched frame
- array of objects:
- list of boxes,labels,confidences of all frames matched
Note:
The same frames are not retrieved depending on whether you set
``download`` to ``True`` or ``False``. When set to ``True``, we use
OpenCV's frame reading logic and when ``False`` we use ZoneMinder's image.php function
which uses time based approximation. Therefore, the retrieve different frame offsets, but I assume
they should be reasonably close.
"""
self.ml_overrides = ml_overrides
self.stream_options = options
frame_strategy = self.stream_options.get('frame_strategy',
'most_models')
all_matches = []
matched_b = []
matched_e = []
matched_l = []
matched_c = []
matched_models = []
matched_frame_id = None
matched_images = []
matched_frame_img = None
manual_locking = False
if len(self.model_sequence) > 1:
manual_locking = False
self.logger.Debug(
3,
'Using automatic locking as we are switching between models')
else:
manual_locking = True
self.logger.Debug(
3, 'Using manual locking as we are only using one model')
for seq in self.model_sequence:
self.ml_options[seq]['auto_lock'] = False
t = Timer()
media = MediaStream(stream,
'video',
self.stream_options,
logger=self.logger)
self.media = media
polygons = copy.copy(self.stream_options.get('polygons', []))
# Loops across all frames
while self.media.more():
frame = self.media.read()
if frame is None:
self.logger.Debug(1, 'Ran out of frames to read')
break
#fname = '/tmp/{}.jpg'.format(self.media.get_last_read_frame())
#print (f'Writing to {fname}')
#cv2.imwrite( fname ,frame)
self.logger.Debug(
1, 'perf: Starting for frame:{}'.format(
self.media.get_last_read_frame()))
_labels_in_frame = []
_boxes_in_frame = []
_error_boxes_in_frame = []
_confs_in_frame = []
_models_in_frame = []
# For each frame, loop across all models
found = False
for seq in self.model_sequence:
if seq not in self.ml_overrides.get('model_sequence', seq):
self.logger.Debug(
1, 'Skipping {} as it was overridden in ml_overrides'.
format(seq))
continue
self.logger.Debug(
1,
'============ Frame: {} Running {} model in sequence =================='
.format(self.media.get_last_read_frame(), seq))
pre_existing_labels = self.ml_options.get(seq, {}).get(
'general', {}).get('pre_existing_labels')
if pre_existing_labels:
self.logger.Debug(
2,
'Making sure we have matched one of {} in {} before we proceed'
.format(pre_existing_labels, _labels_in_frame))
if not any(x in _labels_in_frame
for x in pre_existing_labels):
self.logger.Debug(
1,
'Did not find pre existing labels, not running model'
)
continue
if not self.models.get(seq):
try:
self._load_models([seq])
if manual_locking:
for m in self.models[seq]:
m.acquire_lock()
except Exception as e:
self.logger.Error(
'Error loading model for {}:{}'.format(seq, e))
self.logger.Debug(2, traceback.format_exc())
continue
same_model_sequence_strategy = self.ml_options.get(
seq, {}).get('general',
{}).get('same_model_sequence_strategy',
'first')
self.logger.Debug(
3, '{} has a same_model_sequence strategy of {}'.format(
seq, same_model_sequence_strategy))
# start of same model iteration
_b_best_in_same_model = []
_l_best_in_same_model = []
_c_best_in_same_model = []
_e_best_in_same_model = []
cnt = 1
# For each model, loop across different variations
for m in self.models[seq]:
self.logger.Debug(
3,
'--------- Frame:{} Running variation: #{} -------------'
.format(self.media.get_last_read_frame(), cnt))
cnt += 1
try:
_b, _l, _c = m.detect(image=frame)
self.logger.Debug(
4,
'This model iteration inside {} found: labels: {},conf:{}'
.format(seq, _l, _c))
except Exception as e:
self.logger.Error('Error running model: {}'.format(e))
self.logger.Debug(2, traceback.format_exc())
continue
# Now let's make sure the labels match our pattern
_b, _l, _c, _e = self._filter_patterns(
seq, _b, _l, _c, polygons)
if _e:
_e_best_in_same_model.extend(_e)
if not len(_l):
continue
if ((same_model_sequence_strategy == 'first')
or ((same_model_sequence_strategy == 'most') and
(len(_l) > len(_l_best_in_same_model))) or
((same_model_sequence_strategy == 'most_unique') and
(len(set(_l)) > len(set(_l_best_in_same_model))))):
_b_best_in_same_model = _b
_l_best_in_same_model = _l
_c_best_in_same_model = _c
_e_best_in_same_model = _e
if _l_best_in_same_model and self.stream_options.get(
'save_analyzed_frames'
) and self.media.get_debug_filename():
d = self.stream_options.get('save_frames_dir', '/tmp')
f = '{}/{}-analyzed-{}.jpg'.format(
d, self.media.get_debug_filename(),
media.get_last_read_frame())
self.logger.Debug(
4, 'Saving analyzed frame: {}'.format(f))
a = utils.draw_bbox(
frame, _b_best_in_same_model,
_l_best_in_same_model, _c_best_in_same_model,
self.stream_options.get('polygons'))
for _b in _e_best_in_same_model:
cv2.rectangle(a, (_b[0], _b[1]), (_b[2], _b[3]),
(0, 0, 255), 1)
cv2.imwrite(f, a)
if (same_model_sequence_strategy == 'first') and len(_b):
self.logger.Debug(
3,
'breaking out of same model loop, as matches found and strategy is "first"'
)
break
# end of same model sequence iteration
# at this state x_best_in_model contains the best match across
# same model variations
if _l_best_in_same_model:
found = True
_labels_in_frame.extend(_l_best_in_same_model)
_boxes_in_frame.extend(_b_best_in_same_model)
_confs_in_frame.extend(_c_best_in_same_model)
_error_boxes_in_frame.extend(_e_best_in_same_model)
_models_in_frame.append(seq)
if (frame_strategy == 'first'):
self.logger.Debug(
2,
'Breaking out of main model loop as strategy is first'
)
break
else:
self.logger.Debug(
2,
'We did not find any {} matches in frame: {}'.format(
seq, self.media.get_last_read_frame()))
# end of primary model sequence
if found:
all_matches.append({
'frame_id': self.media.get_last_read_frame(),
'boxes': _boxes_in_frame,
'error_boxes': _error_boxes_in_frame,
'labels': _labels_in_frame,
'confidences': _confs_in_frame,
'models': _models_in_frame
})
matched_images.append(frame.copy())
if (frame_strategy == 'first'):
self.logger.Debug(
2,
'Frame strategy is first, breaking out of frame loop')
break
# end of while media loop
#print ('*********** MATCH_STRATEGY {}'.format(model_match_strategy))
for idx, item in enumerate(all_matches):
if ((frame_strategy == 'first')
or ((frame_strategy == 'most') and
(len(item['labels']) > len(matched_l)))
or ((frame_strategy == 'most_models') and
(len(item['models']) > len(matched_models)))
or ((frame_strategy == 'most_unique') and
(len(set(item['labels'])) > len(set(matched_l))))):
matched_b = item['boxes']
matched_e = item['error_boxes']
matched_c = item['confidences']
matched_l = item['labels']
matched_frame_id = item['frame_id']
matched_models = item['models']
matched_frame_img = matched_images[idx]
if manual_locking:
for seq in self.model_sequence:
for m in self.models[seq]:
m.release_lock()
diff_time = t.stop_and_get_ms()
self.logger.Debug(
1,
'perf: TOTAL detection sequence (with image loads) took: {} to process {}'
.format(diff_time, stream))
self.media.stop()
matched_data = {
'boxes': matched_b,
'error_boxes': matched_e,
'labels': matched_l,
'confidences': matched_c,
'frame_id': matched_frame_id,
'image_dimensions': self.media.image_dimensions(),
#'type': matched_type,
'image': matched_frame_img,
'polygons': polygons
}
# if invoked again, we need to resize polys
self.has_rescaled = False
return matched_data, all_matches
