def _add_resized(self, image, bbox):
h0, w0 = image_helper.image_h_w(image)
image = image_helper.resize_if_larger(image, self._dim)
h1, w1 = image_helper.image_h_w(image)
self._ratio_h = h0 / h1
self._ratio_w = w0 / w1
tracker = self._create_tracker(self._tracker_algorithm)
x1, y1, w, h = bbox
bbox = (int(x1 / self._ratio_w), int(y1 / self._ratio_h),
int(w / self._ratio_w), int(h / self._ratio_h))
tracker.init(image, bbox)
self._cv_trackers.append(tracker)
def yolo_label_maker(frame, rects, class_id, save_dir):
lines = []
for rect in rects:
y1, x1, y2, x2 = rect
h, w = image_helper.image_h_w(frame)
cx = ((x1 + x2) * 0.5) / w
cy = ((y1 + y2) * 0.5) / h
w = (x2 - x1) / w
h = (y2 - y1) / h
line = "{} {} {} {} {}".format(class_id, cx, cy, w, h)
lines.append(line)
name = str(uuid.uuid4().hex)
image_fn = file_helper.path_join(save_dir, name + ".jpg")
label_fn = file_helper.path_join(save_dir, name + ".txt")
cv2.imwrite(image_fn, frame)
file_helper.write_lines(label_fn, lines)
def process_frame(self, frame, extra_data=None):
super().process_frame(frame)
items_of_groups = {}
results = extra_data.get(constants.EXTRA_DATA_KEY_RESULTS, None)
if results is not None:
h0, w0 = image_helper.image_h_w(frame)
max_x = w0 - 1
max_y = h0 - 1
for runner_name, result in results.items():
remove_indexes = []
for result_index, item in enumerate(result):
rect = item.get(constants.RESULT_KEY_RECT, None)
if rect is not None:
class_name = item.get(constants.RESULT_KEY_CLASS_NAME,
None)
group_name = self._get_group_name(class_name)
if group_name is not None:
if group_name not in items_of_groups:
items_of_groups[group_name] = []
items = items_of_groups[group_name]
bbox = self._rect_to_bbox(rect, max_x, max_y)
if bbox is not None:
item["tr_bbox"] = bbox
item["tr_group_name"] = group_name
remove_indexes.append(result_index)
handled = False
for i, item0 in enumerate(items):
rect0 = item0[constants.RESULT_KEY_RECT]
if geometry_helper.rects_overlap(
rect, rect0, 0.75):
handled = True
if item.get(
constants.RESULT_KEY_SCORE,
0.9) > item0.get(
constants.RESULT_KEY_SCORE,
0.9):
items[i] = item
break
if not handled:
items.append(item)
remove_indexes.sort(reverse=True)
for i in remove_indexes:
del result[i]
res = []
for group_name, items in items_of_groups.items():
if len(items) > 0:
self._cv_tracker[group_name] = CvTracker(
self._tracker_algorithm)
self._last_items[group_name] = items
for item in items:
bbox = item["tr_bbox"]
self._cv_tracker[group_name].add(frame, bbox)
else:
if group_name not in self._last_items[group_name]:
self._last_items[group_name] = []
items = self._last_items[group_name]
items_ok = []
for i, (ok, bbox) in enumerate(
self._cv_tracker[group_name].update(frame)):
if ok:
item = items[i]
item["tr_bbox"] = bbox
item[constants.RESULT_KEY_RECT] = self._bbox_to_rect(
bbox)
items_ok.append(item)
items = items_ok
center_item_index = []
active_counts = {}
class_dets = {}
for i, item in enumerate(items):
if "tr_bbox" in item:
rect = item.get(constants.RESULT_KEY_RECT, None)
group_name1 = item["tr_group_name"]
if group_name1 not in class_dets:
class_dets[group_name1] = []
score = item.get(constants.RESULT_KEY_SCORE, 0.9)
det = [rect[1], rect[0], rect[3], rect[2], score]
if not self._det_exists(class_dets[group_name1], det):
class_dets[group_name1].append(det)
center_item_index.append(
(self._get_box_center(det), item, i))
if group_name not in self._g:
self._g[group_name] = {}
g = self._g[group_name]
g_sorts = g.get("sorts", {})
g_memory = g.get("memory", {})
previous = g_memory.copy()
g_memory = {}
for name, dets in class_dets.items():
if name not in g_sorts:
g_sorts[name] = Sort(max_age=10,
min_hits=1,
iou_threshold=0.001)
# g_sorts[name] = Sort(max_age=1, min_hits=3, iou_threshold=0.3)
# g_sorts[name] = Sort()
# g_sorts[name] = Sort(max_age=1, min_hits=1, iou_threshold=0.03)
# g_sorts[name] = Sort(max_age=10, min_hits=0, iou_threshold=0.000001)
# g_sorts[name] = Sort(max_age=30, min_hits=1, iou_threshold=0.001)
sort = g_sorts[
name] # ref: https://github.com/abewley/sort https://github.com/HodenX/python-traffic-counter-with-yolo-and-sort
dets1 = np.array(dets)
tracks = sort.update(dets1)
active_counts[name] = len(tracks)
boxes = []
track_ids = []
for track in tracks:
boxes.append(
[track[0], track[1], track[2], track[3], track[4]])
track_id = int(track[4])
track_ids.append(track_id)
if track_id in previous:
track0 = previous[track_id]
dist = (math.fabs(track[0] - track0[0]) +
math.fabs(track[1] - track0[1]) +
math.fabs(track[2] - track0[2]) +
math.fabs(track[3] - track0[3])) / 4.0
if dist > 30:
g_memory[track_id] = boxes[-1]
else:
g_memory[track_id] = track0
else:
g_memory[track_id] = boxes[-1]
if len(boxes) > 0:
for i, box in enumerate(boxes):
track_id = track_ids[i]
if track_id in previous:
previous_box = previous.get(track_id, None)
if previous_box is not None:
c = self._get_box_center(box)
min_dist = sys.maxsize
result_item = None
for center, item, i0 in center_item_index:
dist = geometry_helper.distance(c, center)
if dist < min_dist:
min_dist = dist
result_item = item
if result_item is not None:
result_item[constants.
RESULT_KEY_TRACK_ID] = track_id
result_item[
constants.
RESULT_KEY_PREVIEW_KEY] = track_id
else:
result_item = items[i]
result_item[
constants.RESULT_KEY_TRACK_ID] = track_id
result_item[
constants.RESULT_KEY_PREVIEW_KEY] = track_id
else:
for i in range(len(items)):
result_item = items[i]
result_item[constants.RESULT_KEY_TRACK_ID] = -1
result_item[constants.RESULT_KEY_PREVIEW_KEY] = -1
g["sorts"] = g_sorts
g["memory"] = g_memory
res.extend(items)
return res
def _get_preview(self, image, results):
def get_color(obj, class_preview_key_, color=None):
if color is None:
color = [255, 255, 255]
if class_preview_key_ is not None:
if not hasattr(obj, "__colors"):
setattr(obj, "__colors", {})
dic = getattr(obj, "__colors")
if class_preview_key_ in dic:
color = dic[class_preview_key_]
else:
color = dic[class_preview_key_] = self._new_color()
return color
def draw_rect(obj, img, c1_, c2_, class_preview_key_, color=None):
if color is None:
color = get_color(obj, class_preview_key_, color)
cv2.rectangle(img, (c1_[0], c1_[1]), (c2_[0], c2_[1]), color=[1, 1, 1], thickness=3)
cv2.rectangle(img, (c1_[0], c1_[1]), (c2_[0], c2_[1]), color=color, thickness=2)
show_debug_texts = self.__preview_show_debug_texts
show_runner_info = self.__preview_show_runner_info
show_score = self.__preview_show_score
rect_filters = self.__preview_show_rect_filters
show_rect_name = self.__preview_show_rect_name
show_track_id = self.__preview_show_track_id
show_track_pnt = self.__preview_show_track_pnt
h, w, *_ = image.shape
line_height = 20
font_scale = 0.4
current_line = [20, 0]
current_line_bottom = [20, h]
data_added = []
if results is not None:
for result in results:
debug_texts = []
text_type = ""
has_data = False
if show_runner_info:
total_elapsed_time = result.get("total_elapsed_time", None)
if total_elapsed_time is not None:
image_helper.put_text(image, total_elapsed_time, [w - 150, h - 30], color=[200, 200, 128],
font_scale=0.4)
continue
values = {}
elapsed_time = values["elapsed_time"] = result.get("elapsed_time", None)
class_name = values[constants.RESULT_KEY_CLASS_NAME] = result.get(constants.RESULT_KEY_CLASS_NAME, None)
class_preview_key = values[constants.RESULT_KEY_PREVIEW_KEY] = result.get(
constants.RESULT_KEY_PREVIEW_KEY, class_name)
score = values[constants.RESULT_KEY_SCORE] = result.get(constants.RESULT_KEY_SCORE, None)
rect = values[constants.RESULT_KEY_RECT] = result.get(constants.RESULT_KEY_RECT, None)
rect_debug_text = values[constants.RESULT_KEY_RECT_DEBUG_TEXT] = result.get(
constants.RESULT_KEY_RECT_DEBUG_TEXT, None)
data = values[constants.RESULT_KEY_DATA] = result.get(constants.RESULT_KEY_DATA, None)
debug_text = values[constants.RESULT_KEY_DEBUG] = result.get(constants.RESULT_KEY_DEBUG, None)
if debug_text is not None:
debug_texts.append(debug_text)
def _get_score_txt(text_, score_):
res = "%{:.2f} ".format(score_ * 100) if isinstance(score_, float) else score_
if text_ is not None and len(text_) > 0:
return "{}- {} ".format(text_, res)
else:
return res
text = ""
if class_name is not None:
text = NDUUtility.debug_conv_turkish(class_name) + " "
if show_score and score is not None:
text = _get_score_txt(text, score)
elif show_score and score is not None:
text = _get_score_txt(None, score)
if data is not None:
add_txt = " data: " + str(data)
data_added.append(add_txt)
text = text + add_txt
has_data = True
if rect is not None:
c = np.array(rect[:4], dtype=np.int32)
c1, c2 = [c[1], c[0]], (c[3], c[2])
color_rect = get_color(self, class_preview_key, result.get(constants.RESULT_KEY_RECT_COLOR, None))
show_rect = True
if rect_filters is not None:
show_rect = False
for rect_filter in rect_filters:
if string_helper.wildcard(class_name, rect_filter):
show_rect = True
break
if show_rect:
draw_rect(self, image, c1, c2, class_preview_key, color_rect)
else:
text = ""
if show_rect and show_rect_name and len(text) > 0:
# c1[1] = c1[1] + line_height
c1[1] -= 10
if rect_debug_text is not None:
text += " - " + rect_debug_text
image_helper.put_text(image, text, c1)
text = ""
if show_rect and show_track_id:
track_id = result.get(constants.RESULT_KEY_TRACK_ID, None)
if track_id is not None:
# c1[1] -= line_height * 1.5
c1[1] -= 15
image_helper.put_text(image, "{}".format(track_id), c1)
if show_track_pnt:
track_id = result.get(constants.RESULT_KEY_TRACK_ID, None)
if track_id is not None:
if self._track_pnt_layer is None:
h, w = image_helper.image_h_w(image)
self._track_pnt_layer = np.zeros((h, w, 3), np.uint8)
self._track_pnt_layer = image_helper.change_brightness(self._track_pnt_layer, -1)
y1, x1, y2, x2 = tuple(rect)
pnt = (int(x1 + (x2 - x1) * 0.5), int(y2))
pnts_key = str(track_id)
pnts = self._track_pnts.get(pnts_key, [])
pnts.append(pnt)
self._track_pnts[pnts_key] = pnts
if len(pnts) > 1:
pts = np.array(pnts, np.int32)
# cv2.polylines(self._track_pnt_layer, [pts], False, color, 5)
# cv2.polylines(image, [pts], False, color_rect, 5)
thickness = 10
for i in range(len(pnts) - 1, 0, -1):
p0 = pnts[i]
p1 = pnts[i - 1]
cv2.line(image, p0, p1, color_rect, thickness)
if i % 10 == 0:
thickness -= 1
if thickness < 1:
# pnts = pnts[i: len(pnts)]
break
# color = [255,255,255]
# cv2.circle(self._track_pnt_layer, pnt, 1, color, 8)
# image = cv2.addWeighted(image, 1, self._track_pnt_layer, 1,)
# ret, mask = cv2.threshold(self._track_pnt_layer, 1, 255, cv2.THRESH_BINARY)
# mask = cv2.bitwise_not(mask)
# mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
# image = cv2.bitwise_and(image, image, mask=mask)
# image = cv2.addWeighted(image, 1, self._track_pnt_layer, 0.1, 1)
# image += self._track_pnt_layer
# image = self._track_pnt_layer.copy()
if elapsed_time is not None:
text_type = elapsed_time + " " + text_type
if len(text) > 0:
text_type = text_type + text + " "
if show_runner_info and len(text_type) > 0:
current_line[1] += line_height
if show_runner_info:
if not has_data:
image_helper.put_text(image, text_type, current_line, font_scale=font_scale)
else:
image_helper.put_text(image, text_type, current_line, color=[0, 0, 255],
font_scale=font_scale)
if show_debug_texts and len(debug_texts) > 0:
for debug_text in debug_texts:
current_line_bottom[1] -= line_height * 2
image_helper.put_text(image, debug_text, current_line_bottom, color=[255, 250, 99],
font_scale=font_scale * 2.75, thickness=2, back_color=[1, 1, 1])
if show_runner_info:
if len(data_added) > 0:
show_last_data_frame_time = time.time()
for data in data_added:
if data not in self.__preview_last_data:
self.__preview_last_data.append(data)
self.__preview_last_data_show_times.append(show_last_data_frame_time)
else:
self.__preview_last_data_show_times[
self.__preview_last_data.index(data)] = show_last_data_frame_time
for i in reversed(range(len(self.__preview_last_data))):
elapsed = time.time() - self.__preview_last_data_show_times[i]
if elapsed > 5:
del self.__preview_last_data_show_times[i]
del self.__preview_last_data[i]
for last_data in self.__preview_last_data:
current_line[1] += line_height
image_helper.put_text(image, last_data, current_line, color=[0, 255, 255])
return image
def process_frame(self, frame, extra_data=None):
def to_bbox(coordinates, rect_, rh_, rw_):
x1 = coordinates[0]["x"] * rw_
y1 = coordinates[0]["y"] * rh_
x2 = coordinates[2]["x"] * rw_
y2 = coordinates[2]["y"] * rh_
if rect_ is not None:
x1 += rect_[1]
y1 += rect_[0]
x2 += rect_[1]
y2 += rect_[0]
return [y1, x1, y2, x2]
def enumerate_images(frame_):
result = yolo_helper.predict_v4(self.sess_tuple, self.input_size,
self.class_names, frame)
for _class_name, _score, rect0, item_ in NDUUtility.enumerate_result_items(
result, return_item=True):
rect1 = geometry_helper.add_padding_rect(rect0, 0.5)
yield image_helper.crop(frame, rect1), rect0, item_
res = []
for image, rect, item in enumerate_images(frame):
h0, w0 = image_helper.image_h_w(image)
# def order_points(pts):
# # initialzie a list of coordinates that will be ordered
# # such that the first entry in the list is the top-left,
# # the second entry is the top-right, the third is the
# # bottom-right, and the fourth is the bottom-left
# rect = np.zeros((4, 2), dtype="float32")
#
# # the top-left point will have the smallest sum, whereas
# # the bottom-right point will have the largest sum
# s = pts.sum(axis=1)
# rect[0] = pts[np.argmin(s)]
# rect[2] = pts[np.argmax(s)]
#
# # now, compute the difference between the points, the
# # top-right point will have the smallest difference,
# # whereas the bottom-left will have the largest difference
# diff = np.diff(pts, axis=1)
# rect[1] = pts[np.argmin(diff)]
# rect[3] = pts[np.argmax(diff)]
#
# # return the ordered coordinates
# return rect
#
# def four_point_transform(image, pts):
# # obtain a consistent order of the points and unpack them
# # individually
# rect = order_points(pts)
# (tl, tr, br, bl) = rect
#
# # compute the width of the new image, which will be the
# # maximum distance between bottom-right and bottom-left
# # x-coordiates or the top-right and top-left x-coordinates
# widthA = np.sqrt(((br[0] - bl[0]) ** 2) + ((br[1] - bl[1]) ** 2))
# widthB = np.sqrt(((tr[0] - tl[0]) ** 2) + ((tr[1] - tl[1]) ** 2))
# maxWidth = max(int(widthA), int(widthB))
#
# # compute the height of the new image, which will be the
# # maximum distance between the top-right and bottom-right
# # y-coordinates or the top-left and bottom-left y-coordinates
# heightA = np.sqrt(((tr[0] - br[0]) ** 2) + ((tr[1] - br[1]) ** 2))
# heightB = np.sqrt(((tl[0] - bl[0]) ** 2) + ((tl[1] - bl[1]) ** 2))
# maxHeight = max(int(heightA), int(heightB))
#
# # now that we have the dimensions of the new image, construct
# # the set of destination points to obtain a "birds eye view",
# # (i.e. top-down view) of the image, again specifying points
# # in the top-left, top-right, bottom-right, and bottom-left
# # order
# dst = np.array([
# [0, 0],
# [maxWidth - 1, 0],
# [maxWidth - 1, maxHeight - 1],
# [0, maxHeight - 1]], dtype="float32")
#
# # compute the perspective transform matrix and then apply it
# M = cv2.getPerspectiveTransform(rect, dst)
# warped = cv2.warpPerspective(image, M, (maxWidth, maxHeight))
# return warped
#
# def deskew(image):
# gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# gray = cv2.bitwise_not(gray)
# thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
# coords = np.column_stack(np.where(thresh > 0))
# angle = cv2.minAreaRect(coords)[-1]
# if angle < -45:
# angle = -(90 + angle)
# else:
# angle = -angle
# (h, w) = image.shape[:2]
# center = (w // 2, h // 2)
# M = cv2.getRotationMatrix2D(center, angle, 1.0)
# return cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)
#
# # def remove_noise_and_smooth(file_name):
# # img = cv2.imread(file_name, 0)
# # filtered = cv2.adaptiveThreshold(img.astype(np.uint8), 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 9, 41)
# # kernel = np.ones((1, 1), np.uint8)
# # opening = cv2.morphologyEx(filtered, cv2.MORPH_OPEN, kernel)
# # closing = cv2.morphologyEx(opening, cv2.MORPH_CLOSE, kernel)
# # img = image_smoothening(img)
# # or_image = cv2.bitwise_or(img, closing)
# # return or_image
#
# # image = deskew(image)
# # image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
h1, w1 = image_helper.image_h_w(image)
while w1 < 400:
image = cv2.pyrUp(image)
h1, w1 = image_helper.image_h_w(image)
# cv2.imshow("lpr", image)
# cv2.waitKey(500)
success, encoded_image = cv2.imencode('.jpg', image)
content2 = encoded_image.tobytes()
results = self._alpr.recognize_array(content2)
added = False
#print("LPR: ", item.get(constants.RESULT_KEY_RECT))
for plate in results['results']:
txt = plate["plate"]
if len(txt) > 2:
score = plate["confidence"] / 100.0
if score > 0.01:
city_code = txt[0:2]
city_name = None
if city_code in self._cities:
city_name = self._cities[city_code]
if city_name is None:
class_name = "PL: {}".format(txt)
else:
class_name = "PL: {} {}".format(city_name, txt)
val = {
constants.RESULT_KEY_RECT: rect,
constants.RESULT_KEY_SCORE: score,
constants.RESULT_KEY_CLASS_NAME: class_name
}
if self._send_data:
if city_name is None:
val[constants.RESULT_KEY_DATA] = {"pl": txt}
else:
val[constants.RESULT_KEY_DATA] = {
"pl": txt,
"city": city_name
}
res.append(val)
added = True
if not added:
val = {
constants.RESULT_KEY_RECT: rect,
constants.RESULT_KEY_SCORE: 0,
constants.RESULT_KEY_CLASS_NAME: "PL: "
}
res.append(val)
return res