def spatial_box_grouping(self, frame_bboxes, min_combined_ratio=None):
if min_combined_ratio is None:
min_combined_ratio = self.min_combined_ratio
current_boxes = list(frame_bboxes)
merged_boxes = True
while merged_boxes:
merged_boxes = False
# sort bounding boxes by descending size ...
boxes_by_size = []
for x1, y1, x2, y2 in current_boxes:
w = x2 - x1 + 1
h = y2 - y1 + 1
area = w * h
boxes_by_size.append((area, (x1, y1, x2, y2)))
boxes_by_size = sorted(boxes_by_size,
reverse=True,
key=lambda x: x[0])
# print(boxes_by_size)
# create interval index to find matches quicker (all boxes against all boxes from same frame)
int_index_x = IntervalIndex(True)
int_index_y = IntervalIndex(True)
for box_idx, (area, (x1, y1, x2, y2)) in enumerate(boxes_by_size):
int_index_x.add(x1, x2 + 1, box_idx)
int_index_y.add(y1, y2 + 1, box_idx)
# ... find pair-wise matches ...
set_x = set(int_index_x.find_matches(int_index_x))
set_y = set(int_index_y.find_matches(int_index_y))
# .... list of all pairs of boxes with intersecting intervals in X and Y
merge_candidates = sorted(list(set_x.intersection(set_y)))
# ... filter self-matches and repetitions ...
merge_candidates = [(box_idx1, box_idx2)
for box_idx1, box_idx2 in merge_candidates
if box_idx1 < box_idx2]
# ... split by first box ..
candidates_by_box = {idx: [] for idx in range(len(boxes_by_size))}
for box_idx1, box_idx2 in merge_candidates:
candidates_by_box[box_idx1].append(box_idx2)
# print(merge_candidates)
# print(candidates_by_box)
box_added = [False] * len(boxes_by_size)
current_boxes = []
# for each box (sorted by size)
for box_idx in range(len(boxes_by_size)):
# if this box has been previously added (merged with ealier box)
if box_added[box_idx]:
# skip ...
continue
box_added[box_idx] = True
# current box boundaries
c_area, (c_x1, c_y1, c_x2, c_y2) = boxes_by_size[box_idx]
for second_box_idx in candidates_by_box[box_idx]:
if box_added[second_box_idx]:
# skip merge candidate ...
continue
# get box boundaries ...
o_area, (o_x1, o_y1, o_x2,
o_y2) = boxes_by_size[second_box_idx]
comb_area_ratio = self.get_combined_box_area_ratio(
(c_x1, c_y1, c_x2, c_y2), (o_x1, o_y1, o_x2, o_y2))
# print(((c_x1, c_y1, c_x2, c_y2), boxes_by_size[second_box_idx], comb_area_ratio, box_idx, second_box_idx))
if comb_area_ratio >= min_combined_ratio:
# merge!
# expand current bounding box to include the smaller box ...
c_x1 = min(c_x1, o_x1)
c_y1 = min(c_y1, o_y1)
c_x2 = max(c_x2, o_x2)
c_y2 = max(c_y2, o_y2)
# mark second box as added, so it won't be added to the current list ..
box_added[second_box_idx] = True
merged_boxes = True
# add to the next set of accepted boxes
current_boxes.append((c_x1, c_y1, c_x2, c_y2))
"""
if len(frame_bboxes) > 30:
original_img = self.visualize_boxes(frame_bboxes, (255, 0, 0))
final_img = self.visualize_boxes(current_boxes, (0, 255, 0))
final_img[:, :, 0] = original_img[:, :, 0]
debug_img = cv2.resize(final_img,(960, 540))
cv2.imshow("check", debug_img)
cv2.waitKey()
raise Exception("Error!")
"""
return current_boxes
def add_frame(self, frame_bboxes):
current_bboxes = self.spatial_box_grouping(frame_bboxes,
self.min_combined_ratio)
current_bboxes_idxs = []
if self.img_idx == 0:
# simply copy all
for bbox_id, bbox in enumerate(current_bboxes):
# add the box to list of unique boxes ...
self.unique_bbox_objects.append(bbox)
# frames on which the bbox appears, raw label assigned to that CC
self.unique_bbox_frames.append([(0, bbox_id)])
bbox_idx = len(self.unique_bbox_objects) - 1
current_bboxes_idxs.append((bbox_idx, bbox))
self.bbox_last_frame.append(0)
self.bbox_active.append(bbox_idx)
# add to indices ...
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.add(x1, x2, bbox_idx)
self.bbox_int_index_y.add(y1, y2, bbox_idx)
else:
# create indices for current bboxes
other_index_x = IntervalIndex(True)
other_index_y = IntervalIndex(True)
for bbox_idx, (x1, y1, x2, y2) in enumerate(current_bboxes):
other_index_x.add(x1, x2, bbox_idx)
other_index_y.add(y1, y2, bbox_idx)
# compute CC with matching regions
set_x = set(other_index_x.find_matches(self.bbox_int_index_x))
set_y = set(other_index_y.find_matches(self.bbox_int_index_y))
# list of all pairs of CC with intersecting intervals in X and Y
merged = sorted(list(set_x.intersection(set_y)))
self.tempo_count += len(merged)
# check every matching CC
pre_add_size = len(self.bbox_active)
next_match_idx = 0
for bbox_idx, bbox in enumerate(current_bboxes):
found = False
# check all matches in the list of matches for current CC
while next_match_idx < len(
merged) and merged[next_match_idx][0] == bbox_idx:
if not found:
prev_idx = merged[next_match_idx][1]
prev_bbox = self.unique_bbox_objects[prev_idx]
bbox_IOU = self.get_bboxes_IOU(bbox, prev_bbox)
# print(bbox_IOU)
if bbox_IOU >= self.min_temporal_IOU:
# assume they are equivalent
found = True
self.unique_bbox_frames[prev_idx].append(
(self.img_idx, bbox_idx))
current_bboxes_idxs.append((prev_idx, bbox))
# update last frame seen for this cc...
self.bbox_last_frame[prev_idx] = self.img_idx
next_match_idx += 1
# Not match was found?
if not found:
# add
self.unique_bbox_objects.append(bbox)
self.unique_bbox_frames.append([(self.img_idx, bbox_idx)])
new_bbox_idx = len(self.unique_bbox_objects) - 1
current_bboxes_idxs.append((new_bbox_idx, bbox))
self.bbox_last_frame.append(self.img_idx)
self.bbox_active.append(new_bbox_idx)
# add to indices ...
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.add(x1, x2, new_bbox_idx)
self.bbox_int_index_y.add(y1, y2, new_bbox_idx)
# remove CC that are no longer active
pre_remove_size = len(self.bbox_active)
tempo_pos = 0
while tempo_pos < len(self.bbox_active):
bbox_idx = self.bbox_active[tempo_pos]
if self.img_idx - self.bbox_last_frame[
bbox_idx] >= self.max_gap:
# no longer active ..
# delete from active list
del self.bbox_active[tempo_pos]
# delete from interval indices
bbox = self.unique_bbox_objects[bbox_idx]
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.remove(x1, x2, bbox_idx)
self.bbox_int_index_y.remove(y1, y2, bbox_idx)
#print self.cc_last_frame[cc_idx],
else:
# still active
tempo_pos += 1
"""
total_added = pre_remove_size - pre_add_size
total_removed = pre_remove_size - len(self.bbox_active)
msg = "{0:d} , (Added: {1:d}, Removed: {2:d})".format(len(self.bbox_active), total_added, total_removed)
print(msg)
"""
self.bbox_idx_per_frame.append(current_bboxes_idxs)
self.img_idx += 1
if self.verbose:
msg = "[{0:d} ({1:d}, {2:d})]".format(
self.img_idx, len(current_bboxes),
len(self.unique_bbox_objects))
print(msg, end="\r")
def add_frame(self, img, input_binary=False):
# get the CC
if input_binary:
# use given binary
binary = img
else:
# binarize
binary = Binarizer.backgroundSubtractionBinarization(img.astype('uint8'))
current_cc = Labeler.extractSpatioTemporalContent(binary, self.fake_age)
current_cc_idxs = []
if self.img_idx == 0:
# simply copy all
for cc in current_cc:
self.unique_cc_objects.append(cc) # CC objet
# frames on which the CC appears, raw label assigend to that CC
self.unique_cc_frames.append([(0, cc.cc_id + 1)])
cc_idx = len(self.unique_cc_objects) - 1
current_cc_idxs.append((cc_idx, cc))
self.cc_last_frame.append(0)
self.cc_active.append(cc_idx)
# add to indices ...
self.cc_int_index_x.add(cc.min_x, cc.max_x + 1, cc_idx)
self.cc_int_index_y.add(cc.min_y, cc.max_y + 1, cc_idx)
else:
# create indices for current CC
other_index_x = IntervalIndex(True)
other_index_y = IntervalIndex(True)
for cc_idx, cc in enumerate(current_cc):
other_index_x.add(cc.min_x, cc.max_x + 1, cc_idx)
other_index_y.add(cc.min_y, cc.max_y + 1, cc_idx)
# compute CC with matching regions
set_x = set(other_index_x.find_matches(self.cc_int_index_x))
set_y = set(other_index_y.find_matches(self.cc_int_index_y))
# list of all pairs of CC with intersecting intervals in X and Y
merged = sorted(list(set_x.intersection(set_y)))
self.tempo_count += len(merged)
# check every matching CC
pre_add_size = len(self.cc_active)
next_match_idx = 0
for cc_idx, cc in enumerate(current_cc):
found = False
# check all matches in the list of matches for current CC
while next_match_idx < len(merged) and merged[next_match_idx][0] == cc_idx:
if not found:
prev_idx = merged[next_match_idx][1]
prev_cc = self.unique_cc_objects[prev_idx]
recall, precision = cc.getOverlapFMeasure(prev_cc, False, False)
if recall >= self.min_recall and precision >= self.min_precision:
# assume they are equivalent
found = True
self.unique_cc_frames[prev_idx].append((self.img_idx, cc.cc_id + 1))
current_cc_idxs.append((prev_idx, cc))
# update last frame seen for this cc...
self.cc_last_frame[prev_idx] = self.img_idx
next_match_idx += 1
# Not match was found?
if not found:
# add
self.unique_cc_objects.append(cc)
self.unique_cc_frames.append([(self.img_idx, cc.cc_id + 1)])
new_cc_idx = len(self.unique_cc_objects) - 1
current_cc_idxs.append((new_cc_idx, cc))
self.cc_last_frame.append(self.img_idx)
self.cc_active.append(new_cc_idx)
# add to indices ...
self.cc_int_index_x.add(cc.min_x, cc.max_x + 1, new_cc_idx)
self.cc_int_index_y.add(cc.min_y, cc.max_y + 1, new_cc_idx)
# remove CC that are no longer active
pre_remove_size = len(self.cc_active)
tempo_pos = 0
while tempo_pos < len(self.cc_active):
cc_idx = self.cc_active[tempo_pos]
if self.img_idx - self.cc_last_frame[cc_idx] >= self.max_gap:
# no longer active ..
# delete from active list
del self.cc_active[tempo_pos]
# delete from interval indices
cc = self.unique_cc_objects[cc_idx]
self.cc_int_index_x.remove(cc.min_x, cc.max_x + 1, cc_idx)
self.cc_int_index_y.remove(cc.min_y, cc.max_y + 1, cc_idx)
#print self.cc_last_frame[cc_idx],
else:
# still active
tempo_pos += 1
#print(str(len(self.cc_active)) + ", (Added: " + str(pre_remove_size - pre_add_size) + ", Removed: " + str(pre_remove_size - len(self.cc_active)) + ")")
self.cc_idx_per_frame.append(current_cc_idxs)
self.img_idx += 1
if self.verbose:
print("[" + str(self.img_idx) + " (" + str(len(current_cc)) + ", " + str(len(self.unique_cc_objects)) + ")]", end="\r")