class BBoxStabilityEstimator:
RefinePerBoxPerSegment = 1
RefineAllBoxesPerSegment = 2
RefineAllBoxesAllSegments = 3
def __init__(self,
width,
height,
min_combined_ratio,
min_temporal_IOU,
max_gap,
verbose=False):
self.width = width
self.height = height
self.min_combined_ratio = min_combined_ratio
self.min_temporal_IOU = min_temporal_IOU
self.max_gap = max_gap
self.unique_bbox_objects = []
self.unique_bbox_frames = []
self.bbox_idx_per_frame = []
self.bbox_int_index_x = IntervalIndex(True)
self.bbox_int_index_y = IntervalIndex(True)
self.img_idx = 0
self.tempo_count = 0
# optimizing the matching ...
self.bbox_last_frame = []
self.bbox_active = []
self.verbose = verbose
def get_raw_bbox_count(self):
total = 0
for current_frame in self.bbox_idx_per_frame:
total += len(current_frame)
return total
def get_inter_bbox(self, box1, box2):
b1_x1, b1_y1, b1_x2, b1_y2 = box1
b2_x1, b2_y1, b2_x2, b2_y2 = box2
comb_inter_x1 = max(b1_x1, b2_x1)
comb_inter_y1 = max(b1_y1, b2_y1)
comb_inter_x2 = min(b1_x2, b2_x2)
comb_inter_y2 = min(b1_y2, b2_y2)
return comb_inter_x1, comb_inter_y1, comb_inter_x2, comb_inter_y2
def get_outer_bbox(self, box1, box2):
b1_x1, b1_y1, b1_x2, b1_y2 = box1
b2_x1, b2_y1, b2_x2, b2_y2 = box2
comb_outer_x1 = min(b1_x1, b2_x1)
comb_outer_y1 = min(b1_y1, b2_y1)
comb_outer_x2 = max(b1_x2, b2_x2)
comb_outer_y2 = max(b1_y2, b2_y2)
return comb_outer_x1, comb_outer_y1, comb_outer_x2, comb_outer_y2
def get_bbox_area(self, box):
x1, y1, x2, y2 = box
b1_w = x2 - x1
b1_h = y2 - y1
# check for invalid boxes with negative width or height
if b1_w <= 0.0 or b1_h <= 0.0:
return 0.0
return b1_w * b1_h
def get_bboxes_IOU(self, box1, box2):
# area of first box ...
b1_area = self.get_bbox_area(box1)
# area of second box ...
b2_area = self.get_bbox_area(box2)
# intersection between two boxes ...
comb_inter = self.get_inter_bbox(box1, box2)
inter_area = self.get_bbox_area(comb_inter)
combined_union_area = b1_area + b2_area - inter_area
# print((b1_area, b2_area, box1, box2, comb_inter, inter_area, combined_union_area))
return inter_area / combined_union_area
def get_combined_box_area_ratio(self, box1, box2):
# area of first box ...
b1_area = self.get_bbox_area(box1)
# area of second box ...
b2_area = self.get_bbox_area(box2)
# intersection between two boxes ...
comb_inter = self.get_inter_bbox(box1, box2)
inter_area = self.get_bbox_area(comb_inter)
combined_union_area = b1_area + b2_area - inter_area
comb_outer = self.get_outer_bbox(box1, box2)
outer_area = self.get_bbox_area(comb_outer)
area_ratio = combined_union_area / outer_area
return area_ratio
def visualize_boxes(self, bboxes, color=(255, 0, 0)):
out_img = np.zeros((self.height, self.width, 3), np.uint8)
for x1, y1, x2, y2 in bboxes:
cv2.rectangle(out_img, (int(x1), int(y1)), (int(x2), int(y2)),
color,
thickness=2)
return out_img
def spatial_box_grouping(self, frame_bboxes, min_combined_ratio=None):
if min_combined_ratio is None:
min_combined_ratio = self.min_combined_ratio
current_boxes = list(frame_bboxes)
merged_boxes = True
while merged_boxes:
merged_boxes = False
# sort bounding boxes by descending size ...
boxes_by_size = []
for x1, y1, x2, y2 in current_boxes:
w = x2 - x1 + 1
h = y2 - y1 + 1
area = w * h
boxes_by_size.append((area, (x1, y1, x2, y2)))
boxes_by_size = sorted(boxes_by_size,
reverse=True,
key=lambda x: x[0])
# print(boxes_by_size)
# create interval index to find matches quicker (all boxes against all boxes from same frame)
int_index_x = IntervalIndex(True)
int_index_y = IntervalIndex(True)
for box_idx, (area, (x1, y1, x2, y2)) in enumerate(boxes_by_size):
int_index_x.add(x1, x2 + 1, box_idx)
int_index_y.add(y1, y2 + 1, box_idx)
# ... find pair-wise matches ...
set_x = set(int_index_x.find_matches(int_index_x))
set_y = set(int_index_y.find_matches(int_index_y))
# .... list of all pairs of boxes with intersecting intervals in X and Y
merge_candidates = sorted(list(set_x.intersection(set_y)))
# ... filter self-matches and repetitions ...
merge_candidates = [(box_idx1, box_idx2)
for box_idx1, box_idx2 in merge_candidates
if box_idx1 < box_idx2]
# ... split by first box ..
candidates_by_box = {idx: [] for idx in range(len(boxes_by_size))}
for box_idx1, box_idx2 in merge_candidates:
candidates_by_box[box_idx1].append(box_idx2)
# print(merge_candidates)
# print(candidates_by_box)
box_added = [False] * len(boxes_by_size)
current_boxes = []
# for each box (sorted by size)
for box_idx in range(len(boxes_by_size)):
# if this box has been previously added (merged with ealier box)
if box_added[box_idx]:
# skip ...
continue
box_added[box_idx] = True
# current box boundaries
c_area, (c_x1, c_y1, c_x2, c_y2) = boxes_by_size[box_idx]
for second_box_idx in candidates_by_box[box_idx]:
if box_added[second_box_idx]:
# skip merge candidate ...
continue
# get box boundaries ...
o_area, (o_x1, o_y1, o_x2,
o_y2) = boxes_by_size[second_box_idx]
comb_area_ratio = self.get_combined_box_area_ratio(
(c_x1, c_y1, c_x2, c_y2), (o_x1, o_y1, o_x2, o_y2))
# print(((c_x1, c_y1, c_x2, c_y2), boxes_by_size[second_box_idx], comb_area_ratio, box_idx, second_box_idx))
if comb_area_ratio >= min_combined_ratio:
# merge!
# expand current bounding box to include the smaller box ...
c_x1 = min(c_x1, o_x1)
c_y1 = min(c_y1, o_y1)
c_x2 = max(c_x2, o_x2)
c_y2 = max(c_y2, o_y2)
# mark second box as added, so it won't be added to the current list ..
box_added[second_box_idx] = True
merged_boxes = True
# add to the next set of accepted boxes
current_boxes.append((c_x1, c_y1, c_x2, c_y2))
"""
if len(frame_bboxes) > 30:
original_img = self.visualize_boxes(frame_bboxes, (255, 0, 0))
final_img = self.visualize_boxes(current_boxes, (0, 255, 0))
final_img[:, :, 0] = original_img[:, :, 0]
debug_img = cv2.resize(final_img,(960, 540))
cv2.imshow("check", debug_img)
cv2.waitKey()
raise Exception("Error!")
"""
return current_boxes
def add_frame(self, frame_bboxes):
current_bboxes = self.spatial_box_grouping(frame_bboxes,
self.min_combined_ratio)
current_bboxes_idxs = []
if self.img_idx == 0:
# simply copy all
for bbox_id, bbox in enumerate(current_bboxes):
# add the box to list of unique boxes ...
self.unique_bbox_objects.append(bbox)
# frames on which the bbox appears, raw label assigned to that CC
self.unique_bbox_frames.append([(0, bbox_id)])
bbox_idx = len(self.unique_bbox_objects) - 1
current_bboxes_idxs.append((bbox_idx, bbox))
self.bbox_last_frame.append(0)
self.bbox_active.append(bbox_idx)
# add to indices ...
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.add(x1, x2, bbox_idx)
self.bbox_int_index_y.add(y1, y2, bbox_idx)
else:
# create indices for current bboxes
other_index_x = IntervalIndex(True)
other_index_y = IntervalIndex(True)
for bbox_idx, (x1, y1, x2, y2) in enumerate(current_bboxes):
other_index_x.add(x1, x2, bbox_idx)
other_index_y.add(y1, y2, bbox_idx)
# compute CC with matching regions
set_x = set(other_index_x.find_matches(self.bbox_int_index_x))
set_y = set(other_index_y.find_matches(self.bbox_int_index_y))
# list of all pairs of CC with intersecting intervals in X and Y
merged = sorted(list(set_x.intersection(set_y)))
self.tempo_count += len(merged)
# check every matching CC
pre_add_size = len(self.bbox_active)
next_match_idx = 0
for bbox_idx, bbox in enumerate(current_bboxes):
found = False
# check all matches in the list of matches for current CC
while next_match_idx < len(
merged) and merged[next_match_idx][0] == bbox_idx:
if not found:
prev_idx = merged[next_match_idx][1]
prev_bbox = self.unique_bbox_objects[prev_idx]
bbox_IOU = self.get_bboxes_IOU(bbox, prev_bbox)
# print(bbox_IOU)
if bbox_IOU >= self.min_temporal_IOU:
# assume they are equivalent
found = True
self.unique_bbox_frames[prev_idx].append(
(self.img_idx, bbox_idx))
current_bboxes_idxs.append((prev_idx, bbox))
# update last frame seen for this cc...
self.bbox_last_frame[prev_idx] = self.img_idx
next_match_idx += 1
# Not match was found?
if not found:
# add
self.unique_bbox_objects.append(bbox)
self.unique_bbox_frames.append([(self.img_idx, bbox_idx)])
new_bbox_idx = len(self.unique_bbox_objects) - 1
current_bboxes_idxs.append((new_bbox_idx, bbox))
self.bbox_last_frame.append(self.img_idx)
self.bbox_active.append(new_bbox_idx)
# add to indices ...
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.add(x1, x2, new_bbox_idx)
self.bbox_int_index_y.add(y1, y2, new_bbox_idx)
# remove CC that are no longer active
pre_remove_size = len(self.bbox_active)
tempo_pos = 0
while tempo_pos < len(self.bbox_active):
bbox_idx = self.bbox_active[tempo_pos]
if self.img_idx - self.bbox_last_frame[
bbox_idx] >= self.max_gap:
# no longer active ..
# delete from active list
del self.bbox_active[tempo_pos]
# delete from interval indices
bbox = self.unique_bbox_objects[bbox_idx]
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.remove(x1, x2, bbox_idx)
self.bbox_int_index_y.remove(y1, y2, bbox_idx)
#print self.cc_last_frame[cc_idx],
else:
# still active
tempo_pos += 1
"""
total_added = pre_remove_size - pre_add_size
total_removed = pre_remove_size - len(self.bbox_active)
msg = "{0:d} , (Added: {1:d}, Removed: {2:d})".format(len(self.bbox_active), total_added, total_removed)
print(msg)
"""
self.bbox_idx_per_frame.append(current_bboxes_idxs)
self.img_idx += 1
if self.verbose:
msg = "[{0:d} ({1:d}, {2:d})]".format(
self.img_idx, len(current_bboxes),
len(self.unique_bbox_objects))
print(msg, end="\r")
def finish_processing(self):
if self.verbose:
print(".")
def split_stable_bboxes_by_gaps(self, max_gap, stable_min_frames):
splitted_count = 0
n_original_objects = len(self.unique_bbox_objects)
for bbox_idx in range(n_original_objects):
current_frames = self.unique_bbox_frames[bbox_idx]
n_local = len(current_frames)
current_group = [current_frames[0]]
valid_groups = [current_group]
for frame_offset in range(1, n_local):
curr_frame_idx = current_frames[frame_offset][0]
prev_frame_idx = current_frames[frame_offset - 1][0]
current_gap = curr_frame_idx - prev_frame_idx
if current_gap > max_gap:
# not acceptable gap ..
current_group = [current_frames[frame_offset]]
valid_groups.append(current_group)
else:
# acceptable
current_group.append(current_frames[frame_offset])
if len(valid_groups) >= 2 and n_local >= stable_min_frames:
# replace the frames in the original cc list to only the first group ..
self.unique_bbox_frames[bbox_idx] = valid_groups[0]
# for each group (new CC)
for group_offset in range(1, len(valid_groups)):
new_bbox_idx = len(self.unique_bbox_objects)
# add another reference to the original CC
self.unique_bbox_objects.append(
self.unique_bbox_objects[bbox_idx])
# add CC frame reference ...
self.unique_bbox_frames.append(valid_groups[group_offset])
# for each frame where the orginal appeared ..
for frame_idx, local_bbox_idx in valid_groups[
group_offset]:
# find the CC on the frame ...
for offset, (global_bbox_idx, local_bbox) in enumerate(
self.bbox_idx_per_frame[frame_idx]):
if global_bbox_idx == bbox_idx:
# replace
self.bbox_idx_per_frame[frame_idx][offset] = (
new_bbox_idx, local_bbox)
break
splitted_count += 1
return splitted_count
def get_stable_bbox_idxs(self, min_stable_frames):
stable_idxs = []
for bbox_idx in range(len(self.unique_bbox_objects)):
if len(self.unique_bbox_frames[bbox_idx]) >= min_stable_frames:
stable_idxs.append(bbox_idx)
return stable_idxs
def compute_overlapping_stable_bboxes(self, stable_idxs, temporal_window):
n_objects = len(self.unique_bbox_objects)
n_stable = len(stable_idxs)
all_overlapping_cc = [[] for x in range(n_objects)]
time_overlapping_cc = [[] for x in range(n_objects)]
total_intersections = 0
for offset1 in range(n_stable):
# get first stable
bbox_idx_1 = stable_idxs[offset1]
bbox_1 = self.unique_bbox_objects[bbox_idx_1]
bbox_1_t_start = self.unique_bbox_frames[bbox_idx_1][0][0]
bbox_1_t_end = self.unique_bbox_frames[bbox_idx_1][-1][0]
bbox_1_area = self.get_bbox_area(bbox_1)
print("Processing: " + str(offset1), end="\r")
for offset2 in range(offset1 + 1, n_stable):
bbox_idx_2 = stable_idxs[offset2]
bbox_2 = self.unique_bbox_objects[bbox_idx_2]
bbox_2_t_start = self.unique_bbox_frames[bbox_idx_2][0][0]
bbox_2_t_end = self.unique_bbox_frames[bbox_idx_2][-1][0]
# check intersection in space
IOU = self.get_bboxes_IOU(bbox_1, bbox_2)
if IOU > 0.0001:
bbox_2_area = self.get_bbox_area(bbox_2)
inter_box_area = self.get_bbox_area(
self.get_inter_bbox(bbox_1, bbox_2))
all_overlapping_cc[bbox_idx_1].append(
(bbox_idx_2, inter_box_area, bbox_2_area, bbox_1_area))
all_overlapping_cc[bbox_idx_2].append(
(bbox_idx_1, inter_box_area, bbox_1_area, bbox_2_area))
# now, check intersection in time (considering time window)
if ((bbox_1_t_end + temporal_window >= bbox_2_t_start) and
(bbox_2_t_end >= bbox_1_t_start - temporal_window)):
ratio_1 = inter_box_area / bbox_1_area
ratio_2 = inter_box_area / bbox_2_area
# they have some intersection in pixels....
time_overlapping_cc[bbox_idx_1].append(
(bbox_idx_2, ratio_1, ratio_2))
time_overlapping_cc[bbox_idx_2].append(
(bbox_idx_1, ratio_2, ratio_1))
total_intersections += 1
return time_overlapping_cc, total_intersections, all_overlapping_cc
def compute_groups(self, stable_idxs, overlapping_bboxes):
n_stable = len(stable_idxs)
# Determine Groups of CC's that coexist in time and space and treat them as single units...
bboxes_groups = []
group_idx_per_bbox = {}
for offset1 in range(n_stable):
bbox_idx_1 = stable_idxs[offset1]
if bbox_idx_1 in group_idx_per_bbox:
# use the existing group
group_idx = group_idx_per_bbox[bbox_idx_1]
else:
# create new group
group_idx = len(bboxes_groups)
bboxes_groups.append([bbox_idx_1])
group_idx_per_bbox[bbox_idx_1] = group_idx
# for every CC that occupies the same space ..
for bbox_idx_2, _, _ in overlapping_bboxes[bbox_idx_1]:
# if it is not in the current group, add it ...
if bbox_idx_2 not in group_idx_per_bbox:
# add to current group
group_idx_per_bbox[bbox_idx_2] = group_idx
# add to the current group
bboxes_groups[group_idx].append(bbox_idx_2)
else:
other_group_idx = group_idx_per_bbox[bbox_idx_2]
if other_group_idx != group_idx:
# different group? merge ..
#print("Merging groups")
# for each element in the other group
for other_idx_cc in bboxes_groups[other_group_idx]:
# link to the current group
group_idx_per_bbox[other_idx_cc] = group_idx
# add to the current group
bboxes_groups[group_idx].append(other_idx_cc)
# leave the other group empty
bboxes_groups[other_group_idx] = []
# clean up empty groups
final_bbox_groups = []
final_group_idx_per_bbox = {}
for group in bboxes_groups:
if len(group) > 0:
new_group_idx = len(final_bbox_groups)
final_bbox_groups.append(group)
for box_idx in group:
final_group_idx_per_bbox[box_idx] = new_group_idx
return final_bbox_groups, final_group_idx_per_bbox
def compute_groups_temporal_information(self, bbox_groups):
# compute temporal information for each group
n_frames = len(self.bbox_idx_per_frame)
group_ages = {}
groups_per_frame = [[] for frame_idx in range(n_frames)]
for group_idx, group in enumerate(bbox_groups):
if len(group) == 0:
continue
current_ages = []
for cc_idx in group:
g_first = self.unique_bbox_frames[cc_idx][0][0]
g_last = self.unique_bbox_frames[cc_idx][-1][0]
# add first
if g_first not in current_ages:
current_ages.append(g_first)
if g_last not in current_ages:
current_ages.append(g_last)
current_ages = sorted(current_ages)
group_ages[group_idx] = current_ages
for frame_idx in range(current_ages[0],
min(current_ages[-1] + 1, n_frames)):
groups_per_frame[frame_idx].append(group_idx)
return group_ages, groups_per_frame
def compute_conflicting_groups(self, stable_idxs, all_overlapping_bboxes,
n_groups, group_idx_per_bbox):
n_stable = len(stable_idxs)
# for each stable CC
conflicts = {group_idx: {} for group_idx in range(n_groups)}
for offset1 in range(n_stable):
bbox_idx_1 = stable_idxs[offset1]
# for every CC that occupies the same space (same group or not)..
for bbox_idx_2, matched_pixels, size_bbox_2, size_bbox_1 in all_overlapping_bboxes[
bbox_idx_1]:
# consider each link only once
if bbox_idx_1 < bbox_idx_2:
# UNION - Intersection
unmatched_pixels = size_bbox_1 + size_bbox_2 - matched_pixels * 2
# check if they are on different groups
group_idx1 = group_idx_per_bbox[bbox_idx_1]
group_idx2 = group_idx_per_bbox[bbox_idx_2]
if group_idx1 != group_idx2:
# conflict found, add the total of matched pixels in the conflict
if group_idx2 in conflicts[group_idx1]:
conflicts[group_idx1][group_idx2][
"matched"] += matched_pixels
conflicts[group_idx1][group_idx2][
"unmatched"] += unmatched_pixels
else:
conflicts[group_idx1][group_idx2] = {
"matched": matched_pixels,
"unmatched": unmatched_pixels
}
if group_idx1 in conflicts[group_idx2]:
conflicts[group_idx2][group_idx1][
"matched"] += matched_pixels
conflicts[group_idx2][group_idx1][
"unmatched"] += unmatched_pixels
else:
conflicts[group_idx2][group_idx1] = {
"matched": matched_pixels,
"unmatched": unmatched_pixels
}
return conflicts
def find_container_bbox(self, bboxes):
g_x1, g_y1, g_x2, g_y2 = bboxes[0]
for o_x1, o_y1, o_x2, o_y2 in bboxes[1:]:
g_x1 = min(g_x1, o_x1)
g_y1 = min(g_y1, o_y1)
g_x2 = max(g_x2, o_x2)
g_y2 = max(g_y2, o_y2)
return g_x1, g_y1, g_x2, g_y2
def refine_bboxes(self, bboxes_groups, group_ages, temporal_refinement):
groups_bboxes = {}
refined_bboxes = {}
refined_per_group = {}
debug_group_ids = []
for group_idx, group in enumerate(bboxes_groups):
if len(group) == 0:
continue
# initial box ...
bbox_idx = group[0]
g_x1, g_y1, g_x2, g_y2 = self.unique_bbox_objects[bbox_idx]
# expand to contain all images from all boxes for all frames in the current group
for bbox_idx in group:
# stable bbox level ..
frame_idx, local_bbox_idx = self.unique_bbox_frames[bbox_idx][
0]
_, (u_x1, u_y1, u_x2,
u_y2) = self.bbox_idx_per_frame[frame_idx][local_bbox_idx]
# expand the boxes to get the bigger box that contains all the per-frame variations
for frame_idx, local_bbox_idx in self.unique_bbox_frames[
bbox_idx]:
global_bbox_idx, local_bbox = self.bbox_idx_per_frame[
frame_idx][local_bbox_idx]
o_x1, o_y1, o_x2, o_y2 = local_bbox
# stable bbox level ...
u_x1 = min(u_x1, o_x1)
u_y1 = min(u_y1, o_y1)
u_x2 = max(u_x2, o_x2)
u_y2 = max(u_y2, o_y2)
# group level ...
g_x1 = min(g_x1, o_x1)
g_y1 = min(g_y1, o_y1)
g_x2 = max(g_x2, o_x2)
g_y2 = max(g_y2, o_y2)
# print((group_idx, bbox_idx, (u_x1, u_x2, u_y1, u_y2), (g_x1, g_x2, g_y1, g_y2)))
refined_bboxes[bbox_idx] = (u_x1, u_y1, u_x2, u_y2)
groups_bboxes[group_idx] = (g_x1, g_y1, g_x2, g_y2)
# compute the refined boxes for different temporal segments ..
current_images = []
current_ages = group_ages[group_idx]
if group_idx in debug_group_ids:
print((group_idx, groups_bboxes[group_idx],
group_ages[group_idx]))
# ... for each temporal segment ...
for t_segment in range(len(current_ages) - 1):
t_start = current_ages[t_segment]
t_end = current_ages[t_segment + 1]
if temporal_refinement == BBoxStabilityEstimator.RefineAllBoxesAllSegments:
# use larger group on all segments ...
current_images.append([groups_bboxes[group_idx]])
else:
# ... for each box in the group ...
visible_bboxes = []
for bbox_idx in group:
# ... check
bbox_frames = [(f_idx, local_bbox_idx)
for f_idx, local_bbox_idx in
self.unique_bbox_frames[bbox_idx]
if t_start <= f_idx <= t_end]
count_bbox_frames = len(bbox_frames)
# only if this box has visible frames in current time
if count_bbox_frames > 0:
visible_bboxes.append(refined_bboxes[bbox_idx])
# merge visible boxes that have any overlap
visible_bboxes = self.spatial_box_grouping(
visible_bboxes, 0.0)
if temporal_refinement == BBoxStabilityEstimator.RefinePerBoxPerSegment:
# use all the merged boxes separately
current_images.append(visible_bboxes)
else:
# find container for all boxes ...
current_images.append(
[self.find_container_bbox(visible_bboxes)])
refined_per_group[group_idx] = current_images
return refined_bboxes, groups_bboxes, refined_per_group
def refined_per_frame(self,
bbox_groups,
groups_per_frame,
group_ages,
refined_per_group,
save_prefix=None,
stable_min_frames=3):
group_next_segment = [0 for group_idx in range(len(bbox_groups))]
all_reconstructed = []
for img_idx, groups_in_frame in enumerate(groups_per_frame):
reconstructed = []
for group_idx in groups_in_frame:
# find corresponding image ....
current_ages = group_ages[group_idx]
while current_ages[group_next_segment[group_idx] +
1] < img_idx:
# move to the next segment
group_next_segment[group_idx] += 1
# use image of selected segment ...
segment_bboxes = refined_per_group[group_idx][
group_next_segment[group_idx]]
# add to the image
reconstructed += segment_bboxes
if not save_prefix is None:
# draw the reconstructed/refined boxes in white ...
img_boxes = self.visualize_boxes(reconstructed,
(255, 255, 255))
# add the original stable/unstable boxes
img_stable = np.zeros((self.height, self.width, 3), np.uint8)
img_unstable = np.zeros((self.height, self.width, 3), np.uint8)
for bbox_idx, local_bbox in self.bbox_idx_per_frame[img_idx]:
x1, y1, x2, y2 = local_bbox
if len(self.unique_bbox_frames[bbox_idx]
) < stable_min_frames:
# add unstable
# reconstructed[y1:y2 + 1, x1:x2 + 1, 2] = 255
cv2.rectangle(img_unstable, (x1, y1), (x2, y2),
(255, 255, 255),
thickness=2)
else:
# add stable
cv2.rectangle(img_stable, (x1, y1), (x2, y2),
(255, 255, 255),
thickness=2)
# add stable in green channel ...
img_boxes[:, :, 1] = np.bitwise_or(img_stable[:, :, 0],
img_boxes[:, :, 1])
# add unstable in red channel
img_boxes[:, :, 2] = np.bitwise_or(img_unstable[:, :, 0],
img_boxes[:, :, 2])
cv2.imwrite(save_prefix + str(img_idx) + ".png", img_boxes)
all_reconstructed.append(reconstructed)
return all_reconstructed
def compute_group_images(self, bboxes_groups, group_ages, binary_frames,
sum_threshold, use_global):
group_images = {}
group_boundaries = {}
debug_group_ids = []
# TODO: pending, check if grouping by ages could work ... maybe from previous process, avoid creating
# very close ages ... group these maybe????
for group_idx, group in enumerate(bboxes_groups):
if len(group) == 0:
continue
# initial box ...
bbox_idx = group[0]
g_x1, g_y1, g_x2, g_y2 = self.unique_bbox_objects[bbox_idx]
# expand to contain all images from all boxes for all frames in the current group
for bbox_idx in group:
for frame_idx, local_bbox_idx in self.unique_bbox_frames[
bbox_idx]:
global_bbox_idx, local_bbox = self.bbox_idx_per_frame[
frame_idx][local_bbox_idx]
o_x1, o_y1, o_x2, o_y2 = local_bbox
g_x1 = min(g_x1, o_x1)
g_y1 = min(g_y1, o_y1)
g_x2 = max(g_x2, o_x2)
g_y2 = max(g_y2, o_y2)
group_boundaries[group_idx] = (g_x1, g_x2, g_y1, g_y2)
# size ...
g_width = g_x2 - g_x1 + 1
g_height = g_y2 - g_y1 + 1
# compute the actual images using temporal information ..
current_images = []
current_ages = group_ages[group_idx]
g_complete_mask = np.zeros((g_height, g_width), dtype=np.int32)
g_complete_sum = np.zeros((g_height, g_width), dtype=np.float64)
if group_idx in debug_group_ids:
print((group_idx, group_boundaries[group_idx],
group_ages[group_idx]))
for t_segment in range(len(current_ages) - 1):
t_start = current_ages[t_segment]
t_end = current_ages[t_segment + 1]
g_mask = np.zeros((g_height, g_width), dtype=np.int32)
g_sum = np.zeros((g_height, g_width), dtype=np.float64)
# get the sum of CCs for all frames where they appear ...
count_visible = 0
for bbox_idx in group:
# x1, y1, x2, y2 = self.unique_bbox_objects[bbox_idx]
# cc_first = self.unique_cc_frames[cc_idx][0][0]
# cc_last = self.unique_cc_frames[cc_idx][-1][0]
bbox_frames = [(f_idx, local_bbox_idx)
for f_idx, local_bbox_idx in
self.unique_bbox_frames[bbox_idx]
if t_start <= f_idx < t_end]
count_bbox_frames = len(bbox_frames)
# only if this box has visible frames in current time
if count_bbox_frames > 0:
count_visible += 1
# for each frame ...
for f_idx, local_bbox_idx in bbox_frames:
# find the box for this frame ...
_, (
x1, y1, x2, y2
) = self.bbox_idx_per_frame[f_idx][local_bbox_idx]
offset_x = x1 - g_x1
offset_y = y1 - g_y1
b_w = x2 - x1 + 1
b_h = y2 - y1 + 1
# bbox cut ....
# ... image per pixel ....
mask_cut = g_sum[offset_y:offset_y + b_h,
offset_x:offset_x + b_w]
global_mask_cut = g_complete_sum[
offset_y:offset_y + b_h,
offset_x:offset_x + b_w]
# print((x1, y1, x2, y2, b_w, b_h, offset_x, offset_y, g_x1, g_y1))
bin_cut = (
binary_frames[f_idx][y1:y2 + 1, x1:x2 + 1] //
255)
mask_cut += bin_cut
# global image ...
global_mask_cut += bin_cut
# ... counter per pixel ...
mask_cut = g_mask[offset_y:offset_y + b_h,
offset_x:offset_x + b_w]
# all bounding-box pixels add as many times as frames where same box appears
mask_cut += 1
# repeat for global image ...
mask_cut = g_complete_mask[offset_y:offset_y + b_h,
offset_x:offset_x + b_w]
mask_cut += 1
# if local temporal segments are requested ...
if not use_global:
visible_pixels = g_mask > 0
g_sum[visible_pixels] /= g_mask[visible_pixels]
g_sum[g_sum < sum_threshold] = 0.0
g_sum[g_sum >= sum_threshold] = 1.0
g_sum *= 255
segment_img = g_sum.astype(np.uint8)
if group_idx in debug_group_ids:
cv2.imshow(
"Segment IMG #{0:d} ({1:d}-{2:d})".format(
group_idx, t_start, t_end), segment_img)
cv2.waitKey()
# raise Exception("STOP!")
current_images.append(segment_img)
# TODO: test just adding all pixels from region ...
g_sum = np.zeros((g_height, g_width), dtype=np.float64)
for f_idx in range(t_start, t_end + 1):
pass
# if global temporal segments are requested ...
if use_global:
visible_pixels = g_complete_mask > 0
g_complete_sum[visible_pixels] /= g_complete_mask[
visible_pixels]
g_complete_sum[g_complete_sum < sum_threshold] = 0.0
g_complete_sum[g_complete_sum >= sum_threshold] = 1.0
g_complete_sum *= 255
segment_img = g_complete_sum.astype(np.uint8)
if group_idx in debug_group_ids or True:
magnified_segment = cv2.resize(
segment_img,
(segment_img.shape[1] * 3, segment_img.shape[0] * 3))
cv2.imshow("complete img: #{0:d}".format(group_idx),
magnified_segment)
cv2.waitKey()
# raise Exception("STOP!")
current_images = [segment_img] * (len(current_ages) - 1)
group_images[group_idx] = current_images
return group_images, group_boundaries
def frames_from_groups(self,
cc_groups,
group_boundaries,
groups_per_frame,
group_ages,
group_images,
save_prefix=None,
stable_min_frames=3,
show_unstable=True):
group_next_segment = [0 for group_idx in range(len(cc_groups))]
clean_reconstructed = []
for img_idx, groups_in_frame in enumerate(groups_per_frame):
reconstructed = np.zeros((self.height, self.width, 3),
dtype=np.int32)
for group_idx in groups_in_frame:
# find corresponding image ....
current_ages = group_ages[group_idx]
while current_ages[group_next_segment[group_idx] +
1] < img_idx:
# move to the next segment
group_next_segment[group_idx] += 1
# use image of selected segment ...
segment_img = group_images[group_idx][
group_next_segment[group_idx]]
# add to the image
g_min_x, g_max_x, g_min_y, g_max_y = group_boundaries[
group_idx]
reconstructed[g_min_y:g_max_y + 1, g_min_x:g_max_x + 1,
0] += segment_img
reconstructed[g_min_y:g_max_y + 1, g_min_x:g_max_x + 1,
1] += segment_img
reconstructed[g_min_y:g_max_y + 1, g_min_x:g_max_x + 1,
2] += segment_img
reconstructed[reconstructed > 255] = 255
reconstructed = reconstructed.astype(np.uint8)
if show_unstable:
img_stable = np.zeros((self.height, self.width, 3), np.uint8)
img_unstable = np.zeros((self.height, self.width, 3), np.uint8)
for bbox_idx, local_bbox in self.bbox_idx_per_frame[img_idx]:
x1, y1, x2, y2 = local_bbox
if len(self.unique_bbox_frames[bbox_idx]
) < stable_min_frames:
# add unstable
# reconstructed[y1:y2 + 1, x1:x2 + 1, 2] = 255
cv2.rectangle(img_unstable, (x1, y1), (x2, y2),
(255, 255, 255),
thickness=2)
else:
# add stable
cv2.rectangle(img_stable, (x1, y1), (x2, y2),
(255, 255, 255),
thickness=2)
# add stable in green channel ...
reconstructed[:, :, 1] = np.bitwise_or(img_stable[:, :, 0],
reconstructed[:, :, 1])
# add unstable in red channel
reconstructed[:, :, 2] = np.bitwise_or(img_unstable[:, :, 0],
reconstructed[:, :, 2])
#print(group_next_segment)
if not save_prefix is None:
cv2.imwrite(save_prefix + str(img_idx) + ".png", reconstructed)
flag, raw_data = cv2.imencode(".png", reconstructed[:, :, 0])
clean_reconstructed.append(raw_data)
return clean_reconstructed
def add_frame(self, frame_bboxes):
current_bboxes = self.spatial_box_grouping(frame_bboxes,
self.min_combined_ratio)
current_bboxes_idxs = []
if self.img_idx == 0:
# simply copy all
for bbox_id, bbox in enumerate(current_bboxes):
# add the box to list of unique boxes ...
self.unique_bbox_objects.append(bbox)
# frames on which the bbox appears, raw label assigned to that CC
self.unique_bbox_frames.append([(0, bbox_id)])
bbox_idx = len(self.unique_bbox_objects) - 1
current_bboxes_idxs.append((bbox_idx, bbox))
self.bbox_last_frame.append(0)
self.bbox_active.append(bbox_idx)
# add to indices ...
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.add(x1, x2, bbox_idx)
self.bbox_int_index_y.add(y1, y2, bbox_idx)
else:
# create indices for current bboxes
other_index_x = IntervalIndex(True)
other_index_y = IntervalIndex(True)
for bbox_idx, (x1, y1, x2, y2) in enumerate(current_bboxes):
other_index_x.add(x1, x2, bbox_idx)
other_index_y.add(y1, y2, bbox_idx)
# compute CC with matching regions
set_x = set(other_index_x.find_matches(self.bbox_int_index_x))
set_y = set(other_index_y.find_matches(self.bbox_int_index_y))
# list of all pairs of CC with intersecting intervals in X and Y
merged = sorted(list(set_x.intersection(set_y)))
self.tempo_count += len(merged)
# check every matching CC
pre_add_size = len(self.bbox_active)
next_match_idx = 0
for bbox_idx, bbox in enumerate(current_bboxes):
found = False
# check all matches in the list of matches for current CC
while next_match_idx < len(
merged) and merged[next_match_idx][0] == bbox_idx:
if not found:
prev_idx = merged[next_match_idx][1]
prev_bbox = self.unique_bbox_objects[prev_idx]
bbox_IOU = self.get_bboxes_IOU(bbox, prev_bbox)
# print(bbox_IOU)
if bbox_IOU >= self.min_temporal_IOU:
# assume they are equivalent
found = True
self.unique_bbox_frames[prev_idx].append(
(self.img_idx, bbox_idx))
current_bboxes_idxs.append((prev_idx, bbox))
# update last frame seen for this cc...
self.bbox_last_frame[prev_idx] = self.img_idx
next_match_idx += 1
# Not match was found?
if not found:
# add
self.unique_bbox_objects.append(bbox)
self.unique_bbox_frames.append([(self.img_idx, bbox_idx)])
new_bbox_idx = len(self.unique_bbox_objects) - 1
current_bboxes_idxs.append((new_bbox_idx, bbox))
self.bbox_last_frame.append(self.img_idx)
self.bbox_active.append(new_bbox_idx)
# add to indices ...
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.add(x1, x2, new_bbox_idx)
self.bbox_int_index_y.add(y1, y2, new_bbox_idx)
# remove CC that are no longer active
pre_remove_size = len(self.bbox_active)
tempo_pos = 0
while tempo_pos < len(self.bbox_active):
bbox_idx = self.bbox_active[tempo_pos]
if self.img_idx - self.bbox_last_frame[
bbox_idx] >= self.max_gap:
# no longer active ..
# delete from active list
del self.bbox_active[tempo_pos]
# delete from interval indices
bbox = self.unique_bbox_objects[bbox_idx]
x1, y1, x2, y2 = bbox
self.bbox_int_index_x.remove(x1, x2, bbox_idx)
self.bbox_int_index_y.remove(y1, y2, bbox_idx)
#print self.cc_last_frame[cc_idx],
else:
# still active
tempo_pos += 1
"""
total_added = pre_remove_size - pre_add_size
total_removed = pre_remove_size - len(self.bbox_active)
msg = "{0:d} , (Added: {1:d}, Removed: {2:d})".format(len(self.bbox_active), total_added, total_removed)
print(msg)
"""
self.bbox_idx_per_frame.append(current_bboxes_idxs)
self.img_idx += 1
if self.verbose:
msg = "[{0:d} ({1:d}, {2:d})]".format(
self.img_idx, len(current_bboxes),
len(self.unique_bbox_objects))
print(msg, end="\r")
class CCStabilityEstimator:
def __init__(self, width, height, min_recall, min_precision, max_gap, verbose=False):
self.width = width
self.height = height
self.min_recall = min_recall
self.min_precision = min_precision
self.max_gap = max_gap
self.unique_cc_objects = []
self.unique_cc_frames = []
self.cc_idx_per_frame = []
self.cc_int_index_x = IntervalIndex(True)
self.cc_int_index_y = IntervalIndex(True)
self.fake_age = np.zeros((height, width), dtype=np.float32)
self.img_idx = 0
self.tempo_count = 0
# optimizing the matching ...
self.cc_last_frame = []
self.cc_active = []
self.verbose = verbose
def get_raw_cc_count(self):
total = 0
for current_frame in self.cc_idx_per_frame:
total += len(current_frame)
return total
def add_frame(self, img, input_binary=False):
# get the CC
if input_binary:
# use given binary
binary = img
else:
# binarize
binary = Binarizer.backgroundSubtractionBinarization(img.astype('uint8'))
current_cc = Labeler.extractSpatioTemporalContent(binary, self.fake_age)
current_cc_idxs = []
if self.img_idx == 0:
# simply copy all
for cc in current_cc:
self.unique_cc_objects.append(cc) # CC objet
# frames on which the CC appears, raw label assigend to that CC
self.unique_cc_frames.append([(0, cc.cc_id + 1)])
cc_idx = len(self.unique_cc_objects) - 1
current_cc_idxs.append((cc_idx, cc))
self.cc_last_frame.append(0)
self.cc_active.append(cc_idx)
# add to indices ...
self.cc_int_index_x.add(cc.min_x, cc.max_x + 1, cc_idx)
self.cc_int_index_y.add(cc.min_y, cc.max_y + 1, cc_idx)
else:
# create indices for current CC
other_index_x = IntervalIndex(True)
other_index_y = IntervalIndex(True)
for cc_idx, cc in enumerate(current_cc):
other_index_x.add(cc.min_x, cc.max_x + 1, cc_idx)
other_index_y.add(cc.min_y, cc.max_y + 1, cc_idx)
# compute CC with matching regions
set_x = set(other_index_x.find_matches(self.cc_int_index_x))
set_y = set(other_index_y.find_matches(self.cc_int_index_y))
# list of all pairs of CC with intersecting intervals in X and Y
merged = sorted(list(set_x.intersection(set_y)))
self.tempo_count += len(merged)
# check every matching CC
pre_add_size = len(self.cc_active)
next_match_idx = 0
for cc_idx, cc in enumerate(current_cc):
found = False
# check all matches in the list of matches for current CC
while next_match_idx < len(merged) and merged[next_match_idx][0] == cc_idx:
if not found:
prev_idx = merged[next_match_idx][1]
prev_cc = self.unique_cc_objects[prev_idx]
recall, precision = cc.getOverlapFMeasure(prev_cc, False, False)
if recall >= self.min_recall and precision >= self.min_precision:
# assume they are equivalent
found = True
self.unique_cc_frames[prev_idx].append((self.img_idx, cc.cc_id + 1))
current_cc_idxs.append((prev_idx, cc))
# update last frame seen for this cc...
self.cc_last_frame[prev_idx] = self.img_idx
next_match_idx += 1
# Not match was found?
if not found:
# add
self.unique_cc_objects.append(cc)
self.unique_cc_frames.append([(self.img_idx, cc.cc_id + 1)])
new_cc_idx = len(self.unique_cc_objects) - 1
current_cc_idxs.append((new_cc_idx, cc))
self.cc_last_frame.append(self.img_idx)
self.cc_active.append(new_cc_idx)
# add to indices ...
self.cc_int_index_x.add(cc.min_x, cc.max_x + 1, new_cc_idx)
self.cc_int_index_y.add(cc.min_y, cc.max_y + 1, new_cc_idx)
# remove CC that are no longer active
pre_remove_size = len(self.cc_active)
tempo_pos = 0
while tempo_pos < len(self.cc_active):
cc_idx = self.cc_active[tempo_pos]
if self.img_idx - self.cc_last_frame[cc_idx] >= self.max_gap:
# no longer active ..
# delete from active list
del self.cc_active[tempo_pos]
# delete from interval indices
cc = self.unique_cc_objects[cc_idx]
self.cc_int_index_x.remove(cc.min_x, cc.max_x + 1, cc_idx)
self.cc_int_index_y.remove(cc.min_y, cc.max_y + 1, cc_idx)
#print self.cc_last_frame[cc_idx],
else:
# still active
tempo_pos += 1
#print(str(len(self.cc_active)) + ", (Added: " + str(pre_remove_size - pre_add_size) + ", Removed: " + str(pre_remove_size - len(self.cc_active)) + ")")
self.cc_idx_per_frame.append(current_cc_idxs)
self.img_idx += 1
if self.verbose:
print("[" + str(self.img_idx) + " (" + str(len(current_cc)) + ", " + str(len(self.unique_cc_objects)) + ")]", end="\r")
def finish_processing(self):
if self.verbose:
print(".")
print("Erase this final count (after done): " + str(self.tempo_count))
self.fake_age = None
def rebuilt_binary_images(self):
rebuilt_frames = []
for frame_ccs in self.cc_idx_per_frame:
binary = np.zeros((self.height, self.width), dtype=np.uint8)
for idx_cc, local_cc in frame_ccs:
binary[local_cc.min_y:local_cc.max_y +1, local_cc.min_x:local_cc.max_x + 1] += local_cc.img
rebuilt_frames.append(binary)
return rebuilt_frames
def split_stable_cc_by_gaps(self, max_gap, stable_min_frames):
splitted_count = 0
n_original_objects = len(self.unique_cc_objects)
for idx_cc in range(n_original_objects):
current_frames = self.unique_cc_frames[idx_cc]
n_local = len(current_frames)
current_group = [current_frames[0]]
valid_groups = [current_group]
for frame_offset in range(1, n_local):
curr_frame_idx = current_frames[frame_offset][0]
prev_frame_idx = current_frames[frame_offset - 1][0]
current_gap = curr_frame_idx - prev_frame_idx
if current_gap > max_gap:
# not acceptable gap ..
current_group = [current_frames[frame_offset]]
valid_groups.append(current_group)
else:
# acceptable
current_group.append(current_frames[frame_offset])
if len(valid_groups) >= 2 and n_local >= stable_min_frames:
# replace the frames in the original cc list to only the first group ..
self.unique_cc_frames[idx_cc] = valid_groups[0]
# for each group (new CC)
for group_offset in range(1, len(valid_groups)):
new_idx_cc = len(self.unique_cc_objects)
# add another reference to the original CC
self.unique_cc_objects.append(self.unique_cc_objects[idx_cc])
# add CC frame reference ...
self.unique_cc_frames.append(valid_groups[group_offset])
# for each frame where the orgiianl appeared ..
for frame_idx, local_cc in valid_groups[group_offset]:
# find the CC on the frame ...
for offset, (local_cc_idx, local_cc) in enumerate(self.cc_idx_per_frame[frame_idx]):
if local_cc_idx == idx_cc:
# replace
self.cc_idx_per_frame[frame_idx][offset] = (new_idx_cc, local_cc)
break
splitted_count += 1
return splitted_count
def get_stable_cc_idxs(self, min_stable_frames):
stable_idxs = []
for idx_cc in range(len(self.unique_cc_objects)):
if len(self.unique_cc_frames[idx_cc]) >= min_stable_frames:
stable_idxs.append(idx_cc)
return stable_idxs
def get_temporal_index(self):
temporal_index = []
for idxs_per_frame in self.cc_idx_per_frame:
temporal_index.append([cc_idx for cc_idx, local_cc in idxs_per_frame])
return temporal_index
def compute_overlapping_stable_cc(self, stable_idxs, temporal_window):
n_objects = len(self.unique_cc_objects)
n_stable = len(stable_idxs)
all_overlapping_cc = [[] for x in range(n_objects)]
time_overlapping_cc = [[] for x in range(n_objects)]
total_intersections = 0
for offset1 in range(n_stable):
# get first stable
idx_cc1 = stable_idxs[offset1]
cc1 = self.unique_cc_objects[idx_cc1]
cc1_t_start = self.unique_cc_frames[idx_cc1][0][0]
cc1_t_end = self.unique_cc_frames[idx_cc1][-1][0]
print("Processing: " + str(offset1),end="\r")
for offset2 in range(offset1 + 1, n_stable):
idx_cc2 = stable_idxs[offset2]
cc2 = self.unique_cc_objects[idx_cc2]
cc2_t_start = self.unique_cc_frames[idx_cc2][0][0]
cc2_t_end = self.unique_cc_frames[idx_cc2][-1][0]
# check intersection in space
recall, precision = cc1.getOverlapFMeasure(cc2, False, False)
if recall > 0.0 or precision > 0.0:
matched_pixels = int(cc1.size * recall)
all_overlapping_cc[idx_cc1].append((idx_cc2, matched_pixels, cc2.size, cc1.size))
all_overlapping_cc[idx_cc2].append((idx_cc1, matched_pixels, cc1.size, cc2.size))
# now, check intersection in time (considering time window)
if cc1_t_end + temporal_window >= cc2_t_start and cc2_t_end >= cc1_t_start - temporal_window:
# they have some intersection in pixels....
time_overlapping_cc[idx_cc1].append((idx_cc2, recall, precision))
time_overlapping_cc[idx_cc2].append((idx_cc1, precision, recall))
total_intersections += 1
return time_overlapping_cc, total_intersections, all_overlapping_cc
def compute_groups(self, stable_idxs, overlapping_cc):
n_stable = len(stable_idxs)
# Determine Groups of CC's that coexist in time and space and treat them as single units...
cc_groups = []
group_idx_per_cc = {}
for offset1 in range(n_stable):
idx_cc1 = stable_idxs[offset1]
if idx_cc1 in group_idx_per_cc:
# use the existing group
group_idx = group_idx_per_cc[idx_cc1]
else:
# create new group
group_idx = len(cc_groups)
cc_groups.append([idx_cc1])
group_idx_per_cc[idx_cc1] = group_idx
# for every CC that occupies the same space ..
for idx_cc2, recall, precision in overlapping_cc[idx_cc1]:
# if it is not in the current group, add it ...
if idx_cc2 not in group_idx_per_cc:
# add to current group
group_idx_per_cc[idx_cc2] = group_idx
# add to the current group
cc_groups[group_idx].append(idx_cc2)
else:
other_group_idx = group_idx_per_cc[idx_cc2]
if other_group_idx != group_idx:
# different group? merge ..
#print("Merging groups")
# for each element in the other group
for other_idx_cc in cc_groups[other_group_idx]:
# link to the current group
group_idx_per_cc[other_idx_cc] = group_idx
# add to the current group
cc_groups[group_idx].append(other_idx_cc)
# leave the other group empty
cc_groups[other_group_idx] = []
# clean up empty groups
final_cc_groups = []
final_group_idx_per_cc = {}
for group in cc_groups:
if len(group) > 0:
new_group_idx = len(final_cc_groups)
final_cc_groups.append(group)
for idx_cc in group:
final_group_idx_per_cc[idx_cc] = new_group_idx
return final_cc_groups, final_group_idx_per_cc
def compute_groups_temporal_information(self, cc_groups):
# compute temporal information for each group
n_frames = len(self.cc_idx_per_frame)
group_ages = {}
groups_per_frame = [[] for frame_idx in range(n_frames)]
for group_idx, group in enumerate(cc_groups):
if len(group) == 0:
continue
current_ages = []
for cc_idx in group:
g_first = self.unique_cc_frames[cc_idx][0][0]
g_last = self.unique_cc_frames[cc_idx][-1][0]
# add first
if g_first not in current_ages:
current_ages.append(g_first)
if g_last not in current_ages:
current_ages.append(g_last)
current_ages = sorted(current_ages)
group_ages[group_idx] = current_ages
for frame_idx in range(current_ages[0], min(current_ages[-1] + 1, n_frames)):
groups_per_frame[frame_idx].append(group_idx)
return group_ages, groups_per_frame
def compute_conflicting_groups(self, stable_idxs, all_overlapping_cc, n_groups, group_idx_per_cc):
n_stable = len(stable_idxs)
# for each stable CC
conflicts = {group_idx:{} for group_idx in range(n_groups)}
for offset1 in range(n_stable):
idx_cc1 = stable_idxs[offset1]
# for every CC that occupies the same space (same group or not)..
for idx_cc2, matched_pixels, size_cc2, size_cc1 in all_overlapping_cc[idx_cc1]:
# consider each link only once
if idx_cc1 < idx_cc2:
unmatched_pixels = size_cc1 + size_cc2 - matched_pixels * 2
# check if they are on different groups
group_idx1 = group_idx_per_cc[idx_cc1]
group_idx2 = group_idx_per_cc[idx_cc2]
if group_idx1 != group_idx2:
# conflict found, add the total of matched pixels in the conflict
if group_idx2 in conflicts[group_idx1]:
conflicts[group_idx1][group_idx2]["matched"] += matched_pixels
conflicts[group_idx1][group_idx2]["unmatched"] += unmatched_pixels
else:
conflicts[group_idx1][group_idx2] = {
"matched": matched_pixels,
"unmatched": unmatched_pixels
}
if group_idx1 in conflicts[group_idx2]:
conflicts[group_idx2][group_idx1]["matched"] += matched_pixels
conflicts[group_idx2][group_idx1]["unmatched"] += unmatched_pixels
else:
conflicts[group_idx2][group_idx1] = {
"matched": matched_pixels,
"unmatched": unmatched_pixels
}
return conflicts
def compute_group_images_from_raw_binary(self, cc_groups, group_ages, binary_frames, segment_threshold):
group_images = {}
group_boundaries = {}
for group_idx, group in enumerate(cc_groups):
if len(group) == 0:
continue
# first, compute the combined boundaries...
cc_idx = group[0]
g_min_x = self.unique_cc_objects[cc_idx].min_x
g_max_x = self.unique_cc_objects[cc_idx].max_x
g_min_y = self.unique_cc_objects[cc_idx].min_y
g_max_y = self.unique_cc_objects[cc_idx].max_y
for cc_idx in group:
g_min_x = min(g_min_x, self.unique_cc_objects[cc_idx].min_x)
g_max_x = max(g_max_x, self.unique_cc_objects[cc_idx].max_x)
g_min_y = min(g_min_y, self.unique_cc_objects[cc_idx].min_y)
g_max_y = max(g_max_y, self.unique_cc_objects[cc_idx].max_y)
group_boundaries[group_idx] = (g_min_x, g_max_x, g_min_y, g_max_y)
# size ...
g_width = g_max_x - g_min_x + 1
g_height = g_max_y - g_min_y + 1
# compute the actual images using temporal information ..
current_images = []
current_ages = group_ages[group_idx]
for t_segment in range(len(current_ages) - 1):
t_start = current_ages[t_segment]
t_end = current_ages[t_segment + 1]
g_mask = np.zeros((g_height, g_width), dtype=np.int32)
for cc_idx in group:
cc = self.unique_cc_objects[cc_idx]
cc_first = self.unique_cc_frames[cc_idx][0][0]
cc_last = self.unique_cc_frames[cc_idx][-1][0]
# check if cc existed in current time segment ...
if cc_first <= t_end and t_start <= cc_last:
# add to current image
offset_x = cc.min_x - g_min_x
offset_y = cc.min_y - g_min_y
g_mask[offset_y:offset_y + cc.getHeight(), offset_x:offset_x + cc.getWidth()] += (cc.img // 255)
# now get the mask ...
g_mask = (g_mask > 0).astype('uint8') * 255
# use the mask to obtain the image of the group for the current segment of time
segment_img = np.zeros((g_height, g_width), dtype=np.int32)
for frame_idx in range(t_start, t_end + 1):
local_patch = np.bitwise_and(binary_frames[frame_idx][g_min_y:g_max_y + 1, g_min_x:g_max_x + 1], g_mask) // 255
segment_img += local_patch
segment_img = (segment_img * 255) // segment_img.max()
#segment_img[segment_img < min_pixel_val] = 0
segment_img = (segment_img > segment_threshold).astype(dtype=np.uint8) * 255
#g_img = (g_img * 255) / g_img.max()
#cv2.imwrite("output/images/fs_a_group_" + str(group_idx) + "_" + str(t_start) + ".png", segment_img)
current_images.append(segment_img)
group_images[group_idx] = current_images
return group_images, group_boundaries
def compute_group_images(self, cc_groups, group_ages, segment_threshold):
group_images = {}
group_boundaries = {}
for group_idx, group in enumerate(cc_groups):
if len(group) == 0:
continue
# first, compute the combined boundaries...
cc_idx = group[0]
g_min_x = self.unique_cc_objects[cc_idx].min_x
g_max_x = self.unique_cc_objects[cc_idx].max_x
g_min_y = self.unique_cc_objects[cc_idx].min_y
g_max_y = self.unique_cc_objects[cc_idx].max_y
for cc_idx in group:
g_min_x = min(g_min_x, self.unique_cc_objects[cc_idx].min_x)
g_max_x = max(g_max_x, self.unique_cc_objects[cc_idx].max_x)
g_min_y = min(g_min_y, self.unique_cc_objects[cc_idx].min_y)
g_max_y = max(g_max_y, self.unique_cc_objects[cc_idx].max_y)
group_boundaries[group_idx] = (g_min_x, g_max_x, g_min_y, g_max_y)
# size ...
g_width = g_max_x - g_min_x + 1
g_height = g_max_y - g_min_y + 1
# compute the actual images using temporal information ..
current_images = []
current_ages = group_ages[group_idx]
for t_segment in range(len(current_ages) - 1):
t_start = current_ages[t_segment]
t_end = current_ages[t_segment + 1]
g_mask = np.zeros((g_height, g_width), dtype=np.int32)
# get the sum of CCs for all frames where they appear ...
for cc_idx in group:
cc = self.unique_cc_objects[cc_idx]
# cc_first = self.unique_cc_frames[cc_idx][0][0]
# cc_last = self.unique_cc_frames[cc_idx][-1][0]
cc_frames = len([f_idx for f_idx, _ in self.unique_cc_frames[cc_idx] if t_start <= f_idx <= t_end])
if cc_frames > 0:
offset_x = cc.min_x - g_min_x
offset_y = cc.min_y - g_min_y
mask_cut = g_mask[offset_y:offset_y + cc.getHeight(), offset_x:offset_x + cc.getWidth()]
# all CC pixels addes as many times as frames the entire CC appears
mask_cut += (cc.img // 255) * cc_frames
segment_img = ((g_mask.astype(np.float64) / g_mask.max()) >= segment_threshold).astype(np.uint8) * 255
current_images.append(segment_img)
group_images[group_idx] = current_images
return group_images, group_boundaries
def frames_from_groups(self, cc_groups, group_boundaries, groups_per_frame, group_ages, group_images,
save_prefix=None, stable_min_frames=3, show_unstable=True):
group_next_segment = [0 for group_idx in range(len(cc_groups))]
clean_binary = []
for img_idx, groups_in_frame in enumerate(groups_per_frame):
reconstructed = np.zeros((self.height, self.width, 3), dtype=np.uint8)
for group_idx in groups_in_frame:
# find corresponding image ....
current_ages = group_ages[group_idx]
while current_ages[group_next_segment[group_idx] + 1] < img_idx:
# move to the next segment
group_next_segment[group_idx] += 1
# use image of selected segment ...
segment_img = group_images[group_idx][group_next_segment[group_idx]]
# add to the image
g_min_x, g_max_x, g_min_y, g_max_y = group_boundaries[group_idx]
reconstructed[g_min_y:g_max_y + 1, g_min_x:g_max_x + 1, 0] += segment_img
reconstructed[g_min_y:g_max_y + 1, g_min_x:g_max_x + 1, 1] += segment_img
if not show_unstable:
reconstructed[g_min_y:g_max_y + 1, g_min_x:g_max_x + 1, 2] += segment_img
if show_unstable:
for cc_idx, local_cc in self.cc_idx_per_frame[img_idx]:
if len(self.unique_cc_frames[cc_idx]) < stable_min_frames:
# add unstable in the red channel
cc = local_cc
reconstructed[cc.min_y:cc.max_y +1, cc.min_x:cc.max_x + 1, 2] += cc.img
#print(group_next_segment)
if not save_prefix is None:
cv2.imwrite(save_prefix + "_stab_" + str(img_idx) + ".png", reconstructed)
cv2.imwrite(save_prefix + "_clean_" + str(img_idx) + ".png", reconstructed[:,:, 0])
flag, raw_data = cv2.imencode(".png", reconstructed[:,:, 0])
clean_binary.append(raw_data)
return clean_binary
@staticmethod
def find_block_stable_cc(block_images, min_f_score, min_stable_bg = 0.50, verbose=False):
n_points, height, width = block_images.shape
estimator = CCStabilityEstimator(width, height, min_f_score, min_stable_bg, verbose)
for img_idx in range(n_points):
img = block_images[img_idx, :, :]
estimator.add_frame(img)
return estimator.unique_cc_objects, estimator.unique_cc_frames, estimator.cc_idx_per_frame, estimator.stable_background_mask
@staticmethod
def compute_overlapping_CC_groups(cc_objects):
n_objects = len(cc_objects)
all_overlapping_cc = [[x] for x in range(n_objects)]
# compute all pairwise overlaps
for idx1 in range(n_objects):
# get first stable
cc1 = cc_objects[idx1]
for idx2 in range(idx1 + 1, n_objects):
cc2 = cc_objects[idx2]
# check intersection in space
recall, precision = cc1.getOverlapFMeasure(cc2, False, False)
if recall > 0.0 or precision > 0.0:
all_overlapping_cc[idx1].append(idx2)
all_overlapping_cc[idx2].append(idx1)
# find groups containing pairs of overlapping objects (transitive overlap)
group_overlap_idx = [x for x in range(n_objects)]
merged_groups = {x: {x} for x in range(n_objects)}
for idx in range(n_objects):
# check the group of the current object
merged_idx1 = group_overlap_idx[idx]
current_group = all_overlapping_cc[idx]
for other_idx in current_group[1:]:
# check the group of an overlapping obejct
merged_idx2 = group_overlap_idx[other_idx]
# if it is different, then merge!
if merged_idx1 != merged_idx2:
# 1 )create a single larger group
merged_groups[merged_idx1] = merged_groups[merged_idx1].union(merged_groups[merged_idx2])
# 2 ) Redirect each element on the group that will disappear, to the newer larger group
for old_group_idx in merged_groups[merged_idx2]:
group_overlap_idx[old_group_idx] = merged_idx1
# 3) Delete old unreferenced group
del merged_groups[merged_idx2]
# finally, split by overlapping and no overlapping ..
overlapping_groups = []
no_overlaps = []
for group_idx in merged_groups:
merged_list = list(merged_groups[group_idx])
if len(merged_list) == 1:
no_overlaps.append(merged_list[0])
else:
overlapping_groups.append(merged_list)
return overlapping_groups, no_overlaps
def spatial_box_grouping(self, frame_bboxes, min_combined_ratio=None):
if min_combined_ratio is None:
min_combined_ratio = self.min_combined_ratio
current_boxes = list(frame_bboxes)
merged_boxes = True
while merged_boxes:
merged_boxes = False
# sort bounding boxes by descending size ...
boxes_by_size = []
for x1, y1, x2, y2 in current_boxes:
w = x2 - x1 + 1
h = y2 - y1 + 1
area = w * h
boxes_by_size.append((area, (x1, y1, x2, y2)))
boxes_by_size = sorted(boxes_by_size,
reverse=True,
key=lambda x: x[0])
# print(boxes_by_size)
# create interval index to find matches quicker (all boxes against all boxes from same frame)
int_index_x = IntervalIndex(True)
int_index_y = IntervalIndex(True)
for box_idx, (area, (x1, y1, x2, y2)) in enumerate(boxes_by_size):
int_index_x.add(x1, x2 + 1, box_idx)
int_index_y.add(y1, y2 + 1, box_idx)
# ... find pair-wise matches ...
set_x = set(int_index_x.find_matches(int_index_x))
set_y = set(int_index_y.find_matches(int_index_y))
# .... list of all pairs of boxes with intersecting intervals in X and Y
merge_candidates = sorted(list(set_x.intersection(set_y)))
# ... filter self-matches and repetitions ...
merge_candidates = [(box_idx1, box_idx2)
for box_idx1, box_idx2 in merge_candidates
if box_idx1 < box_idx2]
# ... split by first box ..
candidates_by_box = {idx: [] for idx in range(len(boxes_by_size))}
for box_idx1, box_idx2 in merge_candidates:
candidates_by_box[box_idx1].append(box_idx2)
# print(merge_candidates)
# print(candidates_by_box)
box_added = [False] * len(boxes_by_size)
current_boxes = []
# for each box (sorted by size)
for box_idx in range(len(boxes_by_size)):
# if this box has been previously added (merged with ealier box)
if box_added[box_idx]:
# skip ...
continue
box_added[box_idx] = True
# current box boundaries
c_area, (c_x1, c_y1, c_x2, c_y2) = boxes_by_size[box_idx]
for second_box_idx in candidates_by_box[box_idx]:
if box_added[second_box_idx]:
# skip merge candidate ...
continue
# get box boundaries ...
o_area, (o_x1, o_y1, o_x2,
o_y2) = boxes_by_size[second_box_idx]
comb_area_ratio = self.get_combined_box_area_ratio(
(c_x1, c_y1, c_x2, c_y2), (o_x1, o_y1, o_x2, o_y2))
# print(((c_x1, c_y1, c_x2, c_y2), boxes_by_size[second_box_idx], comb_area_ratio, box_idx, second_box_idx))
if comb_area_ratio >= min_combined_ratio:
# merge!
# expand current bounding box to include the smaller box ...
c_x1 = min(c_x1, o_x1)
c_y1 = min(c_y1, o_y1)
c_x2 = max(c_x2, o_x2)
c_y2 = max(c_y2, o_y2)
# mark second box as added, so it won't be added to the current list ..
box_added[second_box_idx] = True
merged_boxes = True
# add to the next set of accepted boxes
current_boxes.append((c_x1, c_y1, c_x2, c_y2))
"""
if len(frame_bboxes) > 30:
original_img = self.visualize_boxes(frame_bboxes, (255, 0, 0))
final_img = self.visualize_boxes(current_boxes, (0, 255, 0))
final_img[:, :, 0] = original_img[:, :, 0]
debug_img = cv2.resize(final_img,(960, 540))
cv2.imshow("check", debug_img)
cv2.waitKey()
raise Exception("Error!")
"""
return current_boxes
def add_frame(self, img, input_binary=False):
# get the CC
if input_binary:
# use given binary
binary = img
else:
# binarize
binary = Binarizer.backgroundSubtractionBinarization(img.astype('uint8'))
current_cc = Labeler.extractSpatioTemporalContent(binary, self.fake_age)
current_cc_idxs = []
if self.img_idx == 0:
# simply copy all
for cc in current_cc:
self.unique_cc_objects.append(cc) # CC objet
# frames on which the CC appears, raw label assigend to that CC
self.unique_cc_frames.append([(0, cc.cc_id + 1)])
cc_idx = len(self.unique_cc_objects) - 1
current_cc_idxs.append((cc_idx, cc))
self.cc_last_frame.append(0)
self.cc_active.append(cc_idx)
# add to indices ...
self.cc_int_index_x.add(cc.min_x, cc.max_x + 1, cc_idx)
self.cc_int_index_y.add(cc.min_y, cc.max_y + 1, cc_idx)
else:
# create indices for current CC
other_index_x = IntervalIndex(True)
other_index_y = IntervalIndex(True)
for cc_idx, cc in enumerate(current_cc):
other_index_x.add(cc.min_x, cc.max_x + 1, cc_idx)
other_index_y.add(cc.min_y, cc.max_y + 1, cc_idx)
# compute CC with matching regions
set_x = set(other_index_x.find_matches(self.cc_int_index_x))
set_y = set(other_index_y.find_matches(self.cc_int_index_y))
# list of all pairs of CC with intersecting intervals in X and Y
merged = sorted(list(set_x.intersection(set_y)))
self.tempo_count += len(merged)
# check every matching CC
pre_add_size = len(self.cc_active)
next_match_idx = 0
for cc_idx, cc in enumerate(current_cc):
found = False
# check all matches in the list of matches for current CC
while next_match_idx < len(merged) and merged[next_match_idx][0] == cc_idx:
if not found:
prev_idx = merged[next_match_idx][1]
prev_cc = self.unique_cc_objects[prev_idx]
recall, precision = cc.getOverlapFMeasure(prev_cc, False, False)
if recall >= self.min_recall and precision >= self.min_precision:
# assume they are equivalent
found = True
self.unique_cc_frames[prev_idx].append((self.img_idx, cc.cc_id + 1))
current_cc_idxs.append((prev_idx, cc))
# update last frame seen for this cc...
self.cc_last_frame[prev_idx] = self.img_idx
next_match_idx += 1
# Not match was found?
if not found:
# add
self.unique_cc_objects.append(cc)
self.unique_cc_frames.append([(self.img_idx, cc.cc_id + 1)])
new_cc_idx = len(self.unique_cc_objects) - 1
current_cc_idxs.append((new_cc_idx, cc))
self.cc_last_frame.append(self.img_idx)
self.cc_active.append(new_cc_idx)
# add to indices ...
self.cc_int_index_x.add(cc.min_x, cc.max_x + 1, new_cc_idx)
self.cc_int_index_y.add(cc.min_y, cc.max_y + 1, new_cc_idx)
# remove CC that are no longer active
pre_remove_size = len(self.cc_active)
tempo_pos = 0
while tempo_pos < len(self.cc_active):
cc_idx = self.cc_active[tempo_pos]
if self.img_idx - self.cc_last_frame[cc_idx] >= self.max_gap:
# no longer active ..
# delete from active list
del self.cc_active[tempo_pos]
# delete from interval indices
cc = self.unique_cc_objects[cc_idx]
self.cc_int_index_x.remove(cc.min_x, cc.max_x + 1, cc_idx)
self.cc_int_index_y.remove(cc.min_y, cc.max_y + 1, cc_idx)
#print self.cc_last_frame[cc_idx],
else:
# still active
tempo_pos += 1
#print(str(len(self.cc_active)) + ", (Added: " + str(pre_remove_size - pre_add_size) + ", Removed: " + str(pre_remove_size - len(self.cc_active)) + ")")
self.cc_idx_per_frame.append(current_cc_idxs)
self.img_idx += 1
if self.verbose:
print("[" + str(self.img_idx) + " (" + str(len(current_cc)) + ", " + str(len(self.unique_cc_objects)) + ")]", end="\r")