def visualize_map_fn(intrvl):
assert isinstance(intrvl, FrameGroupInterval)
assert tkey in intrvl.payload, f"{tkey} not found in {str(intrvl)}"
pts = np.array(list(map(centroid, intrvl.payload[tkey])))
assert pts.shape[1] == 2
# this is tricky: adjust from relative coord in original frame to pixel coord in the crop
pts = (pts - [intrvl['x1'], intrvl['y1']]) / [
_width(intrvl), _height(intrvl)
] * [intrvl.frames[0].shape[1], intrvl.frames[0].shape[0]]
pts = pts.astype(np.int32)
color = (0, 255, 0)
thickness = 2
is_closed = False
new_frames = []
for fid, frame in enumerate(intrvl.frames):
frame = frame.copy()
f1 = cv2.putText(frame, f"visualize-{fid}", (50, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1,
cv2.LINE_AA)
# somehow polylines doesn't work
# f1 = cv2.polylines(f1, pts, is_closed, color, thickness)
for j, (p1, p2) in enumerate(zip(pts[:-1], pts[1:])):
f1 = cv2.line(f1, tuple(p1), tuple(p2), color, thickness)
f1 = cv2.putText(f1, str(j), tuple(p1),
cv2.FONT_HERSHEY_SIMPLEX, 0.2,
(0, 0, 255), 1, cv2.LINE_AA)
new_frames.append(f1)
intrvl.frames = new_frames
return intrvl
def dilate_car(icar: Interval) -> Interval:
carh, carw = _height(icar), _width(icar)
new_bounds = Bounds3D(t1=int(max(0, icar['t1'] - fps)),
t2=int(min(frame_count, icar['t2'] + fps)),
x1=max(0, icar['x1'] - carw),
x2=min(1, icar['x2'] + carw),
y1=max(0, icar['y1'] - carh),
y2=min(1, icar['y2'] + carh))
return Interval(new_bounds)
def dilate_op(i:ImageInterval) -> ImageInterval:
w, h = _width(i), _height(i)
new_bounds = Bounds3D(
i['t1'], i['t2'],
x1=max(0, i['x1']-w*3),
x2=min(1, i['x2']+w*3),
y1=max(0, i['y1']-h/2),
y2=min(1, i['y2']+h/2)
)
return ImageInterval(new_bounds, None, root=i.root)
def merge_candidate(iperson: Interval, icar: Interval) -> Interval:
carh, carw = _height(icar), _width(icar)
new_bounds = Bounds3D(t1=int(max(0, icar['t1'] - fps * 3)),
t2=int(max(0, iperson['t1'] + fps * 3)),
x1=max(0, icar['x1'] - carw),
x2=min(1, icar['x2'] + carw),
y1=max(0, icar['y1'] - carh),
y2=min(1, icar['y2'] + carh))
new_payload = {
'traj_person': iperson.payload['traj_person'],
'stopped_car': icar
}
return Interval(new_bounds, new_payload)
def new_fn_backward(i1: Interval) -> Interval:
tracker = cv2.TrackerCSRT_create()
ret_bounds = i1.bounds
ret_payload = {
trajectory_key: [
VideoFrameInterval(i1.bounds, root_decoder=decoder),
]
}
# buffer all frames in window at once
ts_range = list(range(i1['t1'], max(-1, i1['t1'] - window),
-step)) # reverse order
start_fid = min(ts_range) # inclusive
end_fid = max(ts_range) + 1 # exclusive
frames_to_track = decoder.get_frame_interval(
start_fid, end_fid, step)[::-1] # reverse tracking order
# init tracker. For tracking, we must get whole frames
H, W = frames_to_track[0].shape[:2]
# tracking box in cv2 is the form (x, y, w, h)
init_box = np.array(
[i1['x1'] * W, i1['y1'] * H,
_width(i1) * W,
_height(i1) * H]).astype(np.int32)
tracker.init(frames_to_track[0], tuple(init_box))
# iterate remaining frames and update tracker, get tracked result
for ts, next_frame in zip(ts_range[1:], frames_to_track[1:]):
# tracking backward
(success, next_box) = tracker.update(next_frame)
if success:
x, y, w, h = next_box # pixel coord
x1, y1, x2, y2 = x, y, x + w, y + h
x1, y1, x2, y2 = x1 / W, y1 / H, x2 / W, y2 / H # relative coord
next_bounds = Bounds3D(ts, ts + 1, x1, x2, y1, y2)
ret_bounds = ret_bounds.span(next_bounds)
ret_payload[trajectory_key].insert(
0, VideoFrameInterval(next_bounds, root_decoder=decoder))
else:
break
return Interval(ret_bounds, ret_payload)
def visualize_map_fn(fg: FrameGroupInterval):
assert isinstance(fg, FrameGroupInterval)
assert tkey in fg.payload, f"{tkey} not found in {str(fg)}"
assert fg['t2'] - fg['t1'] == len(fg.frames)
# 1. Draw all visualization on a frame with black background
global_vis_frame = np.zeros_like(fg.frames[0])
trajectory = fg.payload[tkey]
Xmins = list(map(itemgetter('x1'), trajectory))
Xmaxs = list(map(itemgetter('x2'), trajectory))
Ymins = list(map(itemgetter('y1'), trajectory))
Ymaxs = list(map(itemgetter('y2'), trajectory))
Xmins, Xmaxs, Ymins, Ymaxs = list(map(np.array, [Xmins, Xmaxs, Ymins, Ymaxs]))
# this is tricky: adjust from relative coord in original frame to pixel coord in the crop
H, W = fg.frames[0].shape[:2]
Xmins = W * (Xmins - fg['x1']) / _width(fg)
Xmaxs = W * (Xmaxs - fg['x1']) / _width(fg)
Ymins = H * (Ymins - fg['y1']) / _height(fg)
Ymaxs = H * (Ymaxs - fg['y1']) / _height(fg)
Xmins = Xmins.astype(np.int)
Xmaxs = Xmaxs.astype(np.int)
Ymins = Ymins.astype(np.int)
Ymaxs = Ymaxs.astype(np.int)
centroids = np.stack([0.5 * (Xmins + Xmaxs), 0.5 * (Ymins + Ymaxs)], axis=1)
centroids = centroids.astype(np.int)
assert centroids.shape == (len(trajectory), 2)
# somehow polylines doesn't work
# f1 = cv2.polylines(f1, pts, is_closed, color, thickness)
color = (0, 255, 0)
thickness = 2
is_closed = False
for j, (p1, p2) in enumerate(zip(centroids[:-1], centroids[1:])):
if self.draw_line:
global_vis_frame = cv2.line(global_vis_frame, tuple(p1), tuple(p2), color, thickness)
if self.draw_label:
global_vis_frame = cv2.putText(global_vis_frame, str(j), tuple(p1), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1, cv2.LINE_AA)
if self.draw_box:
boxs_to_draw = list(zip([i['t1'] for i in trajectory], Xmins, Ymins, Xmaxs, Ymaxs))
# 2. Overlay it on all frames
new_frames = []
vis_ts_box = None
for j, frame in enumerate(fg.frames):
frame = frame.copy()
ts = fg['t1'] + j
frame = cv2.putText(frame, f"F {ts}", (50,50), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1, cv2.LINE_AA)
assert global_vis_frame.shape == frame.shape, f"{j} {global_vis_frame.shape} {frame.shape}"
frame[global_vis_frame > 0] = global_vis_frame[global_vis_frame > 0]
if self.draw_box:
for b in boxs_to_draw:
if ts == b[0]:
vis_ts_box = b
if vis_ts_box:
_, left, top, right, bottom = vis_ts_box
frame = cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 3)
new_frames.append(frame)
fg.frames = new_frames
return fg
