def det_process(opts, frame_recv, det_res_send, w_img, h_img):
try:
model = init_detector(opts)
# warm up the GPU
_ = inference_detector(model, np.zeros((h_img, w_img, 3), np.uint8))
torch.cuda.synchronize()
while 1:
fidx = frame_recv.recv()
if type(fidx) is list:
# new video, read all images in advance
frame_list = fidx
frames = [imread(img_path) for img_path in frame_list]
# signal ready, no errors
det_res_send.send('ready')
continue
elif fidx is None:
# exit flag
break
fidx, t1 = fidx
img = frames[fidx]
t2 = perf_counter()
t_send_frame = t2 - t1
result = inference_detector(model, img, gpu_pre=not opts.cpu_pre)
torch.cuda.synchronize()
t3 = perf_counter()
det_res_send.send([result, t_send_frame, t3])
except Exception:
# report all errors from the child process to the parent
# forward traceback info as well
det_res_send.send(Exception("".join(traceback.format_exception(*sys.exc_info()))))
def main():
opts = parse_args()
mkdir2(opts.out_dir)
print('Reading detection images')
det_img_files = sorted(glob(join(opts.det_img_dir, '*.jpg')))
det_imgs = []
for path in det_img_files:
det_imgs.append(imread(path))
print('Detection Test at Scale 0.5')
rt_det1 = det_test(opts, 0.5, det_imgs)
print(f'Mean runtime (ms): {1e3*rt_det1:.3g}')
print('Detection Test at Scale 1')
rt_det2 = det_test(opts, 1, det_imgs)
print(f'Mean runtime (ms): {1e3*rt_det2:.3g}')
det_imgs = None
print('Classification Test')
rt_inf, rt_train = classify_test(opts)
print(f'Inference Mean runtime (ms): {1e3*rt_inf:.3g}')
print(f'Training Mean runtime (ms): {1e3*rt_train:.3g}')
out_path = join(opts.out_dir, 'platform_bench.txt')
if opts.overwrite or not isfile(out_path):
with open(out_path, 'w', newline='\n') as f:
writer = csv.writer(f)
writer.writerow([rt_det1, rt_det2, rt_inf, rt_train])
print(f'Saved results to "{out_path}"')
def main():
opts = parse_args()
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
seq = 'f1008c18-e76e-3c24-adcc-da9858fac145'
sid = seqs.index(seq)
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb'))
results_parsed = results['results_parsed']
img = frame_list[153]
result = results_parsed[15]
bboxes, scores, labels, masks = result[:4]
idx = [16]
bboxes = bboxes[idx]
scores = scores[idx]
labels = labels[idx]
masks = masks[idx]
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path).copy()
vis_path = join(opts.vis_dir, 'teaser.jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(
I,
bboxes,
labels,
class_names,
masks,
None,
out_scale=opts.vis_scale,
out_file=vis_path,
)
def main():
opts = parse_args()
out_dir = mkdir2(opts.out_dir) if opts.out_dir else opts.result_dir
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
results_ccf = []
in_time = 0
miss = 0
mismatch = 0
print('Pairing the output with the ground truth')
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb'))
# use raw results when possible in case we change class subset during evaluation
if opts.use_parsed:
results_parsed = results['results_parsed']
else:
results_raw = results.get('results_raw', None)
if results_raw is None:
results_parsed = results['results_parsed']
timestamps = results['timestamps']
input_fidx = results['input_fidx']
tidx_p1 = 0
for ii, img in enumerate(frame_list):
# pred, gt association by time
t = (ii - opts.eta) / opts.fps
while tidx_p1 < len(timestamps) and timestamps[tidx_p1] <= t:
tidx_p1 += 1
if tidx_p1 == 0:
# no output
miss += 1
bboxes, scores, labels = [], [], []
masks, tracks = None, None
else:
tidx = tidx_p1 - 1
ifidx = input_fidx[tidx]
in_time += int(ii == ifidx)
mismatch += ii - ifidx
if opts.use_parsed or results_raw is None:
result = results_parsed[tidx]
bboxes, scores, labels, masks = result[:4]
if len(result) > 4:
tracks = result[4]
else:
tracks = None
else:
result = results_raw[tidx]
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
tracks = None
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
if tracks is None:
vis_det(
I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path,
)
else:
vis_track(
I,
bboxes,
tracks,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path,
)
# convert to coco fmt
n = len(bboxes)
if n:
bboxes_ltwh = ltrb2ltwh(bboxes)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes_ltwh[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
out_path = join(out_dir, 'eval_assoc.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'miss': miss,
'in_time': in_time,
'mismatch': mismatch,
}, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
class_names = [c['name'] for c in cats]
last_sid = None
aidx = 0
for img in tqdm(imgs):
img_name = img['name']
iid = img['id']
sid = img['sid']
if sid != last_sid:
in_dir_seq = join(data_root, seq_dirs[sid])
out_dir_seq = join(out_dir, seqs[sid])
last_sid = sid
I = imread(join(in_dir_seq, img['name']))
bboxes = []
masks = []
labels = []
if show_track:
tracks = []
while aidx < len(annots):
ann = annots[aidx]
if ann['image_id'] != iid:
break
bboxes.append(ann['bbox'])
if 'segmentation' in ann:
masks.append(ann['segmentation'])
else:
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
results_raw = [] # image based, all 80 COCO classes
results_ccf = [] # instance based
for iid, img in tqdm(db.imgs.items()):
img_name = img['name']
sid = img['sid']
seq_name = seqs[sid]
img_path = join(opts.data_root, seq_dirs[sid], img_name)
I = imread(img_path)
result = inference_detector(model, I, gpu_pre=not opts.cpu_pre)
results_raw.append(result)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
if vis_out:
vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path)
# convert to coco fmt
n = len(bboxes)
if n:
ltrb2ltwh_(bboxes)
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'results_raw.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_raw, open(out_path, 'wb'))
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main():
assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing
opts = parse_args()
assert (not opts.mask_timing) or (opts.mask_timing and not opts.no_mask)
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
# warm up the GPU
img = db.imgs[0]
w_img, h_img = img['width'], img['height']
_ = inference_detector(model, np.zeros((h_img, w_img, 3), np.uint8))
torch.cuda.synchronize()
runtime_all = []
n_processed = 0
n_total = 0
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
n_frame = len(frames)
n_total += n_frame
timestamps = []
results_raw = []
results_parsed = []
input_fidx = []
runtime = []
last_fidx = None
if not opts.dynamic_schedule:
stride_cnt = 0
t_total = n_frame/opts.fps
t_start = perf_counter()
while 1:
t1 = perf_counter()
t_elapsed = t1 - t_start
if t_elapsed >= t_total:
break
# identify latest available frame
fidx_continous = t_elapsed*opts.fps
fidx = int(np.floor(fidx_continous))
if fidx == last_fidx:
continue
last_fidx = fidx
if opts.dynamic_schedule:
fidx_remainder = fidx_continous - fidx
if fidx_remainder > 0.5:
continue
else:
if stride_cnt % opts.det_stride == 0:
stride_cnt = 1
else:
stride_cnt += 1
continue
frame = frames[fidx]
result = inference_detector(model, frame, gpu_pre=not opts.cpu_pre, decode_mask=not opts.mask_timing)
if opts.mask_timing:
mask_encoded = result[2]
result = result[:2]
bboxes, scores, labels, _, sel = \
parse_det_result(result, coco_mapping, n_class, return_sel=True)
else:
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
torch.cuda.synchronize()
t2 = perf_counter()
t_elapsed = t2 - t_start
if t_elapsed >= t_total:
break
if opts.mask_timing:
masks = decode_mask(mask_encoded, sel)
timestamps.append(t_elapsed)
results_raw.append(result)
results_parsed.append((bboxes, scores, labels, masks))
input_fidx.append(fidx)
runtime.append(t2 - t1)
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'results_raw': results_raw,
'results_parsed': results_parsed,
'timestamps': timestamps,
'input_fidx': input_fidx,
'runtime': runtime,
}, open(out_path, 'wb'))
runtime_all += runtime
n_processed += len(results_raw)
runtime_all_np = np.asarray(runtime_all)
n_small_runtime = (runtime_all_np < 1.0/opts.fps).sum()
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'runtime_all': runtime_all,
'n_processed': n_processed,
'n_total': n_total,
'n_small_runtime': n_small_runtime,
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3*x
print(f'{n_processed}/{n_total} frames processed')
print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms)
print(f'Runtime smaller than unit time interval: '
f'{n_small_runtime}/{n_processed} '
f'({100.0*n_small_runtime/n_processed:.4g}%)')
def main():
opts = parse_args()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
if opts.cached_res:
cache_in_ccf = '_ccf' in basename(opts.cached_res)
if cache_in_ccf:
# speed up based on the assumption of sequential storage
cache_end_idx = 0
cached_res = pickle.load(open(opts.cached_res, 'rb'))
else:
assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing
model = init_detector(opts)
np.random.seed(opts.seed)
runtime = pickle.load(open(opts.runtime, 'rb'))
runtime_dist = dist_from_dict(runtime, opts.perf_factor)
runtime_all = []
n_processed = 0
n_total = 0
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
n_frame = len(frame_list)
n_total += n_frame
if not opts.cached_res:
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
timestamps = []
results_parsed = []
input_fidx = []
runtime = []
last_fidx = None
if opts.cached_res and cache_in_ccf:
results_raw = None
else:
results_raw = []
t_total = n_frame/opts.fps
t_elapsed = 0
if opts.dynamic_schedule:
mean_rtf = runtime_dist.mean()*opts.fps
else:
stride_cnt = 0
while 1:
if t_elapsed >= t_total:
break
# identify latest available frame
fidx_continous = t_elapsed*opts.fps
fidx = int(np.floor(fidx_continous))
if fidx == last_fidx:
# algorithm is fast and has some idle time
fidx += 1
if fidx == n_frame:
break
t_elapsed = fidx/opts.fps
last_fidx = fidx
if opts.dynamic_schedule:
if mean_rtf > 1:
# when runtime <= 1, it should always process every frame
fidx_remainder = fidx_continous - fidx
if mean_rtf < np.floor(fidx_remainder + mean_rtf):
# wait till next frame
continue
else:
if stride_cnt % opts.det_stride == 0:
stride_cnt = 1
else:
stride_cnt += 1
continue
if opts.cached_res:
img = frame_list[fidx]
if cache_in_ccf:
cache_end_idx, bboxes, scores, labels, masks = \
result_from_ccf(cached_res, img['id'], cache_end_idx)
ltwh2ltrb_(bboxes)
else:
result = cached_res[img['id']]
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
else:
frame = frames[fidx]
result = inference_detector(model, frame)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
rt_this = runtime_dist.draw()
t_elapsed += rt_this
if t_elapsed >= t_total:
break
timestamps.append(t_elapsed)
if results_raw is not None:
results_raw.append(result)
results_parsed.append((bboxes, scores, labels, masks))
input_fidx.append(fidx)
runtime.append(rt_this)
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
out_dict = {
'results_parsed': results_parsed,
'timestamps': timestamps,
'input_fidx': input_fidx,
'runtime': runtime,
}
if results_raw is not None:
out_dict['results_raw'] = results_raw
pickle.dump(out_dict, open(out_path, 'wb'))
runtime_all += runtime
n_processed += len(results_parsed)
runtime_all_np = np.array(runtime_all)
n_small_runtime = (runtime_all_np < 1.0/opts.fps).sum()
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'runtime_all': runtime_all,
'n_processed': n_processed,
'n_total': n_total,
'n_small_runtime': n_small_runtime,
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3*x
print(f'{n_processed}/{n_total} frames processed')
print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms)
print(f'Runtime smaller than unit time interval: '
f'{n_small_runtime}/{n_processed} '
f'({100.0*n_small_runtime/n_processed:.4g}%)')
def vis_obj_fancy(
img,
bboxes,
labels,
class_names,
color_palette,
color_scheme='class',
masks=None,
scores=None,
score_th=0.3,
tracks=None,
iscrowd=None,
out_scale=1,
out_file=None,
show_label=True,
show_score=True,
font=None,
):
thickness = 2
alpha = 0.2
if isinstance(img, str):
img = imread(img)
img = np.array(img) # create a writable copy
if out_scale != 1:
img = cv2.resize(
img,
fx=out_scale,
fy=out_scale,
interpolation=cv2.INTER_LINEAR,
)
bboxes = np.asarray(bboxes)
labels = np.asarray(labels)
if masks is not None:
masks = np.asarray(masks)
empty = len(bboxes) == 0
if not empty and scores is not None and score_th > 0:
sel = scores >= score_th
bboxes = bboxes[sel]
labels = labels[sel]
if masks is not None:
masks = masks[sel]
empty = len(bboxes) == 0
if empty:
if out_file is not None:
imwrite(img, out_file)
return img
if out_scale != 1:
bboxes = out_scale * bboxes
bboxes = bboxes.round().astype(np.int32)
# draw masks or filled rectangles
if masks is None:
img_filled = img.copy()
for i, (bbox, label) in enumerate(zip(bboxes, labels)):
if color_scheme == 'class':
color = color_palette[label]
cv2.rectangle(
img,
(bbox[0], bbox[1]),
(bbox[2], bbox[3]),
color,
thickness=-1,
)
img = cv2.addWeighted(img_filled, (1 - alpha), img, alpha, 0)
# draw box contours
for i, (bbox, label) in enumerate(zip(bboxes, labels)):
if color_scheme == 'class':
color = color_palette[label]
cv2.rectangle(
img,
(bbox[0], bbox[1]),
(bbox[2], bbox[3]),
color,
thickness=thickness,
)
else:
for i, (mask, label) in enumerate(zip(masks, labels)):
if color_scheme == 'class':
color = np.array(color_palette[label])
m = maskUtils.decode(mask)
if out_scale != 1:
m = cv2.resize(
m.astype(np.uint8),
fx=out_scale,
fy=out_scale,
interpolation=cv2.INTER_NEAREST,
)
m = m.astype(np.bool)
img[m] = (1 - alpha) * img[m] + alpha * color
b = find_boundaries(m)
img[b] = color
# put label text
if show_label or show_score:
if font is not None:
# using TrueType supported in PIL
img = Image.fromarray(img)
draw = ImageDraw.Draw(img)
for i, (bbox, label) in enumerate(zip(bboxes, labels)):
if color_scheme == 'class':
color = color_palette[label]
lt = (bbox[0], bbox[1])
if show_label:
if class_names is None:
label_text = f'class {label}'
else:
label_text = class_names[label]
else:
label_text = ''
if show_score and scores is not None:
score_text = f'{scores[i]:.02f}'
else:
score_text = ''
if label_text and score_text:
text = label_text + ' - ' + score_text
elif label_text:
text = label_text
elif score_text:
text = score_text
else:
text = ''
if font is None:
cv2.putText(
img,
text,
(lt[0], lt[1] - 2),
cv2.FONT_HERSHEY_DUPLEX,
1.7,
color,
thickness=1,
)
else:
draw.text(
(lt[0], lt[1] - font.size),
text,
(*color, 1), # RGBA
font=font,
)
if font is not None:
img = np.asarray(img)
if out_file is not None:
imwrite(img, out_file)
return img
def main():
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
box_ccf = pickle.load(open(opts.box_ccf_path, 'rb'))
mask_ccf = pickle.load(open(opts.mask_ccf_path, 'rb'))
box_end_idx = 0
mask_end_idx = 0
results_ccf = [] # instance based
for iid, img in tqdm(db.imgs.items()):
img_name = img['name']
sid = img['sid']
seq_name = seqs[sid]
box_start_idx = box_end_idx
while box_start_idx < len(box_ccf) and box_ccf[box_start_idx]['image_id'] < img['id']:
box_start_idx += 1
box_end_idx = box_start_idx
while box_end_idx < len(box_ccf) and box_ccf[box_end_idx]['image_id'] == img['id']:
box_end_idx += 1
box_dets = box_ccf[box_start_idx:box_end_idx]
mask_start_idx = mask_end_idx
while mask_start_idx < len(mask_ccf) and mask_ccf[mask_start_idx]['image_id'] < img['id']:
mask_start_idx += 1
mask_end_idx = mask_start_idx
while mask_end_idx < len(mask_ccf) and mask_ccf[mask_end_idx]['image_id'] == img['id']:
mask_end_idx += 1
mask_dets = mask_ccf[mask_start_idx:mask_end_idx]
if len(box_dets) == 0:
bboxes1, scores1, labels1, masks1 = [], [], [], None
elif len(mask_dets) == 0:
bboxes1 = np.array([d['bbox'] for d in box_dets])
scores1 = np.array([d['score'] for d in box_dets])
labels1 = np.array([d['category_id'] for d in box_dets])
masks1 = np.array([d['segmentation'] for d in box_dets])
else:
# the slow version, but works with out of order ccf results
# dets = [r for r in box_ccf if r['image_id'] == img['id']]
bboxes1 = np.array([d['bbox'] for d in box_dets])
scores1 = np.array([d['score'] for d in box_dets])
labels1 = np.array([d['category_id'] for d in box_dets])
masks1 = np.array([d['segmentation'] for d in box_dets])
bboxes2 = np.array([d['bbox'] for d in mask_dets])
scores2 = np.array([d['score'] for d in mask_dets])
labels2 = np.array([d['category_id'] for d in mask_dets])
masks2 = np.array([d['segmentation'] for d in mask_dets])
score_argsort = np.argsort(scores1)[::-1]
bboxes1 = bboxes1[score_argsort]
scores1 = scores1[score_argsort]
labels1 = labels1[score_argsort]
masks1 = masks1[score_argsort]
score_argsort = np.argsort(scores2)[::-1]
bboxes2 = bboxes2[score_argsort]
scores2 = scores2[score_argsort]
labels2 = labels2[score_argsort]
masks2 = masks2[score_argsort]
order1, order2, n_matched12, _, _ = iou_assoc(
bboxes1, labels1, np.arange(len(bboxes1)), 0,
bboxes2, labels2, opts.match_iou_th,
no_unmatched1=False,
)
bboxes1 = bboxes1[order1]
scores1 = scores1[order1]
labels1 = labels1[order1]
masks1 = masks1[order1]
bboxes2 = bboxes2[order2]
scores2 = scores2[order2]
labels2 = labels2[order2]
masks2 = masks2[order2]
mask1_fix = warp_mask_to_box(
masks2[:n_matched12],
bboxes2[:n_matched12],
bboxes1[:n_matched12]
)
masks1 = np.concatenate((mask1_fix, masks1[n_matched12:]))
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img_name)
I = imread(img_path)
vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg')
bboxes = ltwh2ltrb(bboxes1) if len(bboxes1) else []
if opts.overwrite or not isfile(vis_path):
vis_det(
I, bboxes, labels1,
class_names, masks1, scores1,
out_scale=opts.vis_scale,
out_file=vis_path
)
n = len(bboxes1)
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes1[i],
'score': scores1[i],
'category_id': labels1[i],
}
if masks1 is not None:
result_dict['segmentation'] = masks1[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main():
opts = parse_args()
# caches = [
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/retina50_s0.2/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.5/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.75/val/results_ccf.pkl'),
# ]
# # should be sorted
# rtfs = [1, 2, 3]
# caches = [
# join(opts.exp_dir, 'ArgoVerse1.1/output/mrcnn50_nm_th0_s0.75/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1/output/mrcnn50_nm_th0_s1.0/val/results_ccf.pkl'),
# ]
# # should be sorted
# rtfs = [3, 5]
caches = [
join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.75/val/results_ccf.pkl'),
join(opts.exp_dir, 'ArgoVerse1.1/output/cmrcnn101_nm_s1.0/val/results_ccf.pkl'),
]
# should be sorted
rtfs = [3, 5]
n_method = len(caches)
max_history = max(rtfs)
cache_ccfs = [
pickle.load(open(path, 'rb'))
for path in caches
]
cache_end_idx = n_method*[0]
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
results_ccf = []
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
w_img, h_img = db.imgs[0]['width'], db.imgs[0]['height']
bf = BboxFilterEx(
w_img, h_img,
forecast=opts.forecast,
forecast_before_assoc=opts.forecast_before_assoc,
)
# backward in time
# cur, cur - 1, ..., cur - max_history
result_buffer = (max_history + 1)*[None]
result_buffer_marker = (max_history + 1)*[None]
min_rtf = min(rtfs)
for ii, img in enumerate(frame_list):
# fetch results
for i in range(n_method):
ifidx = ii - rtfs[i]
if ifidx < 0:
break
cache_end_idx[i], bboxes, scores, labels = \
result_from_ccf(cache_ccfs[i], frame_list[ifidx]['id'], cache_end_idx[i], mask=False)
result_buffer[rtfs[i]] = (bboxes, scores, labels)
result_buffer_marker[rtfs[i]] = i == 0
if ii < min_rtf:
# no method has finished yet
bboxes, scores, labels, tracks = [], [], [], []
else:
s = max_history
while s >= 0 and result_buffer[s] is None:
s -= 1
# if first result is one step ahead
t = ii - s
birth_tracks = True
kill_tracks = True
if s == max_history:
bf.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
bf_this = deepcopy(bf)
else:
bf_this = deepcopy(bf)
bf_this.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
while 1:
# find next non-empty result
s -= 1
while s >= 0 and result_buffer[s] is None:
s -= 1
if s < 0:
break
if result_buffer_marker[s]:
birth_tracks = opts.fast_birth_tracks
kill_tracks = opts.fast_kill_tracks
else:
birth_tracks = True
kill_tracks = True
t = ii - s
bf_this.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
bboxes, scores, labels, _, tracks = \
bf_this.predict(ii)
# shift result buffer
result_buffer.pop(max_history)
result_buffer.insert(0, None)
result_buffer_marker.pop(max_history)
result_buffer_marker.insert(0, None)
n = len(bboxes)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
# if masks is not None:
# result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
bboxes_ltrb = ltwh2ltrb(bboxes) if n else []
if opts.overwrite or not isfile(vis_path):
vis_track(
I, bboxes_ltrb, tracks, labels,
class_names, None, scores,
out_scale=opts.vis_scale,
out_file=vis_path
)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
if opts.weights_base is not None:
# for distillation purpose
load_checkpoint(model, opts.weights_base)
if opts.cpu_pre:
img_transform = ImageTransform(
size_divisor=model.cfg.data.test.size_divisor,
**model.cfg.img_norm_cfg)
else:
img_transform = ImageTransformGPU(
size_divisor=model.cfg.data.test.size_divisor,
**model.cfg.img_norm_cfg)
device = next(model.parameters()).device # model device
n_history = model.cfg.data.train.n_history if opts.n_history is None else opts.n_history
n_future = model.cfg.data.train.n_future if opts.n_future is None else opts.n_future
results_ccf = [] # instance based
runtime_all = []
for sid, seq in enumerate(tqdm(seqs)):
# print(seq)
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
n_frame = len(frame_list)
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
with torch.no_grad():
preprocessed = []
for i in range(n_history):
data = _prepare_data(frames[i], img_transform, model.cfg,
device)
preprocessed.append(data)
for ii in range(n_history, n_frame - n_future):
# target frame
iid = frame_list[ii + n_future]['id']
img_name = frame_list[ii + n_future]['name']
I = frames[ii + n_future]
t_start = perf_counter()
# input frame
data = _prepare_data(frames[ii], img_transform, model.cfg,
device)
# if n_history == 0:
# data_merge = data
# # print(data['img'])
# # print(data['img'][0].shape)
# # print(data['img'][0][0][0][300][300:305])
# # import sys
# # sys.exit()
# else:
preprocessed.append(data)
# print(preprocessed[0]['img'][0].data_ptr())
# print(preprocessed[2]['img'][0].data_ptr())
# print(torch.all(preprocessed[0]['img'][0] == preprocessed[2]['img'][0]))
imgs = [d['img'][0] for d in preprocessed]
imgs = torch.cat(imgs, 0)
imgs = imgs.unsqueeze(0)
data_merge = {
'img': [imgs],
'img_meta': data['img_meta'],
}
# print(data_merge['img'][0][0][2][0][300][300:305])
# import sys
# sys.exit()
result = model(return_loss=False,
rescale=True,
numpy_res=True,
**data_merge)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
# if ii == 2:
# print(ii, scores)
# import sys
# sys.exit()
# if n_history != 0:
del preprocessed[0]
t_end = perf_counter()
runtime_all.append(t_end - t_start)
if vis_out:
vis_path = join(opts.vis_dir, seq, img_name[:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path)
# convert to coco fmt
n = len(bboxes)
if n:
bboxes[:, 2:] -= bboxes[:, :2]
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'runtime_all': runtime_all,
'n_total': len(runtime_all),
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3 * x
print_stats(runtime_all, 'Runtime (ms)', cvt=s2ms)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main():
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
assert class_names[5] == 'truck'
assert class_names[2] == 'car'
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
in_ccf = pickle.load(open(join(opts.in_dir, 'results_ccf.pkl'), 'rb'))
results_ccf = []
i_start = 0
for sid, seq in enumerate(tqdm(seqs)):
# if seq != '5ab2697b-6e3e-3454-a36a-aba2c6f27818':
# continue
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
for ii, img in enumerate(frame_list):
iid = img['id']
i_end = i_start
while i_end < len(in_ccf) and in_ccf[i_end]['image_id'] == iid:
i_end += 1
dets = in_ccf[i_start:i_end]
i_start = i_end
# dets = [r for r in in_ccf if r['image_id'] == img['id']]
bboxes = np.array([d['bbox'] for d in dets])
labels = np.array([d['category_id'] for d in dets])
scores = np.array([d['score'] for d in dets])
with_mask = len(dets) and 'segmentation' in dets[0]
if with_mask:
masks = np.array([d['segmentation'] for d in dets])
score_argsort = np.argsort(scores)[::-1]
bboxes = bboxes[score_argsort]
scores = scores[score_argsort]
labels = labels[score_argsort]
if with_mask:
masks = masks[score_argsort]
remove_idx = remove_car_for_pickup(bboxes, labels, scores)
bboxes = np.delete(bboxes, remove_idx, 0)
labels = np.delete(labels, remove_idx, 0)
scores = np.delete(scores, remove_idx, 0)
if with_mask:
masks = np.delete(masks, remove_idx, 0)
n = len(bboxes)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
if with_mask:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
bboxes_ltrb = bboxes.copy()
if len(bboxes_ltrb):
bboxes_ltrb[:, 2:] += bboxes_ltrb[:, :2]
if opts.overwrite or not isfile(vis_path):
vis_det(I,
bboxes_ltrb,
labels,
class_names,
masks if with_mask else None,
scores,
out_scale=opts.vis_scale,
out_file=vis_path)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps 30')
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
# warm up the GPU
img = db.imgs[0]
w_img, h_img = img['width'], img['height']
_ = inference_detector(model, np.zeros((h_img, w_img, 3), np.uint8))
torch.cuda.synchronize()
runtime_all = []
n_processed = 0
n_total = 0
# global pr
import cProfile
pr = cProfile.Profile()
for sid, seq in enumerate(tqdm(seqs)):
# if sid > 1:
# break
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
n_frame = len(frames)
n_total += n_frame
timestamps = []
results = []
input_fidx = []
runtime = []
last_fidx = None
t_total = n_frame / opts.fps
t_start = perf_counter()
while 1:
t1 = perf_counter()
t_elapsed = t1 - t_start
if t_elapsed >= t_total:
break
# identify latest available frame
fidx = int(np.floor(t_elapsed * opts.fps))
# t_elapsed/t_total *n_frame
# = t_elapsed*opts.fps
if fidx == last_fidx:
continue
last_fidx = fidx
frame = frames[fidx]
pr.enable()
result = inference_detector(model, frame)
torch.cuda.synchronize()
pr.disable()
t2 = perf_counter()
t_elapsed = t2 - t_start
if t_elapsed >= t_total:
break
timestamps.append(t_elapsed)
results.append(result)
input_fidx.append(fidx)
runtime.append(t2 - t1)
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump((
results,
timestamps,
input_fidx,
runtime,
), open(out_path, 'wb'))
runtime_all += runtime
n_processed += len(results)
pr.dump_stats('_par_/mrcnn50_s0.5_1080ti_gpupre_blocking.prof')
# pr.dump_stats('debug.prof')
runtime_all_np = np.array(runtime_all)
n_small_runtime = (runtime_all_np < 1.0 / opts.fps).sum()
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(
{
'runtime_all': runtime_all,
'n_processed': n_processed,
'n_total': n_total,
'n_small_runtime': n_small_runtime,
}, open(out_path, 'wb'))
# convert to ms for display
runtime_all_np *= 1e3
print(f'{n_processed}/{n_total} frames processed')
print('Runtime (ms): mean: %g; std: %g; min: %g; max: %g' % (
runtime_all_np.mean(),
runtime_all_np.std(ddof=1),
runtime_all_np.min(),
runtime_all_np.max(),
))
print(f'Runtime smaller than unit time interval: '
f'{n_small_runtime}/{n_processed} '
f'({100.0*n_small_runtime/n_processed:.4g}%)')
def main():
opts = parse_args()
assert opts.forecast_before_assoc, "Not implemented"
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
results_ccf = []
in_time = 0
miss = 0
shifts = 0
given_tracks = opts.assoc == 'given'
assert not given_tracks, "Not implemented"
t_assoc = []
t_forecast = []
# tt = []
with torch.no_grad():
kf_F = torch.eye(8)
# kf_F[3, 7] = 1
kf_Q = torch.eye(8)
kf_R = 10 * torch.eye(4)
kf_P_init = 100 * torch.eye(8).unsqueeze(0)
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results = pickle.load(open(join(opts.in_dir, seq + '.pkl'), 'rb'))
# use raw results when possible in case we change class subset during evaluation
results_raw = results.get('results_raw', None)
if results_raw is None:
results_parsed = results['results_parsed']
timestamps = results['timestamps']
input_fidx = results['input_fidx']
# t1 -> det1, t2 -> det2, interpolate at t3 (t3 is the current time)
det_latest_p1 = 0 # latest detection index + 1
det_t2 = None # detection index at t2
kf_x = torch.empty((0, 8, 1))
kf_P = torch.empty((0, 8, 8))
n_matched12 = 0
if not given_tracks:
tkidx = 0 # track starting index
for ii, img in enumerate(frame_list):
# pred, gt association by time
t = (ii - opts.eta) / opts.fps
while det_latest_p1 < len(
timestamps) and timestamps[det_latest_p1] <= t:
det_latest_p1 += 1
if det_latest_p1 == 0:
# no detection output
miss += 1
bboxes_t3, scores, labels, tracks = [], [], [], []
else:
det_latest = det_latest_p1 - 1
ifidx = input_fidx[det_latest]
in_time += int(ii == ifidx)
shifts += ii - ifidx
if det_latest != det_t2:
# new detection
# we can now throw away old result (t1)
# the old one is kept for forecasting purpose
if len(kf_x) and opts.forecast_before_assoc:
dt = ifidx - input_fidx[det_t2]
dt = int(
dt
) # convert from numpy to basic python format
w_img, h_img = img['width'], img['height']
kf_F = make_F(kf_F, dt)
kf_Q = make_Q(kf_Q, dt)
kf_x, kf_P = batch_kf_predict(
kf_F, kf_x, kf_P, kf_Q)
bboxes_f = x2bbox(kf_x)
det_t2 = det_latest
if results_raw is None:
result_t2 = results_parsed[det_t2]
bboxes_t2, scores_t2, labels_t2, _ = result_t2[:4]
if not given_tracks and len(result_t2) > 4:
tracks_t2 = np.asarray(result_t2[4])
else:
tracks_t2 = np.empty((0, ), np.uint32)
else:
result_t2 = results_raw[det_t2]
bboxes_t2, scores_t2, labels_t2, _ = \
parse_det_result(result_t2, coco_mapping, n_class)
t1 = perf_counter()
n = len(bboxes_t2)
if n:
if not given_tracks:
# put high scores det first for better iou matching
score_argsort = np.argsort(scores_t2)[::-1]
bboxes_t2 = bboxes_t2[score_argsort]
scores_t2 = scores_t2[score_argsort]
labels_t2 = labels_t2[score_argsort]
ltrb2ltwh_(bboxes_t2)
if given_tracks:
raise NotImplementedError
order1, order2, n_matched12 = track_based_shuffle(
tracks, tracks_t2, no_unmatched1=True)
tracks = tracks[order2]
else:
updated = False
if len(kf_x):
order1, order2, n_matched12, tracks, tkidx = iou_assoc(
bboxes_f,
labels,
tracks,
tkidx,
bboxes_t2,
labels_t2,
opts.match_iou_th,
no_unmatched1=True,
)
if n_matched12:
kf_x = kf_x[order1]
kf_P = kf_P[order1]
kf_x, kf_P = batch_kf_update(
bbox2z(bboxes_t2[
order2[:n_matched12]]),
kf_x,
kf_P,
kf_R,
)
kf_x_new = bbox2x(
bboxes_t2[order2[n_matched12:]])
n_unmatched2 = len(
bboxes_t2) - n_matched12
kf_P_new = kf_P_init.expand(
n_unmatched2, -1, -1)
kf_x = torch.cat((kf_x, kf_x_new))
kf_P = torch.cat((kf_P, kf_P_new))
labels = labels_t2[order2]
scores = scores_t2[order2]
updated = True
if not updated:
# start from scratch
kf_x = bbox2x(bboxes_t2)
kf_P = kf_P_init.expand(
len(bboxes_t2), -1, -1)
labels = labels_t2
scores = scores_t2
if not given_tracks:
tracks = np.arange(tkidx,
tkidx + n,
dtype=np.uint32)
tkidx += n
t2 = perf_counter()
t_assoc.append(t2 - t1)
t3 = perf_counter()
if len(kf_x):
dt = ii - ifidx
w_img, h_img = img['width'], img['height']
# PyTorch small matrix multiplication is slow
# use numpy instead
# kf_F = make_F(kf_F, dt)
# bboxes_t3 = x2bbox(batch_kf_predict_only(kf_F, kf_x))
# t5 = perf_counter()
kf_x_np = kf_x[:, :, 0].numpy()
# t6 = perf_counter()
bboxes_t3 = kf_x_np[:n_matched12, :
4] + dt * kf_x_np[:n_matched12, 4:]
if n_matched12 < len(kf_x):
bboxes_t3 = np.concatenate(
(bboxes_t3, kf_x_np[n_matched12:, :4]))
bboxes_t3, keep = extrap_clean_up(bboxes_t3,
w_img,
h_img,
lt=True)
# torch_keep = torch.from_numpy(keep) # boolean mask
# kf_x = kf_x[torch_keep]
# kf_P = kf_P[torch_keep]
# labels = labels[keep]
# scores = scores[keep]
# tracks = tracks[keep]
labels_t3 = labels[keep]
scores_t3 = scores[keep]
tracks_t3 = tracks[keep]
# tt.append(t6 - t5)
t4 = perf_counter()
t_forecast.append(t4 - t3)
n = len(bboxes_t3)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes_t3[i],
'score': scores_t3[i],
'category_id': labels_t3[i],
}
results_ccf.append(result_dict)
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq,
img['name'][:-3] + 'jpg')
bboxes = bboxes_t3.copy()
if n:
ltwh2ltrb_(bboxes)
if opts.overwrite or not isfile(vis_path):
vis_track(
I,
bboxes,
tracks,
labels,
class_names,
None,
scores,
out_scale=opts.vis_scale,
out_file=vis_path,
)
# pr.disable()
# pr.dump_stats('pps_iou_lin_extrap.prof')
# pickle.dump([t_assoc, t_forecast], open('pps_iou_lin_forecast_time', 'wb'))
s2ms = lambda x: 1e3 * x
if len(t_assoc):
print_stats(t_assoc, "RT association (ms)", cvt=s2ms)
if len(t_forecast):
print_stats(t_forecast, "RT forecasting (ms)", cvt=s2ms)
# if len(tt):
# print_stats(tt, "RT forecasting2 (ms)", cvt=s2ms)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
def main():
opts = parse_args()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
if opts.cached_res:
cache_in_ccf = '_ccf' in basename(opts.cached_res)
if cache_in_ccf:
# speed up based on the assumption of sequential storage
cache_end_idx = 0
cached_res = pickle.load(open(opts.cached_res, 'rb'))
else:
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
model = init_detector(opts)
np.random.seed(opts.seed)
runtime = pickle.load(open(opts.runtime, 'rb'))
runtime_dist = dist_from_dict(runtime, opts.perf_factor)
runtime_all = []
n_processed = 0
n_total = 0
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
n_frame = len(frame_list)
n_total += n_frame
if not opts.cached_res:
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
timestamps = []
results_raw = []
results_parsed = []
input_fidx = []
runtime = []
if opts.cached_res and cache_in_ccf:
results_raw = None
else:
results_raw = []
for ii in range(n_frame):
t = ii / opts.fps
if opts.cached_res:
img = frame_list[ii]
if cache_in_ccf:
cache_end_idx, bboxes, scores, labels, masks = \
result_from_ccf(cached_res, img['id'], cache_end_idx)
ltwh2ltrb_(bboxes)
else:
result = cached_res[img['id']]
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
else:
frame = frames[ii]
result = inference_detector(model, frame)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
rt_this = runtime_dist.draw()
t_finishing = t + rt_this
timestamps.append(t_finishing)
if results_raw is not None:
results_raw.append(result)
results_parsed.append((bboxes, scores, labels, masks))
input_fidx.append(ii)
runtime.append(rt_this)
# since parallel excecution, the order is not guaranteed
idx = np.argsort(timestamps)
timestamps = [timestamps[i] for i in idx]
if results_raw is not None:
results_raw = [results_raw[i] for i in idx]
results_parsed = [results_parsed[i] for i in idx]
input_fidx = [input_fidx[i] for i in idx]
runtime = [runtime[i] for i in idx]
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
out_dict = {
'results_parsed': results_parsed,
'timestamps': timestamps,
'input_fidx': input_fidx,
'runtime': runtime,
}
if results_raw is not None:
out_dict['results_raw'] = results_raw
pickle.dump(out_dict, open(out_path, 'wb'))
runtime_all += runtime
n_processed += len(results_parsed)
runtime_all_np = np.array(runtime_all)
n_small_runtime = (runtime_all_np < 1.0 / opts.fps).sum()
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(
{
'runtime_all': runtime_all,
'n_processed': n_processed,
'n_total': n_total,
'n_small_runtime': n_small_runtime,
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3 * x
print(f'{n_processed}/{n_total} frames processed')
print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms)
print(f'Runtime smaller than unit time interval: '
f'{n_small_runtime}/{n_processed} '
f'({100.0*n_small_runtime/n_processed:.4g}%)')
