def main():
opts = parse_args()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
eval_summary = eval_ccf(db, opts.result_path, None, opts.per_class)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, opts.result_path, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
def main():
opts = parse_args()
out_dir = mkdir2(opts.out_dir) if opts.out_dir else opts.result_dir
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
results_ccf = []
in_time = 0
miss = 0
mismatch = 0
print('Pairing the output with the ground truth')
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb'))
# use raw results when possible in case we change class subset during evaluation
if opts.use_parsed:
results_parsed = results['results_parsed']
else:
results_raw = results.get('results_raw', None)
if results_raw is None:
results_parsed = results['results_parsed']
timestamps = results['timestamps']
input_fidx = results['input_fidx']
tidx_p1 = 0
for ii, img in enumerate(frame_list):
# pred, gt association by time
t = (ii - opts.eta) / opts.fps
while tidx_p1 < len(timestamps) and timestamps[tidx_p1] <= t:
tidx_p1 += 1
if tidx_p1 == 0:
# no output
miss += 1
bboxes, scores, labels = [], [], []
masks, tracks = None, None
else:
tidx = tidx_p1 - 1
ifidx = input_fidx[tidx]
in_time += int(ii == ifidx)
mismatch += ii - ifidx
if opts.use_parsed or results_raw is None:
result = results_parsed[tidx]
bboxes, scores, labels, masks = result[:4]
if len(result) > 4:
tracks = result[4]
else:
tracks = None
else:
result = results_raw[tidx]
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
tracks = None
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
if tracks is None:
vis_det(
I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path,
)
else:
vis_track(
I,
bboxes,
tracks,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path,
)
# convert to coco fmt
n = len(bboxes)
if n:
bboxes_ltwh = ltrb2ltwh(bboxes)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes_ltwh[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
out_path = join(out_dir, 'eval_assoc.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'miss': miss,
'in_time': in_time,
'mismatch': mismatch,
}, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
results_raw = [] # image based, all 80 COCO classes
results_ccf = [] # instance based
for iid, img in tqdm(db.imgs.items()):
img_name = img['name']
sid = img['sid']
seq_name = seqs[sid]
img_path = join(opts.data_root, seq_dirs[sid], img_name)
I = imread(img_path)
result = inference_detector(model, I, gpu_pre=not opts.cpu_pre)
results_raw.append(result)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
if vis_out:
vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path)
# convert to coco fmt
n = len(bboxes)
if n:
ltrb2ltwh_(bboxes)
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'results_raw.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_raw, open(out_path, 'wb'))
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main():
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
box_ccf = pickle.load(open(opts.box_ccf_path, 'rb'))
mask_ccf = pickle.load(open(opts.mask_ccf_path, 'rb'))
box_end_idx = 0
mask_end_idx = 0
results_ccf = [] # instance based
for iid, img in tqdm(db.imgs.items()):
img_name = img['name']
sid = img['sid']
seq_name = seqs[sid]
box_start_idx = box_end_idx
while box_start_idx < len(box_ccf) and box_ccf[box_start_idx]['image_id'] < img['id']:
box_start_idx += 1
box_end_idx = box_start_idx
while box_end_idx < len(box_ccf) and box_ccf[box_end_idx]['image_id'] == img['id']:
box_end_idx += 1
box_dets = box_ccf[box_start_idx:box_end_idx]
mask_start_idx = mask_end_idx
while mask_start_idx < len(mask_ccf) and mask_ccf[mask_start_idx]['image_id'] < img['id']:
mask_start_idx += 1
mask_end_idx = mask_start_idx
while mask_end_idx < len(mask_ccf) and mask_ccf[mask_end_idx]['image_id'] == img['id']:
mask_end_idx += 1
mask_dets = mask_ccf[mask_start_idx:mask_end_idx]
if len(box_dets) == 0:
bboxes1, scores1, labels1, masks1 = [], [], [], None
elif len(mask_dets) == 0:
bboxes1 = np.array([d['bbox'] for d in box_dets])
scores1 = np.array([d['score'] for d in box_dets])
labels1 = np.array([d['category_id'] for d in box_dets])
masks1 = np.array([d['segmentation'] for d in box_dets])
else:
# the slow version, but works with out of order ccf results
# dets = [r for r in box_ccf if r['image_id'] == img['id']]
bboxes1 = np.array([d['bbox'] for d in box_dets])
scores1 = np.array([d['score'] for d in box_dets])
labels1 = np.array([d['category_id'] for d in box_dets])
masks1 = np.array([d['segmentation'] for d in box_dets])
bboxes2 = np.array([d['bbox'] for d in mask_dets])
scores2 = np.array([d['score'] for d in mask_dets])
labels2 = np.array([d['category_id'] for d in mask_dets])
masks2 = np.array([d['segmentation'] for d in mask_dets])
score_argsort = np.argsort(scores1)[::-1]
bboxes1 = bboxes1[score_argsort]
scores1 = scores1[score_argsort]
labels1 = labels1[score_argsort]
masks1 = masks1[score_argsort]
score_argsort = np.argsort(scores2)[::-1]
bboxes2 = bboxes2[score_argsort]
scores2 = scores2[score_argsort]
labels2 = labels2[score_argsort]
masks2 = masks2[score_argsort]
order1, order2, n_matched12, _, _ = iou_assoc(
bboxes1, labels1, np.arange(len(bboxes1)), 0,
bboxes2, labels2, opts.match_iou_th,
no_unmatched1=False,
)
bboxes1 = bboxes1[order1]
scores1 = scores1[order1]
labels1 = labels1[order1]
masks1 = masks1[order1]
bboxes2 = bboxes2[order2]
scores2 = scores2[order2]
labels2 = labels2[order2]
masks2 = masks2[order2]
mask1_fix = warp_mask_to_box(
masks2[:n_matched12],
bboxes2[:n_matched12],
bboxes1[:n_matched12]
)
masks1 = np.concatenate((mask1_fix, masks1[n_matched12:]))
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img_name)
I = imread(img_path)
vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg')
bboxes = ltwh2ltrb(bboxes1) if len(bboxes1) else []
if opts.overwrite or not isfile(vis_path):
vis_det(
I, bboxes, labels1,
class_names, masks1, scores1,
out_scale=opts.vis_scale,
out_file=vis_path
)
n = len(bboxes1)
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes1[i],
'score': scores1[i],
'category_id': labels1[i],
}
if masks1 is not None:
result_dict['segmentation'] = masks1[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main():
opts = parse_args()
# caches = [
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/retina50_s0.2/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.5/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.75/val/results_ccf.pkl'),
# ]
# # should be sorted
# rtfs = [1, 2, 3]
# caches = [
# join(opts.exp_dir, 'ArgoVerse1.1/output/mrcnn50_nm_th0_s0.75/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1/output/mrcnn50_nm_th0_s1.0/val/results_ccf.pkl'),
# ]
# # should be sorted
# rtfs = [3, 5]
caches = [
join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.75/val/results_ccf.pkl'),
join(opts.exp_dir, 'ArgoVerse1.1/output/cmrcnn101_nm_s1.0/val/results_ccf.pkl'),
]
# should be sorted
rtfs = [3, 5]
n_method = len(caches)
max_history = max(rtfs)
cache_ccfs = [
pickle.load(open(path, 'rb'))
for path in caches
]
cache_end_idx = n_method*[0]
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
results_ccf = []
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
w_img, h_img = db.imgs[0]['width'], db.imgs[0]['height']
bf = BboxFilterEx(
w_img, h_img,
forecast=opts.forecast,
forecast_before_assoc=opts.forecast_before_assoc,
)
# backward in time
# cur, cur - 1, ..., cur - max_history
result_buffer = (max_history + 1)*[None]
result_buffer_marker = (max_history + 1)*[None]
min_rtf = min(rtfs)
for ii, img in enumerate(frame_list):
# fetch results
for i in range(n_method):
ifidx = ii - rtfs[i]
if ifidx < 0:
break
cache_end_idx[i], bboxes, scores, labels = \
result_from_ccf(cache_ccfs[i], frame_list[ifidx]['id'], cache_end_idx[i], mask=False)
result_buffer[rtfs[i]] = (bboxes, scores, labels)
result_buffer_marker[rtfs[i]] = i == 0
if ii < min_rtf:
# no method has finished yet
bboxes, scores, labels, tracks = [], [], [], []
else:
s = max_history
while s >= 0 and result_buffer[s] is None:
s -= 1
# if first result is one step ahead
t = ii - s
birth_tracks = True
kill_tracks = True
if s == max_history:
bf.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
bf_this = deepcopy(bf)
else:
bf_this = deepcopy(bf)
bf_this.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
while 1:
# find next non-empty result
s -= 1
while s >= 0 and result_buffer[s] is None:
s -= 1
if s < 0:
break
if result_buffer_marker[s]:
birth_tracks = opts.fast_birth_tracks
kill_tracks = opts.fast_kill_tracks
else:
birth_tracks = True
kill_tracks = True
t = ii - s
bf_this.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
bboxes, scores, labels, _, tracks = \
bf_this.predict(ii)
# shift result buffer
result_buffer.pop(max_history)
result_buffer.insert(0, None)
result_buffer_marker.pop(max_history)
result_buffer_marker.insert(0, None)
n = len(bboxes)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
# if masks is not None:
# result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
bboxes_ltrb = ltwh2ltrb(bboxes) if n else []
if opts.overwrite or not isfile(vis_path):
vis_track(
I, bboxes_ltrb, tracks, labels,
class_names, None, scores,
out_scale=opts.vis_scale,
out_file=vis_path
)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
def main():
assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
n_class = len(db.cats)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
if 'KMOTS' in opts.data_root:
class_mapping = coco2kmots
class_names = kmots_classes
elif 'ArgoVerse' in opts.data_root:
class_mapping = coco2av
class_names = av_classes
else:
raise Exception('Unknown dataset')
config = mmcv.Config.fromfile(opts.config)
if opts.in_scale is not None:
config.data.test.img_scale = opts.in_scale
# mainly for SSD
config.data.test.resize_keep_ratio = True
if opts.no_mask:
if 'mask_head' in config.model:
config.model['mask_head'] = None
model = init_detector(config, opts.weights)
model.eval()
results_ccf = []
for iid, img in tqdm(db.imgs.items()):
img_name = img['name']
sid = img['sid']
seq_name = seqs[sid]
img_path = join(opts.data_root, seq_dirs[sid], img_name)
I = mmcv.imread(img_path)
if iid % opts.det_stride == 0:
result = inference_detector(model, I)
bboxes, scores, labels, masks = parse_mmdet_result(result, class_mapping, n_class)
if len(bboxes):
bboxes_ltwh = bboxes.copy()
# convert to coco fmt
bboxes_ltwh[:, 2:] -= bboxes_ltwh[:, :2]
bboxes_ltwh = bboxes_ltwh.tolist()
else:
bboxes_ltwh = []
if vis_out:
vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(
I, bboxes, labels,
class_names, masks, scores,
out_scale=opts.vis_scale,
out_file=vis_path
)
for i in range(len(bboxes_ltwh)):
result_dict = {
'image_id': iid,
'bbox': bboxes_ltwh[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
if opts.weights_base is not None:
# for distillation purpose
load_checkpoint(model, opts.weights_base)
if opts.cpu_pre:
img_transform = ImageTransform(
size_divisor=model.cfg.data.test.size_divisor,
**model.cfg.img_norm_cfg)
else:
img_transform = ImageTransformGPU(
size_divisor=model.cfg.data.test.size_divisor,
**model.cfg.img_norm_cfg)
device = next(model.parameters()).device # model device
n_history = model.cfg.data.train.n_history if opts.n_history is None else opts.n_history
n_future = model.cfg.data.train.n_future if opts.n_future is None else opts.n_future
results_ccf = [] # instance based
runtime_all = []
for sid, seq in enumerate(tqdm(seqs)):
# print(seq)
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
n_frame = len(frame_list)
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
with torch.no_grad():
preprocessed = []
for i in range(n_history):
data = _prepare_data(frames[i], img_transform, model.cfg,
device)
preprocessed.append(data)
for ii in range(n_history, n_frame - n_future):
# target frame
iid = frame_list[ii + n_future]['id']
img_name = frame_list[ii + n_future]['name']
I = frames[ii + n_future]
t_start = perf_counter()
# input frame
data = _prepare_data(frames[ii], img_transform, model.cfg,
device)
# if n_history == 0:
# data_merge = data
# # print(data['img'])
# # print(data['img'][0].shape)
# # print(data['img'][0][0][0][300][300:305])
# # import sys
# # sys.exit()
# else:
preprocessed.append(data)
# print(preprocessed[0]['img'][0].data_ptr())
# print(preprocessed[2]['img'][0].data_ptr())
# print(torch.all(preprocessed[0]['img'][0] == preprocessed[2]['img'][0]))
imgs = [d['img'][0] for d in preprocessed]
imgs = torch.cat(imgs, 0)
imgs = imgs.unsqueeze(0)
data_merge = {
'img': [imgs],
'img_meta': data['img_meta'],
}
# print(data_merge['img'][0][0][2][0][300][300:305])
# import sys
# sys.exit()
result = model(return_loss=False,
rescale=True,
numpy_res=True,
**data_merge)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
# if ii == 2:
# print(ii, scores)
# import sys
# sys.exit()
# if n_history != 0:
del preprocessed[0]
t_end = perf_counter()
runtime_all.append(t_end - t_start)
if vis_out:
vis_path = join(opts.vis_dir, seq, img_name[:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path)
# convert to coco fmt
n = len(bboxes)
if n:
bboxes[:, 2:] -= bboxes[:, :2]
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'runtime_all': runtime_all,
'n_total': len(runtime_all),
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3 * x
print_stats(runtime_all, 'Runtime (ms)', cvt=s2ms)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main():
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
assert class_names[5] == 'truck'
assert class_names[2] == 'car'
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
in_ccf = pickle.load(open(join(opts.in_dir, 'results_ccf.pkl'), 'rb'))
results_ccf = []
i_start = 0
for sid, seq in enumerate(tqdm(seqs)):
# if seq != '5ab2697b-6e3e-3454-a36a-aba2c6f27818':
# continue
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
for ii, img in enumerate(frame_list):
iid = img['id']
i_end = i_start
while i_end < len(in_ccf) and in_ccf[i_end]['image_id'] == iid:
i_end += 1
dets = in_ccf[i_start:i_end]
i_start = i_end
# dets = [r for r in in_ccf if r['image_id'] == img['id']]
bboxes = np.array([d['bbox'] for d in dets])
labels = np.array([d['category_id'] for d in dets])
scores = np.array([d['score'] for d in dets])
with_mask = len(dets) and 'segmentation' in dets[0]
if with_mask:
masks = np.array([d['segmentation'] for d in dets])
score_argsort = np.argsort(scores)[::-1]
bboxes = bboxes[score_argsort]
scores = scores[score_argsort]
labels = labels[score_argsort]
if with_mask:
masks = masks[score_argsort]
remove_idx = remove_car_for_pickup(bboxes, labels, scores)
bboxes = np.delete(bboxes, remove_idx, 0)
labels = np.delete(labels, remove_idx, 0)
scores = np.delete(scores, remove_idx, 0)
if with_mask:
masks = np.delete(masks, remove_idx, 0)
n = len(bboxes)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
if with_mask:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
bboxes_ltrb = bboxes.copy()
if len(bboxes_ltrb):
bboxes_ltrb[:, 2:] += bboxes_ltrb[:, :2]
if opts.overwrite or not isfile(vis_path):
vis_det(I,
bboxes_ltrb,
labels,
class_names,
masks if with_mask else None,
scores,
out_scale=opts.vis_scale,
out_file=vis_path)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps 30')
def main():
opts = parse_args()
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
print('Merging results')
results_ccf = []
in_time = 0
miss = 0
shifts = 0
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results, timestamps, input_fidx, _ = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb'))
tidx_p1 = 0
for i, img in enumerate(frame_list):
# pred, gt association by time
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = mmcv.imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
t = (i + 1)/opts.fps
while tidx_p1 < len(timestamps) and timestamps[tidx_p1] <= t:
tidx_p1 += 1
if tidx_p1 == 0:
# no output
miss += 1
if vis_out:
if opts.vis_scale != 1:
I = mmcv.imrescale(I, opts.vis_scale, interpolation='bilinear')
mmcv.imwrite(I, vis_path)
continue
tidx = tidx_p1 - 1
result = results[tidx]
ifidx = input_fidx[tidx]
in_time += int(i == ifidx)
shifts += i - ifidx
bboxes = result['bboxes']
scores = result['scores']
labels = result['labels']
masks = result['masks'] if 'masks' in result else None
tracks = result['tracks'] if 'tracks' in result else None
if vis_out:
if opts.overwrite or not isfile(vis_path):
if tracks is None:
vis_det(
I, bboxes, labels,
class_names, masks, scores,
out_scale=opts.vis_scale,
out_file=vis_path,
score_th=0,
)
else:
vis_track(
I, bboxes, tracks, labels,
class_names, masks, scores,
out_scale=opts.vis_scale,
out_file=vis_path,
score_th=0,
)
# convert to coco fmt
bboxes_ltwh = bboxes.copy()
if len(bboxes_ltwh):
bboxes_ltwh[:, 2:] -= bboxes_ltwh[:, :2]
bboxes_ltwh = bboxes_ltwh.tolist()
for j in range(len(bboxes_ltwh)):
result_dict = {
'image_id': img['id'],
'bbox': bboxes_ltwh[j],
'score': scores[j],
'category_id': labels[j],
}
if masks is not None:
result_dict['segmentation'] = masks[j]
results_ccf.append(result_dict)
out_path = join(opts.result_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
out_path = join(opts.result_dir, 'time_extra.txt')
if opts.overwrite or not isfile(out_path):
np.savetxt(out_path, [miss, in_time, shifts], fmt='%d')
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.result_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
def main():
opts = parse_args()
assert opts.forecast_before_assoc, "Not implemented"
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
results_ccf = []
in_time = 0
miss = 0
shifts = 0
given_tracks = opts.assoc == 'given'
assert not given_tracks, "Not implemented"
t_assoc = []
t_forecast = []
# tt = []
with torch.no_grad():
kf_F = torch.eye(8)
# kf_F[3, 7] = 1
kf_Q = torch.eye(8)
kf_R = 10 * torch.eye(4)
kf_P_init = 100 * torch.eye(8).unsqueeze(0)
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results = pickle.load(open(join(opts.in_dir, seq + '.pkl'), 'rb'))
# use raw results when possible in case we change class subset during evaluation
results_raw = results.get('results_raw', None)
if results_raw is None:
results_parsed = results['results_parsed']
timestamps = results['timestamps']
input_fidx = results['input_fidx']
# t1 -> det1, t2 -> det2, interpolate at t3 (t3 is the current time)
det_latest_p1 = 0 # latest detection index + 1
det_t2 = None # detection index at t2
kf_x = torch.empty((0, 8, 1))
kf_P = torch.empty((0, 8, 8))
n_matched12 = 0
if not given_tracks:
tkidx = 0 # track starting index
for ii, img in enumerate(frame_list):
# pred, gt association by time
t = (ii - opts.eta) / opts.fps
while det_latest_p1 < len(
timestamps) and timestamps[det_latest_p1] <= t:
det_latest_p1 += 1
if det_latest_p1 == 0:
# no detection output
miss += 1
bboxes_t3, scores, labels, tracks = [], [], [], []
else:
det_latest = det_latest_p1 - 1
ifidx = input_fidx[det_latest]
in_time += int(ii == ifidx)
shifts += ii - ifidx
if det_latest != det_t2:
# new detection
# we can now throw away old result (t1)
# the old one is kept for forecasting purpose
if len(kf_x) and opts.forecast_before_assoc:
dt = ifidx - input_fidx[det_t2]
dt = int(
dt
) # convert from numpy to basic python format
w_img, h_img = img['width'], img['height']
kf_F = make_F(kf_F, dt)
kf_Q = make_Q(kf_Q, dt)
kf_x, kf_P = batch_kf_predict(
kf_F, kf_x, kf_P, kf_Q)
bboxes_f = x2bbox(kf_x)
det_t2 = det_latest
if results_raw is None:
result_t2 = results_parsed[det_t2]
bboxes_t2, scores_t2, labels_t2, _ = result_t2[:4]
if not given_tracks and len(result_t2) > 4:
tracks_t2 = np.asarray(result_t2[4])
else:
tracks_t2 = np.empty((0, ), np.uint32)
else:
result_t2 = results_raw[det_t2]
bboxes_t2, scores_t2, labels_t2, _ = \
parse_det_result(result_t2, coco_mapping, n_class)
t1 = perf_counter()
n = len(bboxes_t2)
if n:
if not given_tracks:
# put high scores det first for better iou matching
score_argsort = np.argsort(scores_t2)[::-1]
bboxes_t2 = bboxes_t2[score_argsort]
scores_t2 = scores_t2[score_argsort]
labels_t2 = labels_t2[score_argsort]
ltrb2ltwh_(bboxes_t2)
if given_tracks:
raise NotImplementedError
order1, order2, n_matched12 = track_based_shuffle(
tracks, tracks_t2, no_unmatched1=True)
tracks = tracks[order2]
else:
updated = False
if len(kf_x):
order1, order2, n_matched12, tracks, tkidx = iou_assoc(
bboxes_f,
labels,
tracks,
tkidx,
bboxes_t2,
labels_t2,
opts.match_iou_th,
no_unmatched1=True,
)
if n_matched12:
kf_x = kf_x[order1]
kf_P = kf_P[order1]
kf_x, kf_P = batch_kf_update(
bbox2z(bboxes_t2[
order2[:n_matched12]]),
kf_x,
kf_P,
kf_R,
)
kf_x_new = bbox2x(
bboxes_t2[order2[n_matched12:]])
n_unmatched2 = len(
bboxes_t2) - n_matched12
kf_P_new = kf_P_init.expand(
n_unmatched2, -1, -1)
kf_x = torch.cat((kf_x, kf_x_new))
kf_P = torch.cat((kf_P, kf_P_new))
labels = labels_t2[order2]
scores = scores_t2[order2]
updated = True
if not updated:
# start from scratch
kf_x = bbox2x(bboxes_t2)
kf_P = kf_P_init.expand(
len(bboxes_t2), -1, -1)
labels = labels_t2
scores = scores_t2
if not given_tracks:
tracks = np.arange(tkidx,
tkidx + n,
dtype=np.uint32)
tkidx += n
t2 = perf_counter()
t_assoc.append(t2 - t1)
t3 = perf_counter()
if len(kf_x):
dt = ii - ifidx
w_img, h_img = img['width'], img['height']
# PyTorch small matrix multiplication is slow
# use numpy instead
# kf_F = make_F(kf_F, dt)
# bboxes_t3 = x2bbox(batch_kf_predict_only(kf_F, kf_x))
# t5 = perf_counter()
kf_x_np = kf_x[:, :, 0].numpy()
# t6 = perf_counter()
bboxes_t3 = kf_x_np[:n_matched12, :
4] + dt * kf_x_np[:n_matched12, 4:]
if n_matched12 < len(kf_x):
bboxes_t3 = np.concatenate(
(bboxes_t3, kf_x_np[n_matched12:, :4]))
bboxes_t3, keep = extrap_clean_up(bboxes_t3,
w_img,
h_img,
lt=True)
# torch_keep = torch.from_numpy(keep) # boolean mask
# kf_x = kf_x[torch_keep]
# kf_P = kf_P[torch_keep]
# labels = labels[keep]
# scores = scores[keep]
# tracks = tracks[keep]
labels_t3 = labels[keep]
scores_t3 = scores[keep]
tracks_t3 = tracks[keep]
# tt.append(t6 - t5)
t4 = perf_counter()
t_forecast.append(t4 - t3)
n = len(bboxes_t3)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes_t3[i],
'score': scores_t3[i],
'category_id': labels_t3[i],
}
results_ccf.append(result_dict)
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq,
img['name'][:-3] + 'jpg')
bboxes = bboxes_t3.copy()
if n:
ltwh2ltrb_(bboxes)
if opts.overwrite or not isfile(vis_path):
vis_track(
I,
bboxes,
tracks,
labels,
class_names,
None,
scores,
out_scale=opts.vis_scale,
out_file=vis_path,
)
# pr.disable()
# pr.dump_stats('pps_iou_lin_extrap.prof')
# pickle.dump([t_assoc, t_forecast], open('pps_iou_lin_forecast_time', 'wb'))
s2ms = lambda x: 1e3 * x
if len(t_assoc):
print_stats(t_assoc, "RT association (ms)", cvt=s2ms)
if len(t_forecast):
print_stats(t_forecast, "RT forecasting (ms)", cvt=s2ms)
# if len(tt):
# print_stats(tt, "RT forecasting2 (ms)", cvt=s2ms)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
