def main():
opts = parse_args()
mkdir2(opts.out_dir)
print('Reading detection images')
det_img_files = sorted(glob(join(opts.det_img_dir, '*.jpg')))
det_imgs = []
for path in det_img_files:
det_imgs.append(imread(path))
print('Detection Test at Scale 0.5')
rt_det1 = det_test(opts, 0.5, det_imgs)
print(f'Mean runtime (ms): {1e3*rt_det1:.3g}')
print('Detection Test at Scale 1')
rt_det2 = det_test(opts, 1, det_imgs)
print(f'Mean runtime (ms): {1e3*rt_det2:.3g}')
det_imgs = None
print('Classification Test')
rt_inf, rt_train = classify_test(opts)
print(f'Inference Mean runtime (ms): {1e3*rt_inf:.3g}')
print(f'Training Mean runtime (ms): {1e3*rt_train:.3g}')
out_path = join(opts.out_dir, 'platform_bench.txt')
if opts.overwrite or not isfile(out_path):
with open(out_path, 'w', newline='\n') as f:
writer = csv.writer(f)
writer.writerow([rt_det1, rt_det2, rt_inf, rt_train])
print(f'Saved results to "{out_path}"')
def imwrite(img, path, method='PIL', auto_mkdir=True):
if method == 'PIL':
if auto_mkdir:
mkdir2(dirname(path))
Image.fromarray(img).save(path)
else:
mmcv.imwrite(img, path, auto_mkdir=auto_mkdir)
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
config = mmcv.Config.fromfile(opts.config)
if opts.in_scale is not None:
config.data.test.img_scale = opts.in_scale
# mainly for SSD
config.data.test.resize_keep_ratio = True
if opts.no_mask:
if 'mask_head' in config.model:
config.model['mask_head'] = None
model = init_detector(config, opts.weights)
model.eval()
img_list = sorted(glob(join(opts.in_dir, '*.jpg')))
img_list = img_list[::opts.image_stride]
for img_path in tqdm(img_list):
img_name = basename(img_path)
img = mmcv.imread(img_path)
result = inference_detector(model, img)
out_path = join(opts.out_dir, img_name[:-3] + 'pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(result, open(out_path, 'wb'))
def main():
opts = parse_args()
if not opts.overwrite and isfile(opts.out_path):
return
mkdir2(dirname(opts.out_path))
time_info = pickle.load(open(opts.time_info, 'rb'))
if opts.method_type == 'det':
rt_samples = time_info['runtime_all']
rt_dist = {'type': 'empirical', 'samples': rt_samples}
else:
raise ValueError(f'Unknown method type "{opts.method_type}"')
pickle.dump(rt_dist, open(opts.out_path, 'wb'))
def main():
opts = parse_args()
if not opts.overwrite and isfile(opts.out_path):
return
mkdir2(dirname(opts.out_path))
class_subset = avhd_subset
cats = []
class_names = []
for i, c in enumerate(class_subset):
cats.append({'id': i, 'name': coco_classes[c]})
class_names.append(coco_classes[c])
n_coco = len(coco_classes)
coco_mapping = n_coco * [n_coco]
for i, c in enumerate(class_subset):
coco_mapping[c] = i
frames = sorted(
[item.name for item in scandir(opts.img_folder) if item.is_file()])
if opts.start is not None:
frames = frames[opts.start:]
img = Image.open(join(opts.img_folder, frames[0]))
img_width, img_height = img.size
seqs = [opts.prefix]
seq_dirs = [opts.prefix]
imgs = []
for i, name in enumerate(frames):
imgs.append({
'id': i,
'sid': 0,
'fid': i,
'name': name,
'width': img_width,
'height': img_height,
})
dataset = {
'categories': cats,
'images': imgs,
'sequences': seqs,
'seq_dirs': seq_dirs,
'coco_subset': class_subset,
'coco_mapping': coco_mapping,
}
json.dump(dataset, open(opts.out_path, 'w'))
def main():
opts = parse_args()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
eval_summary = eval_ccf(db, opts.result_path, None, opts.per_class)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, opts.result_path, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
def main():
opts = parse_args()
if not opts.overwrite and isfile(opts.out_path):
return
mkdir2(dirname(opts.out_path))
time_info = pickle.load(open(opts.time_info, 'rb'))
if opts.method_type == 'det':
rt_samples = time_info['runtime_all']
rt_dist = {'type': 'empirical', 'samples': rt_samples}
elif opts.method_type == 'dat':
which_cfg_all = np.array(time_info['which_cfg_all'])
rt_samples = np.array(time_info['runtime_all'])[which_cfg_all == 1]
rt_samples = rt_samples[rt_samples >
0.002] # ignore zero detection trivial cases
rt_dist = {'type': 'empirical', 'samples': rt_samples.tolist()}
else:
raise ValueError(f'Unknown method type "{opts.method_type}"')
pickle.dump(rt_dist, open(opts.out_path, 'wb'))
def main():
opts = parse_args()
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
seq = 'f1008c18-e76e-3c24-adcc-da9858fac145'
sid = seqs.index(seq)
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb'))
results_parsed = results['results_parsed']
img = frame_list[153]
result = results_parsed[15]
bboxes, scores, labels, masks = result[:4]
idx = [16]
bboxes = bboxes[idx]
scores = scores[idx]
labels = labels[idx]
masks = masks[idx]
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path).copy()
vis_path = join(opts.vis_dir, 'teaser.jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(
I,
bboxes,
labels,
class_names,
masks,
None,
out_scale=opts.vis_scale,
out_file=vis_path,
)
def main():
opts = parse_args()
out_dir = mkdir2(opts.out_dir) if opts.out_dir else opts.result_dir
db = COCO(opts.annot_path)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
n_concurrent = []
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb'))
# use raw results when possible in case we change class subset during evaluation
if opts.type == 'det':
timestamps = results['timestamps']
input_fidx = results['input_fidx']
else:
det1_timestamps = results['det1_timestamps']
det2_timestamps = results['det2_timestamps']
det1_input_fidx = results['det1_input_fidx']
det2_input_fidx = results['det2_input_fidx']
timestamps = np.concatenate((det1_timestamps, det2_timestamps))
input_fidx = np.concatenate((det1_input_fidx, det2_input_fidx))
t_start = np.asarray(input_fidx) / opts.fps
t_end = np.asarray(timestamps)
t_all = np.concatenate((t_start, t_end))
order = np.argsort(t_all)
n_output = len(t_start)
n_current = 0
max_current = 0
for i in order:
if i < n_output:
# start
n_current += 1
max_current = max(max_current, n_current)
else:
# end
n_current -= 1
n_concurrent.append(max_current)
print(f'Max number of concurrent jobs {max(n_concurrent)}')
out_path = join(out_dir, 'n_concurrent.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(n_concurrent, open(out_path, 'wb'))
def main():
assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing
opts = parse_args()
assert (not opts.mask_timing) or (opts.mask_timing and not opts.no_mask)
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
# warm up the GPU
img = db.imgs[0]
w_img, h_img = img['width'], img['height']
_ = inference_detector(model, np.zeros((h_img, w_img, 3), np.uint8))
torch.cuda.synchronize()
runtime_all = []
n_processed = 0
n_total = 0
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
n_frame = len(frames)
n_total += n_frame
timestamps = []
results_raw = []
results_parsed = []
input_fidx = []
runtime = []
last_fidx = None
if not opts.dynamic_schedule:
stride_cnt = 0
t_total = n_frame/opts.fps
t_start = perf_counter()
while 1:
t1 = perf_counter()
t_elapsed = t1 - t_start
if t_elapsed >= t_total:
break
# identify latest available frame
fidx_continous = t_elapsed*opts.fps
fidx = int(np.floor(fidx_continous))
if fidx == last_fidx:
continue
last_fidx = fidx
if opts.dynamic_schedule:
fidx_remainder = fidx_continous - fidx
if fidx_remainder > 0.5:
continue
else:
if stride_cnt % opts.det_stride == 0:
stride_cnt = 1
else:
stride_cnt += 1
continue
frame = frames[fidx]
result = inference_detector(model, frame, gpu_pre=not opts.cpu_pre, decode_mask=not opts.mask_timing)
if opts.mask_timing:
mask_encoded = result[2]
result = result[:2]
bboxes, scores, labels, _, sel = \
parse_det_result(result, coco_mapping, n_class, return_sel=True)
else:
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
torch.cuda.synchronize()
t2 = perf_counter()
t_elapsed = t2 - t_start
if t_elapsed >= t_total:
break
if opts.mask_timing:
masks = decode_mask(mask_encoded, sel)
timestamps.append(t_elapsed)
results_raw.append(result)
results_parsed.append((bboxes, scores, labels, masks))
input_fidx.append(fidx)
runtime.append(t2 - t1)
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'results_raw': results_raw,
'results_parsed': results_parsed,
'timestamps': timestamps,
'input_fidx': input_fidx,
'runtime': runtime,
}, open(out_path, 'wb'))
runtime_all += runtime
n_processed += len(results_raw)
runtime_all_np = np.asarray(runtime_all)
n_small_runtime = (runtime_all_np < 1.0/opts.fps).sum()
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'runtime_all': runtime_all,
'n_processed': n_processed,
'n_total': n_total,
'n_small_runtime': n_small_runtime,
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3*x
print(f'{n_processed}/{n_total} frames processed')
print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms)
print(f'Runtime smaller than unit time interval: '
f'{n_small_runtime}/{n_processed} '
f'({100.0*n_small_runtime/n_processed:.4g}%)')
from os.path import join, basename, isfile
from glob import glob
import numpy as np
from html4vision import Col, imagetable
import sys; sys.path.insert(0, '..'); sys.path.insert(0, '.')
from util import mkdir2
data_dir = '/data2/mengtial'
data_root = join(data_dir, 'ArgoVerse1.1/tracking')
split = 'val'
annot_file = join(data_root, 'coco_fmt/htc_dconv2_ms_s1.0_mask_' + split + '.json')
out_dir = mkdir2(join(data_dir, 'Exp/ArgoVerse1.1/vis'))
out_name = 'mask.html'
title = 'Mask'
metric = 'Mask AP'
link_video = False
n_show = 90
n_consec = 15
align = False # align to the stride in each sequence
stride = 30
random.seed(0)
names = [
'Offline Accurate',
'Real-time Accurate',
'Real-time Ours',
def main():
line_color_np = np.array(line_color, dtype=np.uint8).reshape((1, 1, 3))
seq_dir_out = mkdir2(join(out_dir, seq))
frame_list = [
item.name for item in scandir(join(dirs[0], seq))
if item.is_file() and item.name.endswith('.jpg')
]
frame_list = sorted(frame_list)
n_method = len(text)
fidx = 0
j_start = 0
for i in tqdm(range(n_method)):
for j in range(j_start, t_duration):
fidx += 1
out_path = join(seq_dir_out, '%06d.jpg' % fidx)
if not overwrite and isfile(out_path):
continue
img_A = Image.open(join(dirs[i], seq, frame_list[j + clip_start]))
# cropping
img_A = np.array(img_A)
img_A = img_A[crop_top:-crop_bottom]
progress_text = ease_in_out(
(j - j_start) /
t_text_appear) if j - j_start < t_text_appear else 1
# render text region
img_A[text_region[1]:text_region[3] + 1, text_region[0]:text_region[2] + 1] = \
np.round((1 + progress_text*(text_region_alpha - 1))*img_A[text_region[1]:text_region[3] + 1, text_region[0]:text_region[2] + 1]).astype(np.uint8)
img_A_with_text = Image.fromarray(img_A)
# using TrueType supported in PIL
draw = ImageDraw.Draw(img_A_with_text)
draw.text(
(text_xy[0], text_xy[1]),
text[i][0],
(*text_color, 255), # RGBA
font=font,
)
draw.text(
(text_xy[0], text_xy[1] + text_line_sep),
text[i][1],
(*text_color, 255), # RGBA
font=font,
)
img_A_with_text = np.array(img_A_with_text)
img_A = np.round(progress_text * img_A_with_text +
(1 - progress_text) * img_A).astype(np.uint8)
img_A = Image.fromarray(img_A)
if i < n_method - 1 and j >= t_duration - t_transition:
# transition period
j_start = t_transition
j2 = j + t_transition - t_duration
img_B = Image.open(
join(dirs[i + 1], seq, frame_list[j2 + clip_start]))
img_B = np.array(img_B)
img_B = img_B[crop_top:-crop_bottom]
h, w, _ = img_B.shape
split_pos = split_anime_accelerate(j2 / t_transition, w,
line_width)
split_pos = int(round(split_pos))
line_start = split_pos - (line_width - 1) // 2
line_end = split_pos + line_width // 2 # inclusive
if split_pos <= 0:
img = img_B
else:
img = np.array(img_A)
img_B = np.asarray(img_B)
img[:, split_pos:] = img_B[:, split_pos:]
if line_start < w and line_end >= 0:
# line is visible
line_start = max(0, line_start)
line_end = min(w, line_end)
img[:, line_start:line_end] = line_color_np
out_img = Image.fromarray(img)
else:
out_img = img_A
out_img.save(out_path)
if make_video:
out_path = seq_dir_out + '.mp4'
if overwrite or not isfile(out_path):
print('Making the video')
class Dummy():
fps
opts = Dummy()
opts.fps = fps
make_video_func((seq_dir_out, opts))
else:
print(
f'python vis/make_videos_numbered.py "{opts.out_dir}" --fps {opts.fps}'
)
def main():
opts = parse_args()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
if opts.cached_res:
cache_in_ccf = '_ccf' in basename(opts.cached_res)
if cache_in_ccf:
# speed up based on the assumption of sequential storage
cache_end_idx = 0
cached_res = pickle.load(open(opts.cached_res, 'rb'))
else:
assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing
model = init_detector(opts)
np.random.seed(opts.seed)
runtime = pickle.load(open(opts.runtime, 'rb'))
runtime_dist = dist_from_dict(runtime, opts.perf_factor)
runtime_all = []
n_processed = 0
n_total = 0
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
n_frame = len(frame_list)
n_total += n_frame
if not opts.cached_res:
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
timestamps = []
results_parsed = []
input_fidx = []
runtime = []
last_fidx = None
if opts.cached_res and cache_in_ccf:
results_raw = None
else:
results_raw = []
t_total = n_frame/opts.fps
t_elapsed = 0
if opts.dynamic_schedule:
mean_rtf = runtime_dist.mean()*opts.fps
else:
stride_cnt = 0
while 1:
if t_elapsed >= t_total:
break
# identify latest available frame
fidx_continous = t_elapsed*opts.fps
fidx = int(np.floor(fidx_continous))
if fidx == last_fidx:
# algorithm is fast and has some idle time
fidx += 1
if fidx == n_frame:
break
t_elapsed = fidx/opts.fps
last_fidx = fidx
if opts.dynamic_schedule:
if mean_rtf > 1:
# when runtime <= 1, it should always process every frame
fidx_remainder = fidx_continous - fidx
if mean_rtf < np.floor(fidx_remainder + mean_rtf):
# wait till next frame
continue
else:
if stride_cnt % opts.det_stride == 0:
stride_cnt = 1
else:
stride_cnt += 1
continue
if opts.cached_res:
img = frame_list[fidx]
if cache_in_ccf:
cache_end_idx, bboxes, scores, labels, masks = \
result_from_ccf(cached_res, img['id'], cache_end_idx)
ltwh2ltrb_(bboxes)
else:
result = cached_res[img['id']]
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
else:
frame = frames[fidx]
result = inference_detector(model, frame)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
rt_this = runtime_dist.draw()
t_elapsed += rt_this
if t_elapsed >= t_total:
break
timestamps.append(t_elapsed)
if results_raw is not None:
results_raw.append(result)
results_parsed.append((bboxes, scores, labels, masks))
input_fidx.append(fidx)
runtime.append(rt_this)
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
out_dict = {
'results_parsed': results_parsed,
'timestamps': timestamps,
'input_fidx': input_fidx,
'runtime': runtime,
}
if results_raw is not None:
out_dict['results_raw'] = results_raw
pickle.dump(out_dict, open(out_path, 'wb'))
runtime_all += runtime
n_processed += len(results_parsed)
runtime_all_np = np.array(runtime_all)
n_small_runtime = (runtime_all_np < 1.0/opts.fps).sum()
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'runtime_all': runtime_all,
'n_processed': n_processed,
'n_total': n_total,
'n_small_runtime': n_small_runtime,
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3*x
print(f'{n_processed}/{n_total} frames processed')
print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms)
print(f'Runtime smaller than unit time interval: '
f'{n_small_runtime}/{n_processed} '
f'({100.0*n_small_runtime/n_processed:.4g}%)')
def main():
opts = parse_args()
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
coco_subset = np.asarray(db.dataset['coco_subset'])
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
color_palette = [
class_palette[k] for k in coco_subset if k in class_palette
]
if opts.gt:
results_ccf = db.dataset['annotations']
else:
results_ccf = pickle.load(open(opts.result_path, 'rb'))
if opts.seq is not None:
if opts.seq.isdigit():
idx = int(opts.seq)
else:
idx = seqs.index(opts.seq)
seqs = [seqs[idx]]
else:
idx = None
# font_path = r'C:\Windows\Fonts\AdobeGurmukhi-Regular.otf'
# font = ImageFont.truetype(font_path, size=40)
font = None
for sid, seq in enumerate(tqdm(seqs)):
if idx is not None:
sid = idx
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
for ii, img in enumerate(frame_list):
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
dets = [r for r in results_ccf if r['image_id'] == img['id']]
bboxes = np.array([d['bbox'] for d in dets])
masks = None
if len(bboxes):
bboxes[:, 2:] += bboxes[:, :2]
if 'segmentation' in dets[0]:
masks = [d['segmentation'] for d in dets]
labels = np.array([d['category_id'] for d in dets])
if opts.gt:
iscrowd = np.array([d['iscrowd'] for d in dets])
scores = None
else:
iscrowd = None
scores = np.array([d['score'] for d in dets])
vis_path = join(opts.vis_dir, seq,
'%06d.jpg' % (ii + 1)) # img['name'][:-3] + 'jpg'
if opts.overwrite or not isfile(vis_path):
vis_obj_fancy(
img_path,
bboxes,
labels,
class_names,
color_palette,
masks=masks,
show_label=False,
scores=scores,
show_score=False,
iscrowd=iscrowd,
score_th=opts.score_th,
out_scale=opts.vis_scale,
out_file=vis_path,
font=font,
)
if opts.make_video:
seq_dir_out = join(opts.vis_dir, seq)
out_path = seq_dir_out + '.mp4'
if opts.overwrite or not isfile(out_path):
make_video((seq_dir_out, opts))
if not opts.make_video:
print(
f'python vis/make_videos_numbered.py "{opts.vis_dir}" --fps {opts.fps}'
)
import numpy as np
from html4vision import Col, imagetable
import sys
sys.path.insert(0, '..')
sys.path.insert(0, '.')
from util import mkdir2
data_dir = '/data2/mengtial'
data_root = join(data_dir, 'ArgoVerse1.1/tracking')
split = 'val'
annot_file = join(data_root, 'coco_fmt/' + split + '.json')
out_dir = mkdir2(join(data_dir, 'Exp', 'ArgoVerse1.1', 'vis-inf'))
out_name = 'inf_multi1.html'
title = 'Infinite GPU multi-detector 1'
metric = 'AP'
link_video = True
n_show = 90
n_consec = 30
align = True # align to the stride in each sequence
stride = 30
random.seed(0)
names = [
'Ground Truth',
'Offline MRCNN R50 s0.5',
'Offline MRCNN R50 s1.0',
'Multi-detector Forecasting',
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
if opts.weights_base is not None:
# for distillation purpose
load_checkpoint(model, opts.weights_base)
if opts.cpu_pre:
img_transform = ImageTransform(
size_divisor=model.cfg.data.test.size_divisor,
**model.cfg.img_norm_cfg)
else:
img_transform = ImageTransformGPU(
size_divisor=model.cfg.data.test.size_divisor,
**model.cfg.img_norm_cfg)
device = next(model.parameters()).device # model device
n_history = model.cfg.data.train.n_history if opts.n_history is None else opts.n_history
n_future = model.cfg.data.train.n_future if opts.n_future is None else opts.n_future
results_ccf = [] # instance based
runtime_all = []
for sid, seq in enumerate(tqdm(seqs)):
# print(seq)
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
n_frame = len(frame_list)
# load all frames in advance
frames = []
for img in frame_list:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames.append(imread(img_path))
with torch.no_grad():
preprocessed = []
for i in range(n_history):
data = _prepare_data(frames[i], img_transform, model.cfg,
device)
preprocessed.append(data)
for ii in range(n_history, n_frame - n_future):
# target frame
iid = frame_list[ii + n_future]['id']
img_name = frame_list[ii + n_future]['name']
I = frames[ii + n_future]
t_start = perf_counter()
# input frame
data = _prepare_data(frames[ii], img_transform, model.cfg,
device)
# if n_history == 0:
# data_merge = data
# # print(data['img'])
# # print(data['img'][0].shape)
# # print(data['img'][0][0][0][300][300:305])
# # import sys
# # sys.exit()
# else:
preprocessed.append(data)
# print(preprocessed[0]['img'][0].data_ptr())
# print(preprocessed[2]['img'][0].data_ptr())
# print(torch.all(preprocessed[0]['img'][0] == preprocessed[2]['img'][0]))
imgs = [d['img'][0] for d in preprocessed]
imgs = torch.cat(imgs, 0)
imgs = imgs.unsqueeze(0)
data_merge = {
'img': [imgs],
'img_meta': data['img_meta'],
}
# print(data_merge['img'][0][0][2][0][300][300:305])
# import sys
# sys.exit()
result = model(return_loss=False,
rescale=True,
numpy_res=True,
**data_merge)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
# if ii == 2:
# print(ii, scores)
# import sys
# sys.exit()
# if n_history != 0:
del preprocessed[0]
t_end = perf_counter()
runtime_all.append(t_end - t_start)
if vis_out:
vis_path = join(opts.vis_dir, seq, img_name[:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path)
# convert to coco fmt
n = len(bboxes)
if n:
bboxes[:, 2:] -= bboxes[:, :2]
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'runtime_all': runtime_all,
'n_total': len(runtime_all),
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3 * x
print_stats(runtime_all, 'Runtime (ms)', cvt=s2ms)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mp.set_start_method('spawn')
frame_recv, frame_send = mp.Pipe(False)
det_res_recv, det_res_send = mp.Pipe(False)
det_proc = mp.Process(target=det_process,
args=(opts, frame_recv, det_res_send))
det_proc.start()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
runtime_all = []
n_processed = 0
n_total = 0
t_send_frame_all = []
t_recv_res_all = []
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
frame_list = [
join(opts.data_root, seq_dirs[sid], img['name'])
for img in frame_list
]
n_frame = len(frame_list)
n_total += n_frame
timestamps = []
results_raw = []
results_parsed = []
input_fidx = []
runtime = []
last_fidx = None
stride_cnt = 0
# let detector process to read all the frames
frame_send.send(frame_list)
init_error = det_res_recv.recv() # wait till the detector is ready
if init_error is not None:
raise init_error
t_total = n_frame / opts.fps
t_start = perf_counter()
while 1:
t1 = perf_counter()
t_elapsed = t1 - t_start
if t_elapsed >= t_total:
break
# identify latest available frame
fidx = int(np.floor(t_elapsed * opts.fps))
# t_elapsed/t_total *n_frame
# = t_elapsed*opts.fps
if fidx == last_fidx:
continue
last_fidx = fidx
if stride_cnt % opts.det_stride == 0:
stride_cnt = 1
else:
stride_cnt += 1
continue
t_start_frame = perf_counter()
frame_send.send((fidx, t_start_frame))
result = det_res_recv.recv() # wait
if isinstance(result, Exception):
raise result
result, t_send_frame, t_start_res = result
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
t2 = perf_counter()
t_send_frame_all.append(t_send_frame)
t_recv_res_all.append(t2 - t_start_res)
t_elapsed = t2 - t_start
if t_elapsed >= t_total:
break
timestamps.append(t_elapsed)
results_raw.append(result)
results_parsed.append((bboxes, scores, labels, masks))
input_fidx.append(fidx)
runtime.append(t2 - t1)
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(
{
'results_raw': results_raw,
'results_parsed': results_parsed,
'timestamps': timestamps,
'input_fidx': input_fidx,
'runtime': runtime,
}, open(out_path, 'wb'))
runtime_all += runtime
n_processed += len(results_raw)
# terminates the child process
frame_send.send(None)
runtime_all_np = np.asarray(runtime_all)
n_small_runtime = (runtime_all_np < 1.0 / opts.fps).sum()
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(
{
'runtime_all': runtime_all,
'n_processed': n_processed,
'n_total': n_total,
'n_small_runtime': n_small_runtime,
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3 * x
print(f'{n_processed}/{n_total} frames processed')
print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms)
print(f'Runtime smaller than unit time interval: '
f'{n_small_runtime}/{n_processed} '
f'({100.0*n_small_runtime/n_processed:.4g}%)')
print_stats(t_send_frame_all,
'Time spent on sending the frame (ms)',
cvt=s2ms)
print_stats(t_recv_res_all,
'Time spent on receiving the result (ms)',
cvt=s2ms)
def main():
opts = parse_args()
out_dir = mkdir2(opts.out_dir) if opts.out_dir else opts.result_dir
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
results_ccf = []
in_time = 0
miss = 0
mismatch = 0
print('Pairing the output with the ground truth')
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb'))
# use raw results when possible in case we change class subset during evaluation
if opts.use_parsed:
results_parsed = results['results_parsed']
else:
results_raw = results.get('results_raw', None)
if results_raw is None:
results_parsed = results['results_parsed']
timestamps = results['timestamps']
input_fidx = results['input_fidx']
tidx_p1 = 0
for ii, img in enumerate(frame_list):
# pred, gt association by time
t = (ii - opts.eta) / opts.fps
while tidx_p1 < len(timestamps) and timestamps[tidx_p1] <= t:
tidx_p1 += 1
if tidx_p1 == 0:
# no output
miss += 1
bboxes, scores, labels = [], [], []
masks, tracks = None, None
else:
tidx = tidx_p1 - 1
ifidx = input_fidx[tidx]
in_time += int(ii == ifidx)
mismatch += ii - ifidx
if opts.use_parsed or results_raw is None:
result = results_parsed[tidx]
bboxes, scores, labels, masks = result[:4]
if len(result) > 4:
tracks = result[4]
else:
tracks = None
else:
result = results_raw[tidx]
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
tracks = None
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
if tracks is None:
vis_det(
I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path,
)
else:
vis_track(
I,
bboxes,
tracks,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path,
)
# convert to coco fmt
n = len(bboxes)
if n:
bboxes_ltwh = ltrb2ltwh(bboxes)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes_ltwh[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
out_path = join(out_dir, 'eval_assoc.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'miss': miss,
'in_time': in_time,
'mismatch': mismatch,
}, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
import numpy as np import sys sys.path.insert(0, '..') sys.path.insert(0, '.') from util import mkdir2 from det import vis_det, imread from track import vis_track data_dir = '/data2/mengtial' data_root = join(data_dir, 'ArgoVerse1.1/tracking') split = 'val' annot_file = join(data_root, 'coco_fmt/' + split + '_track.json') show_track = True out_dir = mkdir2(join(data_root, 'coco_fmt/vis/' + split + '_track')) out_scale = 1 ## # not assuming consecutive storage imgs and annots (obj, frame, seq) dataset = json.load(open(annot_file)) cats = dataset['categories'] imgs = dataset['images'] annots = dataset['annotations'] seqs = dataset['sequences'] seq_dirs = dataset['seq_dirs'] class_names = [c['name'] for c in cats] last_sid = None
def main():
opts = parse_args()
# caches = [
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/retina50_s0.2/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.5/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.75/val/results_ccf.pkl'),
# ]
# # should be sorted
# rtfs = [1, 2, 3]
# caches = [
# join(opts.exp_dir, 'ArgoVerse1.1/output/mrcnn50_nm_th0_s0.75/val/results_ccf.pkl'),
# join(opts.exp_dir, 'ArgoVerse1.1/output/mrcnn50_nm_th0_s1.0/val/results_ccf.pkl'),
# ]
# # should be sorted
# rtfs = [3, 5]
caches = [
join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.75/val/results_ccf.pkl'),
join(opts.exp_dir, 'ArgoVerse1.1/output/cmrcnn101_nm_s1.0/val/results_ccf.pkl'),
]
# should be sorted
rtfs = [3, 5]
n_method = len(caches)
max_history = max(rtfs)
cache_ccfs = [
pickle.load(open(path, 'rb'))
for path in caches
]
cache_end_idx = n_method*[0]
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
results_ccf = []
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
w_img, h_img = db.imgs[0]['width'], db.imgs[0]['height']
bf = BboxFilterEx(
w_img, h_img,
forecast=opts.forecast,
forecast_before_assoc=opts.forecast_before_assoc,
)
# backward in time
# cur, cur - 1, ..., cur - max_history
result_buffer = (max_history + 1)*[None]
result_buffer_marker = (max_history + 1)*[None]
min_rtf = min(rtfs)
for ii, img in enumerate(frame_list):
# fetch results
for i in range(n_method):
ifidx = ii - rtfs[i]
if ifidx < 0:
break
cache_end_idx[i], bboxes, scores, labels = \
result_from_ccf(cache_ccfs[i], frame_list[ifidx]['id'], cache_end_idx[i], mask=False)
result_buffer[rtfs[i]] = (bboxes, scores, labels)
result_buffer_marker[rtfs[i]] = i == 0
if ii < min_rtf:
# no method has finished yet
bboxes, scores, labels, tracks = [], [], [], []
else:
s = max_history
while s >= 0 and result_buffer[s] is None:
s -= 1
# if first result is one step ahead
t = ii - s
birth_tracks = True
kill_tracks = True
if s == max_history:
bf.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
bf_this = deepcopy(bf)
else:
bf_this = deepcopy(bf)
bf_this.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
while 1:
# find next non-empty result
s -= 1
while s >= 0 and result_buffer[s] is None:
s -= 1
if s < 0:
break
if result_buffer_marker[s]:
birth_tracks = opts.fast_birth_tracks
kill_tracks = opts.fast_kill_tracks
else:
birth_tracks = True
kill_tracks = True
t = ii - s
bf_this.update(t, *result_buffer[s], None, birth_tracks, kill_tracks)
bboxes, scores, labels, _, tracks = \
bf_this.predict(ii)
# shift result buffer
result_buffer.pop(max_history)
result_buffer.insert(0, None)
result_buffer_marker.pop(max_history)
result_buffer_marker.insert(0, None)
n = len(bboxes)
for i in range(n):
result_dict = {
'image_id': img['id'],
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
# if masks is not None:
# result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
I = imread(img_path)
vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg')
bboxes_ltrb = ltwh2ltrb(bboxes) if n else []
if opts.overwrite or not isfile(vis_path):
vis_track(
I, bboxes_ltrb, tracks, labels,
class_names, None, scores,
out_scale=opts.vis_scale,
out_file=vis_path
)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
# CUDA runtime does not support the fork
mp.set_start_method('spawn', force=True)
# calculate the image size for shared memory allocation
if 'max_seq_len' in db.dataset:
max_seq_len = db.dataset['max_seq_len']
else:
seq_len = [
len([img for img in db.imgs.values() if img['sid'] == sid])
for sid in range(len(seqs))
]
max_seq_len = max(seq_len)
frame_buffer = mp.RawArray('B', max_seq_len * img_height * img_width * 3)
frames_a = np.ctypeslib.as_array(frame_buffer)
print(frames_a.dtype)
frames_b = np.frombuffer(frame_buffer, np.uint8)
print(frames_a.__array_interface__['data'][0])
print(frames_b.__array_interface__['data'][0])
frames = frames_a.reshape(-1, img_height, img_width, 3)
child_recv, parent_send = mp.Pipe(False)
parent_recv, child_send = mp.Pipe(False)
det_proc = mp.Process(target=det_process,
args=(opts, child_recv, child_send, frame_buffer))
det_proc.start()
runtime_all = []
n_processed = 0
n_total = 0
t_send_frame_all = 0
t_recv_res_all = 0
init_error = parent_recv.recv() # wait till the detector is ready
if isinstance(init_error, Exception):
raise init_error
for sid, seq in enumerate(tqdm(seqs)):
# if sid != 1:
# continue
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
# load all frames in advance
for i, img in enumerate(frame_list):
img_path = join(opts.data_root, seq_dirs[sid], img['name'])
frames[i] = mmcv.imread(img_path)
n_frame = len(frame_list)
n_total += n_frame
timestamps = []
results = []
input_fidx = []
runtime = []
last_fidx = None
stride_cnt = 0
t_total = n_frame / opts.fps
t_start = perf_counter()
while 1:
t1 = perf_counter()
t_elapsed = t1 - t_start
if t_elapsed >= t_total:
break
# identify latest available frame
fidx = int(np.floor(t_elapsed * opts.fps))
# t_elapsed/t_total *n_frame
# = t_elapsed*opts.fps
if fidx == last_fidx:
continue
last_fidx = fidx
if stride_cnt % opts.det_stride == 0:
stride_cnt = 1
else:
stride_cnt += 1
continue
t_start_frame = perf_counter()
parent_send.send((fidx, t_start_frame))
result = parent_recv.recv() # wait
if isinstance(result, Exception):
raise result
result, t_send_frame, t_start_res = result
t2 = perf_counter()
t_send_frame_all += t_send_frame
t_recv_res_all += t2 - t_start_res
# print('recv: %.3g elapsed: %.3g' % (t2 - t_start_res, t2 - t1))
t_elapsed = t2 - t_start
if t_elapsed >= t_total:
break
timestamps.append(t_elapsed)
results.append(result)
input_fidx.append(fidx)
runtime.append(t2 - t1)
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump((
results,
timestamps,
input_fidx,
runtime,
), open(out_path, 'wb'))
runtime_all += runtime
n_processed += len(results)
# break
# terminates the child process
parent_send.send(None)
runtime_all_np = np.array(runtime_all)
n_small_runtime = (runtime_all_np < 1.0 / opts.fps).sum()
out_path = join(opts.out_dir, 'time_all.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump((
runtime_all,
n_processed,
n_total,
n_small_runtime,
), open(out_path, 'wb'))
# convert to ms for display
runtime_all_np *= 1e3
print(f'{n_processed}/{n_total} frames processed')
print('Runtime (ms): mean: %g; std: %g; min: %g; max: %g' % (
runtime_all_np.mean(),
runtime_all_np.std(ddof=1),
runtime_all_np.min(),
runtime_all_np.max(),
))
print(f'Runtime smaller than unit time interval: '
f'{n_small_runtime}/{n_processed} '
f'({100.0*n_small_runtime/n_processed:.4g}%)')
print(
f'Time spent on sending the frame (ms): {1e3*t_send_frame_all/n_processed}'
)
print(
f'Time spent on receiving the result (ms): {1e3 *t_recv_res_all/n_processed}'
)
def main():
assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
n_class = len(db.cats)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
if 'KMOTS' in opts.data_root:
class_mapping = coco2kmots
class_names = kmots_classes
elif 'ArgoVerse' in opts.data_root:
class_mapping = coco2av
class_names = av_classes
else:
raise Exception('Unknown dataset')
config = mmcv.Config.fromfile(opts.config)
if opts.in_scale is not None:
config.data.test.img_scale = opts.in_scale
# mainly for SSD
config.data.test.resize_keep_ratio = True
if opts.no_mask:
if 'mask_head' in config.model:
config.model['mask_head'] = None
model = init_detector(config, opts.weights)
model.eval()
results_ccf = []
for iid, img in tqdm(db.imgs.items()):
img_name = img['name']
sid = img['sid']
seq_name = seqs[sid]
img_path = join(opts.data_root, seq_dirs[sid], img_name)
I = mmcv.imread(img_path)
if iid % opts.det_stride == 0:
result = inference_detector(model, I)
bboxes, scores, labels, masks = parse_mmdet_result(result, class_mapping, n_class)
if len(bboxes):
bboxes_ltwh = bboxes.copy()
# convert to coco fmt
bboxes_ltwh[:, 2:] -= bboxes_ltwh[:, :2]
bboxes_ltwh = bboxes_ltwh.tolist()
else:
bboxes_ltwh = []
if vis_out:
vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(
I, bboxes, labels,
class_names, masks, scores,
out_scale=opts.vis_scale,
out_file=vis_path
)
for i in range(len(bboxes_ltwh)):
result_dict = {
'image_id': iid,
'bbox': bboxes_ltwh[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
# This script does not need to run in real-time
import argparse
from os import scandir
from os.path import join, isfile
from tqdm import tqdm
import numpy as np
from PIL import Image, ImageFont, ImageDraw
import sys; sys.path.insert(0, '..'); sys.path.insert(0, '.')
from util import mkdir2
from vis.make_videos_numbered import worker_func as make_video_func
data_dir = '/data2/mengtial'
out_dir = mkdir2(join(data_dir, 'Exp/ArgoVerse1.1/vid/det_eccv'))
text = [
['Fast Detector (Baseline)', 'Detection too noisy (AP 5.5)'],
['Accurate Detector (Baseline)', 'Latency too high (AP 6.2)'],
['Optimized Detector (Ours)', 'Balanced accuracy and latency (AP 12.0)'],
# ['+ Dynamic Scheduling (Ours)', 'Reducing temporal aliasing (AP 13.7)'],
]
dirs = [
join(data_dir, 'Exp/ArgoVerse1.1-c3-eta0/visf-th0.5/rt_retina50_s0.2_pkt/val'),
join(data_dir, 'Exp/ArgoVerse1.1-c3-eta0/visf-th0.5/rt_htc_dconv2_ms_cpupre_s1_pkt/val'),
join(data_dir, 'Exp/ArgoVerse1.1-c3-eta0/visf-th0.5/rt_mrcnn50_nm_s0.5_pkt/val'),
# join(data_dir, 'Exp/ArgoVerse1.1-c3-eta0/visf-th0.5/srt_mrcnn50_nm_ds_s0.5_pkt/val'),
]
def main():
assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = db.dataset.get('coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
img = db.imgs[0]
w_img, h_img = img['width'], img['height']
mp.set_start_method('spawn')
frame_recv, frame_send = mp.Pipe(False)
det_res_recv, det_res_send = mp.Pipe(False)
det_proc = mp.Process(target=det_process, args=(opts, frame_recv, det_res_send, w_img, h_img))
det_proc.start()
if opts.dynamic_schedule:
runtime = pickle.load(open(opts.runtime, 'rb'))
runtime_dist = dist_from_dict(runtime, opts.perf_factor)
mean_rtf = runtime_dist.mean()*opts.fps
n_total = 0
t_det_all = []
t_send_frame_all = []
t_recv_res_all = []
t_assoc_all = []
t_forecast_all = []
with torch.no_grad():
kf_F = torch.eye(8)
kf_Q = torch.eye(8)
kf_R = 10*torch.eye(4)
kf_P_init = 100*torch.eye(8).unsqueeze(0)
for sid, seq in enumerate(tqdm(seqs)):
frame_list = [img for img in db.imgs.values() if img['sid'] == sid]
frame_list = [join(opts.data_root, seq_dirs[sid], img['name']) for img in frame_list]
n_frame = len(frame_list)
n_total += n_frame
timestamps = []
results_parsed = []
input_fidx = []
processing = False
fidx_t2 = None # detection input index at t2
fidx_latest = None
tkidx = 0 # track starting index
kf_x = torch.empty((0, 8, 1))
kf_P = torch.empty((0, 8, 8))
n_matched12 = 0
# let detector process to read all the frames
frame_send.send(frame_list)
# it is possible that unfetched results remain in the pipe
while 1:
msg = det_res_recv.recv() # wait till the detector is ready
if msg == 'ready':
break
elif isinstance(msg, Exception):
raise msg
t_total = n_frame/opts.fps
t_unit = 1/opts.fps
t_start = perf_counter()
while 1:
t1 = perf_counter()
t_elapsed = t1 - t_start
if t_elapsed >= t_total:
break
# identify latest available frame
fidx_continous = t_elapsed*opts.fps
fidx = int(np.floor(fidx_continous))
if fidx == fidx_latest:
# algorithm is fast and has some idle time
wait_for_next = True
else:
wait_for_next = False
if opts.dynamic_schedule:
if mean_rtf >= 1:
# when runtime < 1, it should always process every frame
fidx_remainder = fidx_continous - fidx
if mean_rtf < np.floor(fidx_remainder + mean_rtf):
# wait till next frame
wait_for_next = True
if wait_for_next:
# sleep
continue
if not processing:
t_start_frame = perf_counter()
frame_send.send((fidx, t_start_frame))
fidx_latest = fidx
processing = True
# wait till query - forecast-rt-ub
wait_time = t_unit - opts.forecast_rt_ub
if det_res_recv.poll(wait_time): # wait
# new result
result = det_res_recv.recv()
if isinstance(result, Exception):
raise result
result, t_send_frame, t_start_res = result
bboxes_t2, scores_t2, labels_t2, _ = \
parse_det_result(result, coco_mapping, n_class)
processing = False
t_det_end = perf_counter()
t_det_all.append(t_det_end - t_start_frame)
t_send_frame_all.append(t_send_frame)
t_recv_res_all.append(t_det_end - t_start_res)
# associate across frames
t_assoc_start = perf_counter()
if len(kf_x):
dt = fidx_latest - fidx_t2
kf_F = make_F(kf_F, dt)
kf_Q = make_Q(kf_Q, dt)
kf_x, kf_P = batch_kf_predict(kf_F, kf_x, kf_P, kf_Q)
bboxes_f = x2bbox(kf_x)
fidx_t2 = fidx_latest
n = len(bboxes_t2)
if n:
# put high scores det first for better iou matching
score_argsort = np.argsort(scores_t2)[::-1]
bboxes_t2 = bboxes_t2[score_argsort]
scores_t2 = scores_t2[score_argsort]
labels_t2 = labels_t2[score_argsort]
ltrb2ltwh_(bboxes_t2)
updated = False
if len(kf_x):
order1, order2, n_matched12, tracks, tkidx = iou_assoc(
bboxes_f, labels, tracks, tkidx,
bboxes_t2, labels_t2, opts.match_iou_th,
no_unmatched1=True,
)
if n_matched12:
kf_x = kf_x[order1]
kf_P = kf_P[order1]
kf_x, kf_P = batch_kf_update(
bbox2z(bboxes_t2[order2[:n_matched12]]),
kf_x,
kf_P,
kf_R,
)
kf_x_new = bbox2x(bboxes_t2[order2[n_matched12:]])
n_unmatched2 = len(bboxes_t2) - n_matched12
kf_P_new = kf_P_init.expand(n_unmatched2, -1, -1)
kf_x = torch.cat((kf_x, kf_x_new))
kf_P = torch.cat((kf_P, kf_P_new))
labels = labels_t2[order2]
scores = scores_t2[order2]
updated = True
if not updated:
# start from scratch
kf_x = bbox2x(bboxes_t2)
kf_P = kf_P_init.expand(len(bboxes_t2), -1, -1)
labels = labels_t2
scores = scores_t2
tracks = np.arange(tkidx, tkidx + n, dtype=np.uint32)
tkidx += n
t_assoc_end = perf_counter()
t_assoc_all.append(t_assoc_end - t_assoc_start)
# apply forecasting for the current query
t_forecast_start = perf_counter()
query_pointer = fidx + opts.eta + 1
if len(kf_x):
dt = (query_pointer - fidx_t2)
kf_x_np = kf_x[:, :, 0].numpy()
bboxes_t3 = kf_x_np[:n_matched12, :4] + dt*kf_x_np[:n_matched12, 4:]
if n_matched12 < len(kf_x):
bboxes_t3 = np.concatenate((bboxes_t3, kf_x_np[n_matched12:, :4]))
bboxes_t3, keep = extrap_clean_up(bboxes_t3, w_img, h_img, lt=True)
labels_t3 = labels[keep]
scores_t3 = scores[keep]
tracks_t3 = tracks[keep]
else:
bboxes_t3 = np.empty((0, 4), dtype=np.float32)
scores_t3 = np.empty((0,), dtype=np.float32)
labels_t3 = np.empty((0,), dtype=np.int32)
tracks_t3 = np.empty((0,), dtype=np.int32)
t_forecast_end = perf_counter()
t_forecast_all.append(t_forecast_end - t_forecast_start)
t3 = perf_counter()
t_elapsed = t3 - t_start
if t_elapsed >= t_total:
break
if len(bboxes_t3):
ltwh2ltrb_(bboxes_t3)
if fidx_t2 is not None:
timestamps.append(t_elapsed)
results_parsed.append((bboxes_t3, scores_t3, labels_t3, None, tracks_t3))
input_fidx.append(fidx_t2)
out_path = join(opts.out_dir, seq + '.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'results_parsed': results_parsed,
'timestamps': timestamps,
'input_fidx': input_fidx,
}, open(out_path, 'wb'))
# terminates the child process
frame_send.send(None)
out_path = join(opts.out_dir, 'time_info.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump({
'n_total': n_total,
't_det': t_det_all,
't_send_frame': t_send_frame_all,
't_recv_res': t_recv_res_all,
't_assoc': t_assoc_all,
't_forecast': t_forecast_all,
}, open(out_path, 'wb'))
# convert to ms for display
s2ms = lambda x: 1e3*x
print_stats(t_det_all, 'Runtime detection (ms)', cvt=s2ms)
print_stats(t_send_frame_all, 'Runtime sending the frame (ms)', cvt=s2ms)
print_stats(t_recv_res_all, 'Runtime receiving the result (ms)', cvt=s2ms)
print_stats(t_assoc_all, "Runtime association (ms)", cvt=s2ms)
print_stats(t_forecast_all, "Runtime forecasting (ms)", cvt=s2ms)
def main():
assert torch.cuda.device_count(
) == 1 # mmdet only supports single GPU testing
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
n_class = len(class_names)
coco_mapping = None if opts.no_class_mapping else db.dataset.get(
'coco_mapping', None)
if coco_mapping is not None:
coco_mapping = np.asarray(coco_mapping)
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
model = init_detector(opts)
results_raw = [] # image based, all 80 COCO classes
results_ccf = [] # instance based
for iid, img in tqdm(db.imgs.items()):
img_name = img['name']
sid = img['sid']
seq_name = seqs[sid]
img_path = join(opts.data_root, seq_dirs[sid], img_name)
I = imread(img_path)
result = inference_detector(model, I, gpu_pre=not opts.cpu_pre)
results_raw.append(result)
bboxes, scores, labels, masks = \
parse_det_result(result, coco_mapping, n_class)
if vis_out:
vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg')
if opts.overwrite or not isfile(vis_path):
vis_det(I,
bboxes,
labels,
class_names,
masks,
scores,
out_scale=opts.vis_scale,
out_file=vis_path)
# convert to coco fmt
n = len(bboxes)
if n:
ltrb2ltwh_(bboxes)
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes[i],
'score': scores[i],
'category_id': labels[i],
}
if masks is not None:
result_dict['segmentation'] = masks[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'results_raw.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_raw, open(out_path, 'wb'))
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
import csv, pickle from glob import glob from os.path import join, isfile, split import numpy as np import sys sys.path.insert(0, '..') sys.path.insert(0, '.') from util import mkdir2 data_dir = '/data2/mengtial' prefix = join(data_dir, 'Exp/ArgoVerse') out_dir = mkdir2(join(prefix, 'runtime-zoo', '1080ti')) ## Detection # methods = [ # *(4*['mrcnn_r50_no_mask']), # 'retina_r50_no_mask', # ] # scales = [ # *[0.25, 0.5, 0.75, 1], # '0.5', # ] methods = ['mrcnn_r50_no_mask_2'] scales = [0.5] for m, s in zip(methods, scales):
def main():
opts = parse_args()
assert opts.forecast_before_assoc, "Not implemented"
mkdir2(opts.out_dir)
data_root = opts.data_root
if 'DTB70' not in data_root:
seqs = os.listdir(os.path.join(data_root, 'data_seq'))
else:
seqs = os.listdir(data_root)
# db = COCO(opts.annot_path)
# class_names = [c['name'] for c in db.dataset['categories']]
# n_class = len(class_names)
# coco_mapping = db.dataset.get('coco_mapping', None)
# if coco_mapping is not None:
# coco_mapping = np.asarray(coco_mapping)
# seqs = db.dataset['sequences']
# seq_dirs = db.dataset['seq_dirs']
given_tracks = opts.assoc == 'given'
assert not given_tracks, "Not implemented"
for sid, seq in enumerate(seqs):
ou_path = os.path.join(opts.out_dir)
if not os.path.isdir(ou_path):
os.makedirs(ou_path)
result_path = os.path.join(ou_path, '{}.txt'.format(seq))
if isfile(result_path):
print("Evaluated with forcasting seq: {} ({}/{}), done!".format(
seq, sid + 1, len(seqs)))
continue
with torch.no_grad():
kf_F = torch.eye(8)
kf_Q = torch.eye(8)
kf_R = 10 * torch.eye(4)
kf_P_init = 100 * torch.eye(8).unsqueeze(0)
results_ccf = []
in_time = 0
miss = 0
shifts = 0
t_assoc = []
t_forecast = []
if 'DTB70' in data_root:
frame_path = os.path.join(data_root, seq, 'img')
else:
frame_path = os.path.join(data_root, 'data_seq', seq)
frames = os.listdir(frame_path)
frame_list = []
for frame in frames:
frame_list.append(os.path.join(frame_path, frame))
results = pickle.load(
open(join(opts.result_root, seq + '.pkl'), 'rb'))
# use raw results when possible in case we change class subset during evaluation
results_raw = results['results_raw']
timestamps = results['timestamps']
timestamps[0] = 0
input_fidx = results['input_fidx']
# t1 -> det1, t2 -> det2, interpolate at t3 (t3 is the current time)
det_latest_p1 = 0 # latest detection index + 1
det_t2 = None # detection index at t2
kf_x = torch.empty((0, 8, 1))
kf_P = torch.empty((0, 8, 8))
kf_F = torch.eye(8)
kf_Q = torch.eye(8)
kf_R = 10 * torch.eye(4)
kf_P_init = 100 * torch.eye(8).unsqueeze(0)
n_matched12 = 0
if not given_tracks:
tkidx = 0 # track starting index
for ii, img_path in enumerate(frame_list):
# pred, gt association by time
if ii == 0:
img = cv2.imread(img_path)
t = (ii - opts.eta) / opts.fps
while det_latest_p1 < len(
timestamps) and timestamps[det_latest_p1] <= t:
det_latest_p1 += 1
if det_latest_p1 == 0:
continue
# no output
else:
det_latest = det_latest_p1 - 1
ifidx = input_fidx[det_latest]
in_time += int(ii == ifidx)
shifts += ii - ifidx
if det_latest != det_t2:
# new detection
# we can now throw away old result (t1)
# the old one is kept for forecasting purpose
if len(kf_x) and opts.forecast_before_assoc:
dt = ifidx - input_fidx[det_t2]
dt = int(
dt
) # convert from numpy to basic python format
w_img, h_img = img.shape[1], img.shape[0]
kf_F = make_F(kf_F, dt)
kf_Q = make_Q(kf_Q, dt)
kf_x, kf_P = batch_kf_predict(
kf_F, kf_x, kf_P, kf_Q)
bboxes_f = x2bbox(kf_x)
det_t2 = det_latest
bboxes_t2 = np.array([results_raw[det_t2]])
t1 = perf_counter()
n = len(bboxes_t2)
if n:
updated = False
if len(kf_x):
order1, order2, n_matched12, tracks, tkidx = iou_assoc(
bboxes_f,
tracks,
tkidx,
bboxes_t2,
opts.match_iou_th,
no_unmatched1=True,
)
if n_matched12:
kf_x = kf_x[order1]
kf_P = kf_P[order1]
kf_x, kf_P = batch_kf_update(
bbox2z(
bboxes_t2[order2[:n_matched12]]),
kf_x,
kf_P,
kf_R,
)
kf_x_new = bbox2x(
bboxes_t2[order2[n_matched12:]])
n_unmatched2 = len(bboxes_t2) - n_matched12
kf_P_new = kf_P_init.expand(
n_unmatched2, -1, -1)
kf_x = torch.cat((kf_x, kf_x_new))
kf_P = torch.cat((kf_P, kf_P_new))
updated = True
if not updated:
# start from scratch
kf_x = bbox2x(bboxes_t2)
kf_P = kf_P_init.expand(len(bboxes_t2), -1, -1)
if not given_tracks:
tracks = np.arange(tkidx,
tkidx + n,
dtype=np.uint32)
tkidx += n
t2 = perf_counter()
t_assoc.append(t2 - t1)
t3 = perf_counter()
if len(kf_x):
dt = ii - ifidx
w_img, h_img = img.shape[1], img.shape[0]
# PyTorch small matrix multiplication is slow
# use numpy instead
kf_x_np = kf_x[:, :, 0].numpy()
bboxes_t3 = kf_x_np[:n_matched12, :
4] + dt * kf_x_np[:n_matched12, 4:]
if n_matched12 < len(kf_x):
bboxes_t3 = np.concatenate(
(bboxes_t3, kf_x_np[n_matched12:, :4]))
bboxes_t3, keep = extrap_clean_up(bboxes_t3,
w_img,
h_img,
lt=True)
t4 = perf_counter()
t_forecast.append(t4 - t3)
if len(bboxes_t3):
results_ccf.append(list(bboxes_t3[0]))
else:
results_ccf.append(list(bboxes_t2[0]))
miss += 1
# ou_path=os.path.join(opts.out_dir)
# if not os.path.isdir(ou_path):
# os.makedirs(ou_path)
# result_path = os.path.join(ou_path, '{}.txt'.format(seq))
with open(result_path, 'w') as f:
for x in results_ccf:
f.write(','.join([str(i) for i in x]) + '\n')
print("Evaluate with forcasting seq: {} ({}/{}), done!,miss {}".format(
seq, sid + 1, len(seqs), miss))
import argparse
from os import scandir
from os.path import join, isfile
from tqdm import tqdm
import numpy as np
from PIL import Image, ImageFont, ImageDraw
import sys
sys.path.insert(0, '..')
sys.path.insert(0, '.')
from util import mkdir2
from vis.make_videos_numbered import worker_func as make_video_func
data_dir = '/data2/mengtial'
out_dir = mkdir2(join(data_dir, 'Exp/ArgoVerse1.1/vid/compute'))
text = [
['Single GPU Final Solution', 'Optimized and predictive (AP 17.8)'],
['Infinite GPU Final Solution', 'More responsive (AP 20.3)'],
]
dirs = [
join(
data_dir,
'Exp/ArgoVerse1.1/visf-th0.5/pps_mrcnn50_nm_ds_s0.75_kf_fba_iou_lin_pkt/val'
),
join(
data_dir,
'Exp/ArgoVerse1.1/visf-th0.5/pps_mrcnn50_nm_inf_s0.75_kf_fba_iou_lin_pkt/val'
),
import numpy as np
from html4vision import Col, imagetile
import sys
sys.path.insert(0, '..')
sys.path.insert(0, '.')
from util import mkdir2
data_dir = '/data2/mengtial'
data_root = join(data_dir, 'ArgoVerse1.1/tracking')
split = 'val'
data_cfg = 'ArgoVerse1.1'
out_dir = mkdir2(join(data_dir, 'Exp', data_cfg, 'visf'))
out_name = 'best_single_all_seq_mask.html'
title = 'Best Single GPU (Mask)'
link_video = True
n_show = 100
np.random.seed(0)
folder = join(data_dir, 'Exp', data_cfg, 'visf-th0.5',
'pps_mrcnn50_ds_s0.5_kf_fba_iou_lin_pkt', split)
print(f'python vis/make_videos.py "{folder}" --fps 30')
srv_dir = data_dir
srv_port = 40001
# host_name = socket.gethostname()
host_name = 'trinity.vision.cs.cmu.edu'
def main():
opts = parse_args()
mkdir2(opts.out_dir)
vis_out = bool(opts.vis_dir)
if vis_out:
mkdir2(opts.vis_dir)
db = COCO(opts.annot_path)
class_names = [c['name'] for c in db.dataset['categories']]
seqs = db.dataset['sequences']
seq_dirs = db.dataset['seq_dirs']
box_ccf = pickle.load(open(opts.box_ccf_path, 'rb'))
mask_ccf = pickle.load(open(opts.mask_ccf_path, 'rb'))
box_end_idx = 0
mask_end_idx = 0
results_ccf = [] # instance based
for iid, img in tqdm(db.imgs.items()):
img_name = img['name']
sid = img['sid']
seq_name = seqs[sid]
box_start_idx = box_end_idx
while box_start_idx < len(box_ccf) and box_ccf[box_start_idx]['image_id'] < img['id']:
box_start_idx += 1
box_end_idx = box_start_idx
while box_end_idx < len(box_ccf) and box_ccf[box_end_idx]['image_id'] == img['id']:
box_end_idx += 1
box_dets = box_ccf[box_start_idx:box_end_idx]
mask_start_idx = mask_end_idx
while mask_start_idx < len(mask_ccf) and mask_ccf[mask_start_idx]['image_id'] < img['id']:
mask_start_idx += 1
mask_end_idx = mask_start_idx
while mask_end_idx < len(mask_ccf) and mask_ccf[mask_end_idx]['image_id'] == img['id']:
mask_end_idx += 1
mask_dets = mask_ccf[mask_start_idx:mask_end_idx]
if len(box_dets) == 0:
bboxes1, scores1, labels1, masks1 = [], [], [], None
elif len(mask_dets) == 0:
bboxes1 = np.array([d['bbox'] for d in box_dets])
scores1 = np.array([d['score'] for d in box_dets])
labels1 = np.array([d['category_id'] for d in box_dets])
masks1 = np.array([d['segmentation'] for d in box_dets])
else:
# the slow version, but works with out of order ccf results
# dets = [r for r in box_ccf if r['image_id'] == img['id']]
bboxes1 = np.array([d['bbox'] for d in box_dets])
scores1 = np.array([d['score'] for d in box_dets])
labels1 = np.array([d['category_id'] for d in box_dets])
masks1 = np.array([d['segmentation'] for d in box_dets])
bboxes2 = np.array([d['bbox'] for d in mask_dets])
scores2 = np.array([d['score'] for d in mask_dets])
labels2 = np.array([d['category_id'] for d in mask_dets])
masks2 = np.array([d['segmentation'] for d in mask_dets])
score_argsort = np.argsort(scores1)[::-1]
bboxes1 = bboxes1[score_argsort]
scores1 = scores1[score_argsort]
labels1 = labels1[score_argsort]
masks1 = masks1[score_argsort]
score_argsort = np.argsort(scores2)[::-1]
bboxes2 = bboxes2[score_argsort]
scores2 = scores2[score_argsort]
labels2 = labels2[score_argsort]
masks2 = masks2[score_argsort]
order1, order2, n_matched12, _, _ = iou_assoc(
bboxes1, labels1, np.arange(len(bboxes1)), 0,
bboxes2, labels2, opts.match_iou_th,
no_unmatched1=False,
)
bboxes1 = bboxes1[order1]
scores1 = scores1[order1]
labels1 = labels1[order1]
masks1 = masks1[order1]
bboxes2 = bboxes2[order2]
scores2 = scores2[order2]
labels2 = labels2[order2]
masks2 = masks2[order2]
mask1_fix = warp_mask_to_box(
masks2[:n_matched12],
bboxes2[:n_matched12],
bboxes1[:n_matched12]
)
masks1 = np.concatenate((mask1_fix, masks1[n_matched12:]))
if vis_out:
img_path = join(opts.data_root, seq_dirs[sid], img_name)
I = imread(img_path)
vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg')
bboxes = ltwh2ltrb(bboxes1) if len(bboxes1) else []
if opts.overwrite or not isfile(vis_path):
vis_det(
I, bboxes, labels1,
class_names, masks1, scores1,
out_scale=opts.vis_scale,
out_file=vis_path
)
n = len(bboxes1)
for i in range(n):
result_dict = {
'image_id': iid,
'bbox': bboxes1[i],
'score': scores1[i],
'category_id': labels1[i],
}
if masks1 is not None:
result_dict['segmentation'] = masks1[i]
results_ccf.append(result_dict)
out_path = join(opts.out_dir, 'results_ccf.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(results_ccf, open(out_path, 'wb'))
if not opts.no_eval:
eval_summary = eval_ccf(db, results_ccf)
out_path = join(opts.out_dir, 'eval_summary.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if opts.eval_mask:
print('Evaluating instance segmentation')
eval_summary = eval_ccf(db, results_ccf, iou_type='segm')
out_path = join(opts.out_dir, 'eval_summary_mask.pkl')
if opts.overwrite or not isfile(out_path):
pickle.dump(eval_summary, open(out_path, 'wb'))
if vis_out:
print(f'python vis/make_videos.py "{opts.vis_dir}"')
