def main(): opts = parse_args() mkdir2(opts.out_dir) print('Reading detection images') det_img_files = sorted(glob(join(opts.det_img_dir, '*.jpg'))) det_imgs = [] for path in det_img_files: det_imgs.append(imread(path)) print('Detection Test at Scale 0.5') rt_det1 = det_test(opts, 0.5, det_imgs) print(f'Mean runtime (ms): {1e3*rt_det1:.3g}') print('Detection Test at Scale 1') rt_det2 = det_test(opts, 1, det_imgs) print(f'Mean runtime (ms): {1e3*rt_det2:.3g}') det_imgs = None print('Classification Test') rt_inf, rt_train = classify_test(opts) print(f'Inference Mean runtime (ms): {1e3*rt_inf:.3g}') print(f'Training Mean runtime (ms): {1e3*rt_train:.3g}') out_path = join(opts.out_dir, 'platform_bench.txt') if opts.overwrite or not isfile(out_path): with open(out_path, 'w', newline='\n') as f: writer = csv.writer(f) writer.writerow([rt_det1, rt_det2, rt_inf, rt_train]) print(f'Saved results to "{out_path}"')
def imwrite(img, path, method='PIL', auto_mkdir=True): if method == 'PIL': if auto_mkdir: mkdir2(dirname(path)) Image.fromarray(img).save(path) else: mmcv.imwrite(img, path, auto_mkdir=auto_mkdir)
def main(): assert torch.cuda.device_count( ) == 1 # mmdet only supports single GPU testing opts = parse_args() mkdir2(opts.out_dir) vis_out = bool(opts.vis_dir) if vis_out: mkdir2(opts.vis_dir) config = mmcv.Config.fromfile(opts.config) if opts.in_scale is not None: config.data.test.img_scale = opts.in_scale # mainly for SSD config.data.test.resize_keep_ratio = True if opts.no_mask: if 'mask_head' in config.model: config.model['mask_head'] = None model = init_detector(config, opts.weights) model.eval() img_list = sorted(glob(join(opts.in_dir, '*.jpg'))) img_list = img_list[::opts.image_stride] for img_path in tqdm(img_list): img_name = basename(img_path) img = mmcv.imread(img_path) result = inference_detector(model, img) out_path = join(opts.out_dir, img_name[:-3] + 'pkl') if opts.overwrite or not isfile(out_path): pickle.dump(result, open(out_path, 'wb'))
def main(): opts = parse_args() if not opts.overwrite and isfile(opts.out_path): return mkdir2(dirname(opts.out_path)) time_info = pickle.load(open(opts.time_info, 'rb')) if opts.method_type == 'det': rt_samples = time_info['runtime_all'] rt_dist = {'type': 'empirical', 'samples': rt_samples} else: raise ValueError(f'Unknown method type "{opts.method_type}"') pickle.dump(rt_dist, open(opts.out_path, 'wb'))
def main(): opts = parse_args() if not opts.overwrite and isfile(opts.out_path): return mkdir2(dirname(opts.out_path)) class_subset = avhd_subset cats = [] class_names = [] for i, c in enumerate(class_subset): cats.append({'id': i, 'name': coco_classes[c]}) class_names.append(coco_classes[c]) n_coco = len(coco_classes) coco_mapping = n_coco * [n_coco] for i, c in enumerate(class_subset): coco_mapping[c] = i frames = sorted( [item.name for item in scandir(opts.img_folder) if item.is_file()]) if opts.start is not None: frames = frames[opts.start:] img = Image.open(join(opts.img_folder, frames[0])) img_width, img_height = img.size seqs = [opts.prefix] seq_dirs = [opts.prefix] imgs = [] for i, name in enumerate(frames): imgs.append({ 'id': i, 'sid': 0, 'fid': i, 'name': name, 'width': img_width, 'height': img_height, }) dataset = { 'categories': cats, 'images': imgs, 'sequences': seqs, 'seq_dirs': seq_dirs, 'coco_subset': class_subset, 'coco_mapping': coco_mapping, } json.dump(dataset, open(opts.out_path, 'w'))
def main(): opts = parse_args() mkdir2(opts.out_dir) db = COCO(opts.annot_path) eval_summary = eval_ccf(db, opts.result_path, None, opts.per_class) out_path = join(opts.out_dir, 'eval_summary.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if opts.eval_mask: print('Evaluating instance segmentation') eval_summary = eval_ccf(db, opts.result_path, iou_type='segm') out_path = join(opts.out_dir, 'eval_summary_mask.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb'))
def main(): opts = parse_args() if not opts.overwrite and isfile(opts.out_path): return mkdir2(dirname(opts.out_path)) time_info = pickle.load(open(opts.time_info, 'rb')) if opts.method_type == 'det': rt_samples = time_info['runtime_all'] rt_dist = {'type': 'empirical', 'samples': rt_samples} elif opts.method_type == 'dat': which_cfg_all = np.array(time_info['which_cfg_all']) rt_samples = np.array(time_info['runtime_all'])[which_cfg_all == 1] rt_samples = rt_samples[rt_samples > 0.002] # ignore zero detection trivial cases rt_dist = {'type': 'empirical', 'samples': rt_samples.tolist()} else: raise ValueError(f'Unknown method type "{opts.method_type}"') pickle.dump(rt_dist, open(opts.out_path, 'wb'))
def main(): opts = parse_args() mkdir2(opts.vis_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] seq = 'f1008c18-e76e-3c24-adcc-da9858fac145' sid = seqs.index(seq) frame_list = [img for img in db.imgs.values() if img['sid'] == sid] results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb')) results_parsed = results['results_parsed'] img = frame_list[153] result = results_parsed[15] bboxes, scores, labels, masks = result[:4] idx = [16] bboxes = bboxes[idx] scores = scores[idx] labels = labels[idx] masks = masks[idx] img_path = join(opts.data_root, seq_dirs[sid], img['name']) I = imread(img_path).copy() vis_path = join(opts.vis_dir, 'teaser.jpg') if opts.overwrite or not isfile(vis_path): vis_det( I, bboxes, labels, class_names, masks, None, out_scale=opts.vis_scale, out_file=vis_path, )
def main(): opts = parse_args() out_dir = mkdir2(opts.out_dir) if opts.out_dir else opts.result_dir db = COCO(opts.annot_path) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] n_concurrent = [] for sid, seq in enumerate(tqdm(seqs)): frame_list = [img for img in db.imgs.values() if img['sid'] == sid] results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb')) # use raw results when possible in case we change class subset during evaluation if opts.type == 'det': timestamps = results['timestamps'] input_fidx = results['input_fidx'] else: det1_timestamps = results['det1_timestamps'] det2_timestamps = results['det2_timestamps'] det1_input_fidx = results['det1_input_fidx'] det2_input_fidx = results['det2_input_fidx'] timestamps = np.concatenate((det1_timestamps, det2_timestamps)) input_fidx = np.concatenate((det1_input_fidx, det2_input_fidx)) t_start = np.asarray(input_fidx) / opts.fps t_end = np.asarray(timestamps) t_all = np.concatenate((t_start, t_end)) order = np.argsort(t_all) n_output = len(t_start) n_current = 0 max_current = 0 for i in order: if i < n_output: # start n_current += 1 max_current = max(max_current, n_current) else: # end n_current -= 1 n_concurrent.append(max_current) print(f'Max number of concurrent jobs {max(n_concurrent)}') out_path = join(out_dir, 'n_concurrent.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(n_concurrent, open(out_path, 'wb'))
def main(): assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing opts = parse_args() assert (not opts.mask_timing) or (opts.mask_timing and not opts.no_mask) mkdir2(opts.out_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] n_class = len(class_names) coco_mapping = None if opts.no_class_mapping else db.dataset.get('coco_mapping', None) if coco_mapping is not None: coco_mapping = np.asarray(coco_mapping) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] model = init_detector(opts) # warm up the GPU img = db.imgs[0] w_img, h_img = img['width'], img['height'] _ = inference_detector(model, np.zeros((h_img, w_img, 3), np.uint8)) torch.cuda.synchronize() runtime_all = [] n_processed = 0 n_total = 0 for sid, seq in enumerate(tqdm(seqs)): frame_list = [img for img in db.imgs.values() if img['sid'] == sid] # load all frames in advance frames = [] for img in frame_list: img_path = join(opts.data_root, seq_dirs[sid], img['name']) frames.append(imread(img_path)) n_frame = len(frames) n_total += n_frame timestamps = [] results_raw = [] results_parsed = [] input_fidx = [] runtime = [] last_fidx = None if not opts.dynamic_schedule: stride_cnt = 0 t_total = n_frame/opts.fps t_start = perf_counter() while 1: t1 = perf_counter() t_elapsed = t1 - t_start if t_elapsed >= t_total: break # identify latest available frame fidx_continous = t_elapsed*opts.fps fidx = int(np.floor(fidx_continous)) if fidx == last_fidx: continue last_fidx = fidx if opts.dynamic_schedule: fidx_remainder = fidx_continous - fidx if fidx_remainder > 0.5: continue else: if stride_cnt % opts.det_stride == 0: stride_cnt = 1 else: stride_cnt += 1 continue frame = frames[fidx] result = inference_detector(model, frame, gpu_pre=not opts.cpu_pre, decode_mask=not opts.mask_timing) if opts.mask_timing: mask_encoded = result[2] result = result[:2] bboxes, scores, labels, _, sel = \ parse_det_result(result, coco_mapping, n_class, return_sel=True) else: bboxes, scores, labels, masks = \ parse_det_result(result, coco_mapping, n_class) torch.cuda.synchronize() t2 = perf_counter() t_elapsed = t2 - t_start if t_elapsed >= t_total: break if opts.mask_timing: masks = decode_mask(mask_encoded, sel) timestamps.append(t_elapsed) results_raw.append(result) results_parsed.append((bboxes, scores, labels, masks)) input_fidx.append(fidx) runtime.append(t2 - t1) out_path = join(opts.out_dir, seq + '.pkl') if opts.overwrite or not isfile(out_path): pickle.dump({ 'results_raw': results_raw, 'results_parsed': results_parsed, 'timestamps': timestamps, 'input_fidx': input_fidx, 'runtime': runtime, }, open(out_path, 'wb')) runtime_all += runtime n_processed += len(results_raw) runtime_all_np = np.asarray(runtime_all) n_small_runtime = (runtime_all_np < 1.0/opts.fps).sum() out_path = join(opts.out_dir, 'time_info.pkl') if opts.overwrite or not isfile(out_path): pickle.dump({ 'runtime_all': runtime_all, 'n_processed': n_processed, 'n_total': n_total, 'n_small_runtime': n_small_runtime, }, open(out_path, 'wb')) # convert to ms for display s2ms = lambda x: 1e3*x print(f'{n_processed}/{n_total} frames processed') print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms) print(f'Runtime smaller than unit time interval: ' f'{n_small_runtime}/{n_processed} ' f'({100.0*n_small_runtime/n_processed:.4g}%)')
from os.path import join, basename, isfile from glob import glob import numpy as np from html4vision import Col, imagetable import sys; sys.path.insert(0, '..'); sys.path.insert(0, '.') from util import mkdir2 data_dir = '/data2/mengtial' data_root = join(data_dir, 'ArgoVerse1.1/tracking') split = 'val' annot_file = join(data_root, 'coco_fmt/htc_dconv2_ms_s1.0_mask_' + split + '.json') out_dir = mkdir2(join(data_dir, 'Exp/ArgoVerse1.1/vis')) out_name = 'mask.html' title = 'Mask' metric = 'Mask AP' link_video = False n_show = 90 n_consec = 15 align = False # align to the stride in each sequence stride = 30 random.seed(0) names = [ 'Offline Accurate', 'Real-time Accurate', 'Real-time Ours',
def main(): line_color_np = np.array(line_color, dtype=np.uint8).reshape((1, 1, 3)) seq_dir_out = mkdir2(join(out_dir, seq)) frame_list = [ item.name for item in scandir(join(dirs[0], seq)) if item.is_file() and item.name.endswith('.jpg') ] frame_list = sorted(frame_list) n_method = len(text) fidx = 0 j_start = 0 for i in tqdm(range(n_method)): for j in range(j_start, t_duration): fidx += 1 out_path = join(seq_dir_out, '%06d.jpg' % fidx) if not overwrite and isfile(out_path): continue img_A = Image.open(join(dirs[i], seq, frame_list[j + clip_start])) # cropping img_A = np.array(img_A) img_A = img_A[crop_top:-crop_bottom] progress_text = ease_in_out( (j - j_start) / t_text_appear) if j - j_start < t_text_appear else 1 # render text region img_A[text_region[1]:text_region[3] + 1, text_region[0]:text_region[2] + 1] = \ np.round((1 + progress_text*(text_region_alpha - 1))*img_A[text_region[1]:text_region[3] + 1, text_region[0]:text_region[2] + 1]).astype(np.uint8) img_A_with_text = Image.fromarray(img_A) # using TrueType supported in PIL draw = ImageDraw.Draw(img_A_with_text) draw.text( (text_xy[0], text_xy[1]), text[i][0], (*text_color, 255), # RGBA font=font, ) draw.text( (text_xy[0], text_xy[1] + text_line_sep), text[i][1], (*text_color, 255), # RGBA font=font, ) img_A_with_text = np.array(img_A_with_text) img_A = np.round(progress_text * img_A_with_text + (1 - progress_text) * img_A).astype(np.uint8) img_A = Image.fromarray(img_A) if i < n_method - 1 and j >= t_duration - t_transition: # transition period j_start = t_transition j2 = j + t_transition - t_duration img_B = Image.open( join(dirs[i + 1], seq, frame_list[j2 + clip_start])) img_B = np.array(img_B) img_B = img_B[crop_top:-crop_bottom] h, w, _ = img_B.shape split_pos = split_anime_accelerate(j2 / t_transition, w, line_width) split_pos = int(round(split_pos)) line_start = split_pos - (line_width - 1) // 2 line_end = split_pos + line_width // 2 # inclusive if split_pos <= 0: img = img_B else: img = np.array(img_A) img_B = np.asarray(img_B) img[:, split_pos:] = img_B[:, split_pos:] if line_start < w and line_end >= 0: # line is visible line_start = max(0, line_start) line_end = min(w, line_end) img[:, line_start:line_end] = line_color_np out_img = Image.fromarray(img) else: out_img = img_A out_img.save(out_path) if make_video: out_path = seq_dir_out + '.mp4' if overwrite or not isfile(out_path): print('Making the video') class Dummy(): fps opts = Dummy() opts.fps = fps make_video_func((seq_dir_out, opts)) else: print( f'python vis/make_videos_numbered.py "{opts.out_dir}" --fps {opts.fps}' )
def main(): opts = parse_args() mkdir2(opts.out_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] n_class = len(class_names) coco_mapping = None if opts.no_class_mapping else db.dataset.get('coco_mapping', None) if coco_mapping is not None: coco_mapping = np.asarray(coco_mapping) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] if opts.cached_res: cache_in_ccf = '_ccf' in basename(opts.cached_res) if cache_in_ccf: # speed up based on the assumption of sequential storage cache_end_idx = 0 cached_res = pickle.load(open(opts.cached_res, 'rb')) else: assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing model = init_detector(opts) np.random.seed(opts.seed) runtime = pickle.load(open(opts.runtime, 'rb')) runtime_dist = dist_from_dict(runtime, opts.perf_factor) runtime_all = [] n_processed = 0 n_total = 0 for sid, seq in enumerate(tqdm(seqs)): frame_list = [img for img in db.imgs.values() if img['sid'] == sid] n_frame = len(frame_list) n_total += n_frame if not opts.cached_res: # load all frames in advance frames = [] for img in frame_list: img_path = join(opts.data_root, seq_dirs[sid], img['name']) frames.append(imread(img_path)) timestamps = [] results_parsed = [] input_fidx = [] runtime = [] last_fidx = None if opts.cached_res and cache_in_ccf: results_raw = None else: results_raw = [] t_total = n_frame/opts.fps t_elapsed = 0 if opts.dynamic_schedule: mean_rtf = runtime_dist.mean()*opts.fps else: stride_cnt = 0 while 1: if t_elapsed >= t_total: break # identify latest available frame fidx_continous = t_elapsed*opts.fps fidx = int(np.floor(fidx_continous)) if fidx == last_fidx: # algorithm is fast and has some idle time fidx += 1 if fidx == n_frame: break t_elapsed = fidx/opts.fps last_fidx = fidx if opts.dynamic_schedule: if mean_rtf > 1: # when runtime <= 1, it should always process every frame fidx_remainder = fidx_continous - fidx if mean_rtf < np.floor(fidx_remainder + mean_rtf): # wait till next frame continue else: if stride_cnt % opts.det_stride == 0: stride_cnt = 1 else: stride_cnt += 1 continue if opts.cached_res: img = frame_list[fidx] if cache_in_ccf: cache_end_idx, bboxes, scores, labels, masks = \ result_from_ccf(cached_res, img['id'], cache_end_idx) ltwh2ltrb_(bboxes) else: result = cached_res[img['id']] bboxes, scores, labels, masks = \ parse_det_result(result, coco_mapping, n_class) else: frame = frames[fidx] result = inference_detector(model, frame) bboxes, scores, labels, masks = \ parse_det_result(result, coco_mapping, n_class) rt_this = runtime_dist.draw() t_elapsed += rt_this if t_elapsed >= t_total: break timestamps.append(t_elapsed) if results_raw is not None: results_raw.append(result) results_parsed.append((bboxes, scores, labels, masks)) input_fidx.append(fidx) runtime.append(rt_this) out_path = join(opts.out_dir, seq + '.pkl') if opts.overwrite or not isfile(out_path): out_dict = { 'results_parsed': results_parsed, 'timestamps': timestamps, 'input_fidx': input_fidx, 'runtime': runtime, } if results_raw is not None: out_dict['results_raw'] = results_raw pickle.dump(out_dict, open(out_path, 'wb')) runtime_all += runtime n_processed += len(results_parsed) runtime_all_np = np.array(runtime_all) n_small_runtime = (runtime_all_np < 1.0/opts.fps).sum() out_path = join(opts.out_dir, 'time_info.pkl') if opts.overwrite or not isfile(out_path): pickle.dump({ 'runtime_all': runtime_all, 'n_processed': n_processed, 'n_total': n_total, 'n_small_runtime': n_small_runtime, }, open(out_path, 'wb')) # convert to ms for display s2ms = lambda x: 1e3*x print(f'{n_processed}/{n_total} frames processed') print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms) print(f'Runtime smaller than unit time interval: ' f'{n_small_runtime}/{n_processed} ' f'({100.0*n_small_runtime/n_processed:.4g}%)')
def main(): opts = parse_args() mkdir2(opts.vis_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] coco_subset = np.asarray(db.dataset['coco_subset']) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] color_palette = [ class_palette[k] for k in coco_subset if k in class_palette ] if opts.gt: results_ccf = db.dataset['annotations'] else: results_ccf = pickle.load(open(opts.result_path, 'rb')) if opts.seq is not None: if opts.seq.isdigit(): idx = int(opts.seq) else: idx = seqs.index(opts.seq) seqs = [seqs[idx]] else: idx = None # font_path = r'C:\Windows\Fonts\AdobeGurmukhi-Regular.otf' # font = ImageFont.truetype(font_path, size=40) font = None for sid, seq in enumerate(tqdm(seqs)): if idx is not None: sid = idx frame_list = [img for img in db.imgs.values() if img['sid'] == sid] for ii, img in enumerate(frame_list): img_path = join(opts.data_root, seq_dirs[sid], img['name']) dets = [r for r in results_ccf if r['image_id'] == img['id']] bboxes = np.array([d['bbox'] for d in dets]) masks = None if len(bboxes): bboxes[:, 2:] += bboxes[:, :2] if 'segmentation' in dets[0]: masks = [d['segmentation'] for d in dets] labels = np.array([d['category_id'] for d in dets]) if opts.gt: iscrowd = np.array([d['iscrowd'] for d in dets]) scores = None else: iscrowd = None scores = np.array([d['score'] for d in dets]) vis_path = join(opts.vis_dir, seq, '%06d.jpg' % (ii + 1)) # img['name'][:-3] + 'jpg' if opts.overwrite or not isfile(vis_path): vis_obj_fancy( img_path, bboxes, labels, class_names, color_palette, masks=masks, show_label=False, scores=scores, show_score=False, iscrowd=iscrowd, score_th=opts.score_th, out_scale=opts.vis_scale, out_file=vis_path, font=font, ) if opts.make_video: seq_dir_out = join(opts.vis_dir, seq) out_path = seq_dir_out + '.mp4' if opts.overwrite or not isfile(out_path): make_video((seq_dir_out, opts)) if not opts.make_video: print( f'python vis/make_videos_numbered.py "{opts.vis_dir}" --fps {opts.fps}' )
import numpy as np from html4vision import Col, imagetable import sys sys.path.insert(0, '..') sys.path.insert(0, '.') from util import mkdir2 data_dir = '/data2/mengtial' data_root = join(data_dir, 'ArgoVerse1.1/tracking') split = 'val' annot_file = join(data_root, 'coco_fmt/' + split + '.json') out_dir = mkdir2(join(data_dir, 'Exp', 'ArgoVerse1.1', 'vis-inf')) out_name = 'inf_multi1.html' title = 'Infinite GPU multi-detector 1' metric = 'AP' link_video = True n_show = 90 n_consec = 30 align = True # align to the stride in each sequence stride = 30 random.seed(0) names = [ 'Ground Truth', 'Offline MRCNN R50 s0.5', 'Offline MRCNN R50 s1.0', 'Multi-detector Forecasting',
def main(): assert torch.cuda.device_count( ) == 1 # mmdet only supports single GPU testing opts = parse_args() mkdir2(opts.out_dir) vis_out = bool(opts.vis_dir) if vis_out: mkdir2(opts.vis_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] n_class = len(class_names) coco_mapping = None if opts.no_class_mapping else db.dataset.get( 'coco_mapping', None) if coco_mapping is not None: coco_mapping = np.asarray(coco_mapping) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] model = init_detector(opts) if opts.weights_base is not None: # for distillation purpose load_checkpoint(model, opts.weights_base) if opts.cpu_pre: img_transform = ImageTransform( size_divisor=model.cfg.data.test.size_divisor, **model.cfg.img_norm_cfg) else: img_transform = ImageTransformGPU( size_divisor=model.cfg.data.test.size_divisor, **model.cfg.img_norm_cfg) device = next(model.parameters()).device # model device n_history = model.cfg.data.train.n_history if opts.n_history is None else opts.n_history n_future = model.cfg.data.train.n_future if opts.n_future is None else opts.n_future results_ccf = [] # instance based runtime_all = [] for sid, seq in enumerate(tqdm(seqs)): # print(seq) frame_list = [img for img in db.imgs.values() if img['sid'] == sid] n_frame = len(frame_list) # load all frames in advance frames = [] for img in frame_list: img_path = join(opts.data_root, seq_dirs[sid], img['name']) frames.append(imread(img_path)) with torch.no_grad(): preprocessed = [] for i in range(n_history): data = _prepare_data(frames[i], img_transform, model.cfg, device) preprocessed.append(data) for ii in range(n_history, n_frame - n_future): # target frame iid = frame_list[ii + n_future]['id'] img_name = frame_list[ii + n_future]['name'] I = frames[ii + n_future] t_start = perf_counter() # input frame data = _prepare_data(frames[ii], img_transform, model.cfg, device) # if n_history == 0: # data_merge = data # # print(data['img']) # # print(data['img'][0].shape) # # print(data['img'][0][0][0][300][300:305]) # # import sys # # sys.exit() # else: preprocessed.append(data) # print(preprocessed[0]['img'][0].data_ptr()) # print(preprocessed[2]['img'][0].data_ptr()) # print(torch.all(preprocessed[0]['img'][0] == preprocessed[2]['img'][0])) imgs = [d['img'][0] for d in preprocessed] imgs = torch.cat(imgs, 0) imgs = imgs.unsqueeze(0) data_merge = { 'img': [imgs], 'img_meta': data['img_meta'], } # print(data_merge['img'][0][0][2][0][300][300:305]) # import sys # sys.exit() result = model(return_loss=False, rescale=True, numpy_res=True, **data_merge) bboxes, scores, labels, masks = \ parse_det_result(result, coco_mapping, n_class) # if ii == 2: # print(ii, scores) # import sys # sys.exit() # if n_history != 0: del preprocessed[0] t_end = perf_counter() runtime_all.append(t_end - t_start) if vis_out: vis_path = join(opts.vis_dir, seq, img_name[:-3] + 'jpg') if opts.overwrite or not isfile(vis_path): vis_det(I, bboxes, labels, class_names, masks, scores, out_scale=opts.vis_scale, out_file=vis_path) # convert to coco fmt n = len(bboxes) if n: bboxes[:, 2:] -= bboxes[:, :2] for i in range(n): result_dict = { 'image_id': iid, 'bbox': bboxes[i], 'score': scores[i], 'category_id': labels[i], } if masks is not None: result_dict['segmentation'] = masks[i] results_ccf.append(result_dict) out_path = join(opts.out_dir, 'time_info.pkl') if opts.overwrite or not isfile(out_path): pickle.dump({ 'runtime_all': runtime_all, 'n_total': len(runtime_all), }, open(out_path, 'wb')) # convert to ms for display s2ms = lambda x: 1e3 * x print_stats(runtime_all, 'Runtime (ms)', cvt=s2ms) out_path = join(opts.out_dir, 'results_ccf.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(results_ccf, open(out_path, 'wb')) if not opts.no_eval: eval_summary = eval_ccf(db, results_ccf) out_path = join(opts.out_dir, 'eval_summary.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if vis_out: print(f'python vis/make_videos.py "{opts.vis_dir}"')
def main(): assert torch.cuda.device_count( ) == 1 # mmdet only supports single GPU testing opts = parse_args() mp.set_start_method('spawn') frame_recv, frame_send = mp.Pipe(False) det_res_recv, det_res_send = mp.Pipe(False) det_proc = mp.Process(target=det_process, args=(opts, frame_recv, det_res_send)) det_proc.start() mkdir2(opts.out_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] n_class = len(class_names) coco_mapping = db.dataset.get('coco_mapping', None) if coco_mapping is not None: coco_mapping = np.asarray(coco_mapping) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] runtime_all = [] n_processed = 0 n_total = 0 t_send_frame_all = [] t_recv_res_all = [] for sid, seq in enumerate(tqdm(seqs)): frame_list = [img for img in db.imgs.values() if img['sid'] == sid] frame_list = [ join(opts.data_root, seq_dirs[sid], img['name']) for img in frame_list ] n_frame = len(frame_list) n_total += n_frame timestamps = [] results_raw = [] results_parsed = [] input_fidx = [] runtime = [] last_fidx = None stride_cnt = 0 # let detector process to read all the frames frame_send.send(frame_list) init_error = det_res_recv.recv() # wait till the detector is ready if init_error is not None: raise init_error t_total = n_frame / opts.fps t_start = perf_counter() while 1: t1 = perf_counter() t_elapsed = t1 - t_start if t_elapsed >= t_total: break # identify latest available frame fidx = int(np.floor(t_elapsed * opts.fps)) # t_elapsed/t_total *n_frame # = t_elapsed*opts.fps if fidx == last_fidx: continue last_fidx = fidx if stride_cnt % opts.det_stride == 0: stride_cnt = 1 else: stride_cnt += 1 continue t_start_frame = perf_counter() frame_send.send((fidx, t_start_frame)) result = det_res_recv.recv() # wait if isinstance(result, Exception): raise result result, t_send_frame, t_start_res = result bboxes, scores, labels, masks = \ parse_det_result(result, coco_mapping, n_class) t2 = perf_counter() t_send_frame_all.append(t_send_frame) t_recv_res_all.append(t2 - t_start_res) t_elapsed = t2 - t_start if t_elapsed >= t_total: break timestamps.append(t_elapsed) results_raw.append(result) results_parsed.append((bboxes, scores, labels, masks)) input_fidx.append(fidx) runtime.append(t2 - t1) out_path = join(opts.out_dir, seq + '.pkl') if opts.overwrite or not isfile(out_path): pickle.dump( { 'results_raw': results_raw, 'results_parsed': results_parsed, 'timestamps': timestamps, 'input_fidx': input_fidx, 'runtime': runtime, }, open(out_path, 'wb')) runtime_all += runtime n_processed += len(results_raw) # terminates the child process frame_send.send(None) runtime_all_np = np.asarray(runtime_all) n_small_runtime = (runtime_all_np < 1.0 / opts.fps).sum() out_path = join(opts.out_dir, 'time_info.pkl') if opts.overwrite or not isfile(out_path): pickle.dump( { 'runtime_all': runtime_all, 'n_processed': n_processed, 'n_total': n_total, 'n_small_runtime': n_small_runtime, }, open(out_path, 'wb')) # convert to ms for display s2ms = lambda x: 1e3 * x print(f'{n_processed}/{n_total} frames processed') print_stats(runtime_all_np, 'Runtime (ms)', cvt=s2ms) print(f'Runtime smaller than unit time interval: ' f'{n_small_runtime}/{n_processed} ' f'({100.0*n_small_runtime/n_processed:.4g}%)') print_stats(t_send_frame_all, 'Time spent on sending the frame (ms)', cvt=s2ms) print_stats(t_recv_res_all, 'Time spent on receiving the result (ms)', cvt=s2ms)
def main(): opts = parse_args() out_dir = mkdir2(opts.out_dir) if opts.out_dir else opts.result_dir vis_out = bool(opts.vis_dir) if vis_out: mkdir2(opts.vis_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] n_class = len(class_names) coco_mapping = None if opts.no_class_mapping else db.dataset.get( 'coco_mapping', None) if coco_mapping is not None: coco_mapping = np.asarray(coco_mapping) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] results_ccf = [] in_time = 0 miss = 0 mismatch = 0 print('Pairing the output with the ground truth') for sid, seq in enumerate(tqdm(seqs)): frame_list = [img for img in db.imgs.values() if img['sid'] == sid] results = pickle.load(open(join(opts.result_dir, seq + '.pkl'), 'rb')) # use raw results when possible in case we change class subset during evaluation if opts.use_parsed: results_parsed = results['results_parsed'] else: results_raw = results.get('results_raw', None) if results_raw is None: results_parsed = results['results_parsed'] timestamps = results['timestamps'] input_fidx = results['input_fidx'] tidx_p1 = 0 for ii, img in enumerate(frame_list): # pred, gt association by time t = (ii - opts.eta) / opts.fps while tidx_p1 < len(timestamps) and timestamps[tidx_p1] <= t: tidx_p1 += 1 if tidx_p1 == 0: # no output miss += 1 bboxes, scores, labels = [], [], [] masks, tracks = None, None else: tidx = tidx_p1 - 1 ifidx = input_fidx[tidx] in_time += int(ii == ifidx) mismatch += ii - ifidx if opts.use_parsed or results_raw is None: result = results_parsed[tidx] bboxes, scores, labels, masks = result[:4] if len(result) > 4: tracks = result[4] else: tracks = None else: result = results_raw[tidx] bboxes, scores, labels, masks = \ parse_det_result(result, coco_mapping, n_class) tracks = None if vis_out: img_path = join(opts.data_root, seq_dirs[sid], img['name']) I = imread(img_path) vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg') if opts.overwrite or not isfile(vis_path): if tracks is None: vis_det( I, bboxes, labels, class_names, masks, scores, out_scale=opts.vis_scale, out_file=vis_path, ) else: vis_track( I, bboxes, tracks, labels, class_names, masks, scores, out_scale=opts.vis_scale, out_file=vis_path, ) # convert to coco fmt n = len(bboxes) if n: bboxes_ltwh = ltrb2ltwh(bboxes) for i in range(n): result_dict = { 'image_id': img['id'], 'bbox': bboxes_ltwh[i], 'score': scores[i], 'category_id': labels[i], } if masks is not None: result_dict['segmentation'] = masks[i] results_ccf.append(result_dict) out_path = join(out_dir, 'results_ccf.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(results_ccf, open(out_path, 'wb')) out_path = join(out_dir, 'eval_assoc.pkl') if opts.overwrite or not isfile(out_path): pickle.dump({ 'miss': miss, 'in_time': in_time, 'mismatch': mismatch, }, open(out_path, 'wb')) if not opts.no_eval: eval_summary = eval_ccf(db, results_ccf) out_path = join(out_dir, 'eval_summary.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if opts.eval_mask: print('Evaluating instance segmentation') eval_summary = eval_ccf(db, results_ccf, iou_type='segm') out_path = join(out_dir, 'eval_summary_mask.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if vis_out: print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
import numpy as np import sys sys.path.insert(0, '..') sys.path.insert(0, '.') from util import mkdir2 from det import vis_det, imread from track import vis_track data_dir = '/data2/mengtial' data_root = join(data_dir, 'ArgoVerse1.1/tracking') split = 'val' annot_file = join(data_root, 'coco_fmt/' + split + '_track.json') show_track = True out_dir = mkdir2(join(data_root, 'coco_fmt/vis/' + split + '_track')) out_scale = 1 ## # not assuming consecutive storage imgs and annots (obj, frame, seq) dataset = json.load(open(annot_file)) cats = dataset['categories'] imgs = dataset['images'] annots = dataset['annotations'] seqs = dataset['sequences'] seq_dirs = dataset['seq_dirs'] class_names = [c['name'] for c in cats] last_sid = None
def main(): opts = parse_args() # caches = [ # join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/retina50_s0.2/val/results_ccf.pkl'), # join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.5/val/results_ccf.pkl'), # join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.75/val/results_ccf.pkl'), # ] # # should be sorted # rtfs = [1, 2, 3] # caches = [ # join(opts.exp_dir, 'ArgoVerse1.1/output/mrcnn50_nm_th0_s0.75/val/results_ccf.pkl'), # join(opts.exp_dir, 'ArgoVerse1.1/output/mrcnn50_nm_th0_s1.0/val/results_ccf.pkl'), # ] # # should be sorted # rtfs = [3, 5] caches = [ join(opts.exp_dir, 'ArgoVerse1.1-c3-eta0/output/mrcnn50_nm_s0.75/val/results_ccf.pkl'), join(opts.exp_dir, 'ArgoVerse1.1/output/cmrcnn101_nm_s1.0/val/results_ccf.pkl'), ] # should be sorted rtfs = [3, 5] n_method = len(caches) max_history = max(rtfs) cache_ccfs = [ pickle.load(open(path, 'rb')) for path in caches ] cache_end_idx = n_method*[0] mkdir2(opts.out_dir) vis_out = bool(opts.vis_dir) if vis_out: mkdir2(opts.vis_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] results_ccf = [] for sid, seq in enumerate(tqdm(seqs)): frame_list = [img for img in db.imgs.values() if img['sid'] == sid] w_img, h_img = db.imgs[0]['width'], db.imgs[0]['height'] bf = BboxFilterEx( w_img, h_img, forecast=opts.forecast, forecast_before_assoc=opts.forecast_before_assoc, ) # backward in time # cur, cur - 1, ..., cur - max_history result_buffer = (max_history + 1)*[None] result_buffer_marker = (max_history + 1)*[None] min_rtf = min(rtfs) for ii, img in enumerate(frame_list): # fetch results for i in range(n_method): ifidx = ii - rtfs[i] if ifidx < 0: break cache_end_idx[i], bboxes, scores, labels = \ result_from_ccf(cache_ccfs[i], frame_list[ifidx]['id'], cache_end_idx[i], mask=False) result_buffer[rtfs[i]] = (bboxes, scores, labels) result_buffer_marker[rtfs[i]] = i == 0 if ii < min_rtf: # no method has finished yet bboxes, scores, labels, tracks = [], [], [], [] else: s = max_history while s >= 0 and result_buffer[s] is None: s -= 1 # if first result is one step ahead t = ii - s birth_tracks = True kill_tracks = True if s == max_history: bf.update(t, *result_buffer[s], None, birth_tracks, kill_tracks) bf_this = deepcopy(bf) else: bf_this = deepcopy(bf) bf_this.update(t, *result_buffer[s], None, birth_tracks, kill_tracks) while 1: # find next non-empty result s -= 1 while s >= 0 and result_buffer[s] is None: s -= 1 if s < 0: break if result_buffer_marker[s]: birth_tracks = opts.fast_birth_tracks kill_tracks = opts.fast_kill_tracks else: birth_tracks = True kill_tracks = True t = ii - s bf_this.update(t, *result_buffer[s], None, birth_tracks, kill_tracks) bboxes, scores, labels, _, tracks = \ bf_this.predict(ii) # shift result buffer result_buffer.pop(max_history) result_buffer.insert(0, None) result_buffer_marker.pop(max_history) result_buffer_marker.insert(0, None) n = len(bboxes) for i in range(n): result_dict = { 'image_id': img['id'], 'bbox': bboxes[i], 'score': scores[i], 'category_id': labels[i], } # if masks is not None: # result_dict['segmentation'] = masks[i] results_ccf.append(result_dict) if vis_out: img_path = join(opts.data_root, seq_dirs[sid], img['name']) I = imread(img_path) vis_path = join(opts.vis_dir, seq, img['name'][:-3] + 'jpg') bboxes_ltrb = ltwh2ltrb(bboxes) if n else [] if opts.overwrite or not isfile(vis_path): vis_track( I, bboxes_ltrb, tracks, labels, class_names, None, scores, out_scale=opts.vis_scale, out_file=vis_path ) out_path = join(opts.out_dir, 'results_ccf.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(results_ccf, open(out_path, 'wb')) if not opts.no_eval: eval_summary = eval_ccf(db, results_ccf) out_path = join(opts.out_dir, 'eval_summary.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if opts.eval_mask: print('Evaluating instance segmentation') eval_summary = eval_ccf(db, results_ccf, iou_type='segm') out_path = join(opts.out_dir, 'eval_summary_mask.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if vis_out: print(f'python vis/make_videos.py "{opts.vis_dir}" --fps {opts.fps}')
def main(): assert torch.cuda.device_count( ) == 1 # mmdet only supports single GPU testing opts = parse_args() mkdir2(opts.out_dir) db = COCO(opts.annot_path) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] # CUDA runtime does not support the fork mp.set_start_method('spawn', force=True) # calculate the image size for shared memory allocation if 'max_seq_len' in db.dataset: max_seq_len = db.dataset['max_seq_len'] else: seq_len = [ len([img for img in db.imgs.values() if img['sid'] == sid]) for sid in range(len(seqs)) ] max_seq_len = max(seq_len) frame_buffer = mp.RawArray('B', max_seq_len * img_height * img_width * 3) frames_a = np.ctypeslib.as_array(frame_buffer) print(frames_a.dtype) frames_b = np.frombuffer(frame_buffer, np.uint8) print(frames_a.__array_interface__['data'][0]) print(frames_b.__array_interface__['data'][0]) frames = frames_a.reshape(-1, img_height, img_width, 3) child_recv, parent_send = mp.Pipe(False) parent_recv, child_send = mp.Pipe(False) det_proc = mp.Process(target=det_process, args=(opts, child_recv, child_send, frame_buffer)) det_proc.start() runtime_all = [] n_processed = 0 n_total = 0 t_send_frame_all = 0 t_recv_res_all = 0 init_error = parent_recv.recv() # wait till the detector is ready if isinstance(init_error, Exception): raise init_error for sid, seq in enumerate(tqdm(seqs)): # if sid != 1: # continue frame_list = [img for img in db.imgs.values() if img['sid'] == sid] # load all frames in advance for i, img in enumerate(frame_list): img_path = join(opts.data_root, seq_dirs[sid], img['name']) frames[i] = mmcv.imread(img_path) n_frame = len(frame_list) n_total += n_frame timestamps = [] results = [] input_fidx = [] runtime = [] last_fidx = None stride_cnt = 0 t_total = n_frame / opts.fps t_start = perf_counter() while 1: t1 = perf_counter() t_elapsed = t1 - t_start if t_elapsed >= t_total: break # identify latest available frame fidx = int(np.floor(t_elapsed * opts.fps)) # t_elapsed/t_total *n_frame # = t_elapsed*opts.fps if fidx == last_fidx: continue last_fidx = fidx if stride_cnt % opts.det_stride == 0: stride_cnt = 1 else: stride_cnt += 1 continue t_start_frame = perf_counter() parent_send.send((fidx, t_start_frame)) result = parent_recv.recv() # wait if isinstance(result, Exception): raise result result, t_send_frame, t_start_res = result t2 = perf_counter() t_send_frame_all += t_send_frame t_recv_res_all += t2 - t_start_res # print('recv: %.3g elapsed: %.3g' % (t2 - t_start_res, t2 - t1)) t_elapsed = t2 - t_start if t_elapsed >= t_total: break timestamps.append(t_elapsed) results.append(result) input_fidx.append(fidx) runtime.append(t2 - t1) out_path = join(opts.out_dir, seq + '.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(( results, timestamps, input_fidx, runtime, ), open(out_path, 'wb')) runtime_all += runtime n_processed += len(results) # break # terminates the child process parent_send.send(None) runtime_all_np = np.array(runtime_all) n_small_runtime = (runtime_all_np < 1.0 / opts.fps).sum() out_path = join(opts.out_dir, 'time_all.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(( runtime_all, n_processed, n_total, n_small_runtime, ), open(out_path, 'wb')) # convert to ms for display runtime_all_np *= 1e3 print(f'{n_processed}/{n_total} frames processed') print('Runtime (ms): mean: %g; std: %g; min: %g; max: %g' % ( runtime_all_np.mean(), runtime_all_np.std(ddof=1), runtime_all_np.min(), runtime_all_np.max(), )) print(f'Runtime smaller than unit time interval: ' f'{n_small_runtime}/{n_processed} ' f'({100.0*n_small_runtime/n_processed:.4g}%)') print( f'Time spent on sending the frame (ms): {1e3*t_send_frame_all/n_processed}' ) print( f'Time spent on receiving the result (ms): {1e3 *t_recv_res_all/n_processed}' )
def main(): assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing opts = parse_args() mkdir2(opts.out_dir) vis_out = bool(opts.vis_dir) if vis_out: mkdir2(opts.vis_dir) db = COCO(opts.annot_path) n_class = len(db.cats) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] if 'KMOTS' in opts.data_root: class_mapping = coco2kmots class_names = kmots_classes elif 'ArgoVerse' in opts.data_root: class_mapping = coco2av class_names = av_classes else: raise Exception('Unknown dataset') config = mmcv.Config.fromfile(opts.config) if opts.in_scale is not None: config.data.test.img_scale = opts.in_scale # mainly for SSD config.data.test.resize_keep_ratio = True if opts.no_mask: if 'mask_head' in config.model: config.model['mask_head'] = None model = init_detector(config, opts.weights) model.eval() results_ccf = [] for iid, img in tqdm(db.imgs.items()): img_name = img['name'] sid = img['sid'] seq_name = seqs[sid] img_path = join(opts.data_root, seq_dirs[sid], img_name) I = mmcv.imread(img_path) if iid % opts.det_stride == 0: result = inference_detector(model, I) bboxes, scores, labels, masks = parse_mmdet_result(result, class_mapping, n_class) if len(bboxes): bboxes_ltwh = bboxes.copy() # convert to coco fmt bboxes_ltwh[:, 2:] -= bboxes_ltwh[:, :2] bboxes_ltwh = bboxes_ltwh.tolist() else: bboxes_ltwh = [] if vis_out: vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg') if opts.overwrite or not isfile(vis_path): vis_det( I, bboxes, labels, class_names, masks, scores, out_scale=opts.vis_scale, out_file=vis_path ) for i in range(len(bboxes_ltwh)): result_dict = { 'image_id': iid, 'bbox': bboxes_ltwh[i], 'score': scores[i], 'category_id': labels[i], } if masks is not None: result_dict['segmentation'] = masks[i] results_ccf.append(result_dict) out_path = join(opts.out_dir, 'results_ccf.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(results_ccf, open(out_path, 'wb')) if not opts.no_eval: eval_summary = eval_ccf(db, results_ccf) out_path = join(opts.out_dir, 'eval_summary.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if vis_out: print(f'python vis/make_videos.py "{opts.vis_dir}"')
# This script does not need to run in real-time import argparse from os import scandir from os.path import join, isfile from tqdm import tqdm import numpy as np from PIL import Image, ImageFont, ImageDraw import sys; sys.path.insert(0, '..'); sys.path.insert(0, '.') from util import mkdir2 from vis.make_videos_numbered import worker_func as make_video_func data_dir = '/data2/mengtial' out_dir = mkdir2(join(data_dir, 'Exp/ArgoVerse1.1/vid/det_eccv')) text = [ ['Fast Detector (Baseline)', 'Detection too noisy (AP 5.5)'], ['Accurate Detector (Baseline)', 'Latency too high (AP 6.2)'], ['Optimized Detector (Ours)', 'Balanced accuracy and latency (AP 12.0)'], # ['+ Dynamic Scheduling (Ours)', 'Reducing temporal aliasing (AP 13.7)'], ] dirs = [ join(data_dir, 'Exp/ArgoVerse1.1-c3-eta0/visf-th0.5/rt_retina50_s0.2_pkt/val'), join(data_dir, 'Exp/ArgoVerse1.1-c3-eta0/visf-th0.5/rt_htc_dconv2_ms_cpupre_s1_pkt/val'), join(data_dir, 'Exp/ArgoVerse1.1-c3-eta0/visf-th0.5/rt_mrcnn50_nm_s0.5_pkt/val'), # join(data_dir, 'Exp/ArgoVerse1.1-c3-eta0/visf-th0.5/srt_mrcnn50_nm_ds_s0.5_pkt/val'), ]
def main(): assert torch.cuda.device_count() == 1 # mmdet only supports single GPU testing opts = parse_args() mkdir2(opts.out_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] n_class = len(class_names) coco_mapping = db.dataset.get('coco_mapping', None) if coco_mapping is not None: coco_mapping = np.asarray(coco_mapping) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] img = db.imgs[0] w_img, h_img = img['width'], img['height'] mp.set_start_method('spawn') frame_recv, frame_send = mp.Pipe(False) det_res_recv, det_res_send = mp.Pipe(False) det_proc = mp.Process(target=det_process, args=(opts, frame_recv, det_res_send, w_img, h_img)) det_proc.start() if opts.dynamic_schedule: runtime = pickle.load(open(opts.runtime, 'rb')) runtime_dist = dist_from_dict(runtime, opts.perf_factor) mean_rtf = runtime_dist.mean()*opts.fps n_total = 0 t_det_all = [] t_send_frame_all = [] t_recv_res_all = [] t_assoc_all = [] t_forecast_all = [] with torch.no_grad(): kf_F = torch.eye(8) kf_Q = torch.eye(8) kf_R = 10*torch.eye(4) kf_P_init = 100*torch.eye(8).unsqueeze(0) for sid, seq in enumerate(tqdm(seqs)): frame_list = [img for img in db.imgs.values() if img['sid'] == sid] frame_list = [join(opts.data_root, seq_dirs[sid], img['name']) for img in frame_list] n_frame = len(frame_list) n_total += n_frame timestamps = [] results_parsed = [] input_fidx = [] processing = False fidx_t2 = None # detection input index at t2 fidx_latest = None tkidx = 0 # track starting index kf_x = torch.empty((0, 8, 1)) kf_P = torch.empty((0, 8, 8)) n_matched12 = 0 # let detector process to read all the frames frame_send.send(frame_list) # it is possible that unfetched results remain in the pipe while 1: msg = det_res_recv.recv() # wait till the detector is ready if msg == 'ready': break elif isinstance(msg, Exception): raise msg t_total = n_frame/opts.fps t_unit = 1/opts.fps t_start = perf_counter() while 1: t1 = perf_counter() t_elapsed = t1 - t_start if t_elapsed >= t_total: break # identify latest available frame fidx_continous = t_elapsed*opts.fps fidx = int(np.floor(fidx_continous)) if fidx == fidx_latest: # algorithm is fast and has some idle time wait_for_next = True else: wait_for_next = False if opts.dynamic_schedule: if mean_rtf >= 1: # when runtime < 1, it should always process every frame fidx_remainder = fidx_continous - fidx if mean_rtf < np.floor(fidx_remainder + mean_rtf): # wait till next frame wait_for_next = True if wait_for_next: # sleep continue if not processing: t_start_frame = perf_counter() frame_send.send((fidx, t_start_frame)) fidx_latest = fidx processing = True # wait till query - forecast-rt-ub wait_time = t_unit - opts.forecast_rt_ub if det_res_recv.poll(wait_time): # wait # new result result = det_res_recv.recv() if isinstance(result, Exception): raise result result, t_send_frame, t_start_res = result bboxes_t2, scores_t2, labels_t2, _ = \ parse_det_result(result, coco_mapping, n_class) processing = False t_det_end = perf_counter() t_det_all.append(t_det_end - t_start_frame) t_send_frame_all.append(t_send_frame) t_recv_res_all.append(t_det_end - t_start_res) # associate across frames t_assoc_start = perf_counter() if len(kf_x): dt = fidx_latest - fidx_t2 kf_F = make_F(kf_F, dt) kf_Q = make_Q(kf_Q, dt) kf_x, kf_P = batch_kf_predict(kf_F, kf_x, kf_P, kf_Q) bboxes_f = x2bbox(kf_x) fidx_t2 = fidx_latest n = len(bboxes_t2) if n: # put high scores det first for better iou matching score_argsort = np.argsort(scores_t2)[::-1] bboxes_t2 = bboxes_t2[score_argsort] scores_t2 = scores_t2[score_argsort] labels_t2 = labels_t2[score_argsort] ltrb2ltwh_(bboxes_t2) updated = False if len(kf_x): order1, order2, n_matched12, tracks, tkidx = iou_assoc( bboxes_f, labels, tracks, tkidx, bboxes_t2, labels_t2, opts.match_iou_th, no_unmatched1=True, ) if n_matched12: kf_x = kf_x[order1] kf_P = kf_P[order1] kf_x, kf_P = batch_kf_update( bbox2z(bboxes_t2[order2[:n_matched12]]), kf_x, kf_P, kf_R, ) kf_x_new = bbox2x(bboxes_t2[order2[n_matched12:]]) n_unmatched2 = len(bboxes_t2) - n_matched12 kf_P_new = kf_P_init.expand(n_unmatched2, -1, -1) kf_x = torch.cat((kf_x, kf_x_new)) kf_P = torch.cat((kf_P, kf_P_new)) labels = labels_t2[order2] scores = scores_t2[order2] updated = True if not updated: # start from scratch kf_x = bbox2x(bboxes_t2) kf_P = kf_P_init.expand(len(bboxes_t2), -1, -1) labels = labels_t2 scores = scores_t2 tracks = np.arange(tkidx, tkidx + n, dtype=np.uint32) tkidx += n t_assoc_end = perf_counter() t_assoc_all.append(t_assoc_end - t_assoc_start) # apply forecasting for the current query t_forecast_start = perf_counter() query_pointer = fidx + opts.eta + 1 if len(kf_x): dt = (query_pointer - fidx_t2) kf_x_np = kf_x[:, :, 0].numpy() bboxes_t3 = kf_x_np[:n_matched12, :4] + dt*kf_x_np[:n_matched12, 4:] if n_matched12 < len(kf_x): bboxes_t3 = np.concatenate((bboxes_t3, kf_x_np[n_matched12:, :4])) bboxes_t3, keep = extrap_clean_up(bboxes_t3, w_img, h_img, lt=True) labels_t3 = labels[keep] scores_t3 = scores[keep] tracks_t3 = tracks[keep] else: bboxes_t3 = np.empty((0, 4), dtype=np.float32) scores_t3 = np.empty((0,), dtype=np.float32) labels_t3 = np.empty((0,), dtype=np.int32) tracks_t3 = np.empty((0,), dtype=np.int32) t_forecast_end = perf_counter() t_forecast_all.append(t_forecast_end - t_forecast_start) t3 = perf_counter() t_elapsed = t3 - t_start if t_elapsed >= t_total: break if len(bboxes_t3): ltwh2ltrb_(bboxes_t3) if fidx_t2 is not None: timestamps.append(t_elapsed) results_parsed.append((bboxes_t3, scores_t3, labels_t3, None, tracks_t3)) input_fidx.append(fidx_t2) out_path = join(opts.out_dir, seq + '.pkl') if opts.overwrite or not isfile(out_path): pickle.dump({ 'results_parsed': results_parsed, 'timestamps': timestamps, 'input_fidx': input_fidx, }, open(out_path, 'wb')) # terminates the child process frame_send.send(None) out_path = join(opts.out_dir, 'time_info.pkl') if opts.overwrite or not isfile(out_path): pickle.dump({ 'n_total': n_total, 't_det': t_det_all, 't_send_frame': t_send_frame_all, 't_recv_res': t_recv_res_all, 't_assoc': t_assoc_all, 't_forecast': t_forecast_all, }, open(out_path, 'wb')) # convert to ms for display s2ms = lambda x: 1e3*x print_stats(t_det_all, 'Runtime detection (ms)', cvt=s2ms) print_stats(t_send_frame_all, 'Runtime sending the frame (ms)', cvt=s2ms) print_stats(t_recv_res_all, 'Runtime receiving the result (ms)', cvt=s2ms) print_stats(t_assoc_all, "Runtime association (ms)", cvt=s2ms) print_stats(t_forecast_all, "Runtime forecasting (ms)", cvt=s2ms)
def main(): assert torch.cuda.device_count( ) == 1 # mmdet only supports single GPU testing opts = parse_args() mkdir2(opts.out_dir) vis_out = bool(opts.vis_dir) if vis_out: mkdir2(opts.vis_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] n_class = len(class_names) coco_mapping = None if opts.no_class_mapping else db.dataset.get( 'coco_mapping', None) if coco_mapping is not None: coco_mapping = np.asarray(coco_mapping) seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] model = init_detector(opts) results_raw = [] # image based, all 80 COCO classes results_ccf = [] # instance based for iid, img in tqdm(db.imgs.items()): img_name = img['name'] sid = img['sid'] seq_name = seqs[sid] img_path = join(opts.data_root, seq_dirs[sid], img_name) I = imread(img_path) result = inference_detector(model, I, gpu_pre=not opts.cpu_pre) results_raw.append(result) bboxes, scores, labels, masks = \ parse_det_result(result, coco_mapping, n_class) if vis_out: vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg') if opts.overwrite or not isfile(vis_path): vis_det(I, bboxes, labels, class_names, masks, scores, out_scale=opts.vis_scale, out_file=vis_path) # convert to coco fmt n = len(bboxes) if n: ltrb2ltwh_(bboxes) for i in range(n): result_dict = { 'image_id': iid, 'bbox': bboxes[i], 'score': scores[i], 'category_id': labels[i], } if masks is not None: result_dict['segmentation'] = masks[i] results_ccf.append(result_dict) out_path = join(opts.out_dir, 'results_raw.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(results_raw, open(out_path, 'wb')) out_path = join(opts.out_dir, 'results_ccf.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(results_ccf, open(out_path, 'wb')) if not opts.no_eval: eval_summary = eval_ccf(db, results_ccf) out_path = join(opts.out_dir, 'eval_summary.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if opts.eval_mask: print('Evaluating instance segmentation') eval_summary = eval_ccf(db, results_ccf, iou_type='segm') out_path = join(opts.out_dir, 'eval_summary_mask.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if vis_out: print(f'python vis/make_videos.py "{opts.vis_dir}"')
import csv, pickle from glob import glob from os.path import join, isfile, split import numpy as np import sys sys.path.insert(0, '..') sys.path.insert(0, '.') from util import mkdir2 data_dir = '/data2/mengtial' prefix = join(data_dir, 'Exp/ArgoVerse') out_dir = mkdir2(join(prefix, 'runtime-zoo', '1080ti')) ## Detection # methods = [ # *(4*['mrcnn_r50_no_mask']), # 'retina_r50_no_mask', # ] # scales = [ # *[0.25, 0.5, 0.75, 1], # '0.5', # ] methods = ['mrcnn_r50_no_mask_2'] scales = [0.5] for m, s in zip(methods, scales):
def main(): opts = parse_args() assert opts.forecast_before_assoc, "Not implemented" mkdir2(opts.out_dir) data_root = opts.data_root if 'DTB70' not in data_root: seqs = os.listdir(os.path.join(data_root, 'data_seq')) else: seqs = os.listdir(data_root) # db = COCO(opts.annot_path) # class_names = [c['name'] for c in db.dataset['categories']] # n_class = len(class_names) # coco_mapping = db.dataset.get('coco_mapping', None) # if coco_mapping is not None: # coco_mapping = np.asarray(coco_mapping) # seqs = db.dataset['sequences'] # seq_dirs = db.dataset['seq_dirs'] given_tracks = opts.assoc == 'given' assert not given_tracks, "Not implemented" for sid, seq in enumerate(seqs): ou_path = os.path.join(opts.out_dir) if not os.path.isdir(ou_path): os.makedirs(ou_path) result_path = os.path.join(ou_path, '{}.txt'.format(seq)) if isfile(result_path): print("Evaluated with forcasting seq: {} ({}/{}), done!".format( seq, sid + 1, len(seqs))) continue with torch.no_grad(): kf_F = torch.eye(8) kf_Q = torch.eye(8) kf_R = 10 * torch.eye(4) kf_P_init = 100 * torch.eye(8).unsqueeze(0) results_ccf = [] in_time = 0 miss = 0 shifts = 0 t_assoc = [] t_forecast = [] if 'DTB70' in data_root: frame_path = os.path.join(data_root, seq, 'img') else: frame_path = os.path.join(data_root, 'data_seq', seq) frames = os.listdir(frame_path) frame_list = [] for frame in frames: frame_list.append(os.path.join(frame_path, frame)) results = pickle.load( open(join(opts.result_root, seq + '.pkl'), 'rb')) # use raw results when possible in case we change class subset during evaluation results_raw = results['results_raw'] timestamps = results['timestamps'] timestamps[0] = 0 input_fidx = results['input_fidx'] # t1 -> det1, t2 -> det2, interpolate at t3 (t3 is the current time) det_latest_p1 = 0 # latest detection index + 1 det_t2 = None # detection index at t2 kf_x = torch.empty((0, 8, 1)) kf_P = torch.empty((0, 8, 8)) kf_F = torch.eye(8) kf_Q = torch.eye(8) kf_R = 10 * torch.eye(4) kf_P_init = 100 * torch.eye(8).unsqueeze(0) n_matched12 = 0 if not given_tracks: tkidx = 0 # track starting index for ii, img_path in enumerate(frame_list): # pred, gt association by time if ii == 0: img = cv2.imread(img_path) t = (ii - opts.eta) / opts.fps while det_latest_p1 < len( timestamps) and timestamps[det_latest_p1] <= t: det_latest_p1 += 1 if det_latest_p1 == 0: continue # no output else: det_latest = det_latest_p1 - 1 ifidx = input_fidx[det_latest] in_time += int(ii == ifidx) shifts += ii - ifidx if det_latest != det_t2: # new detection # we can now throw away old result (t1) # the old one is kept for forecasting purpose if len(kf_x) and opts.forecast_before_assoc: dt = ifidx - input_fidx[det_t2] dt = int( dt ) # convert from numpy to basic python format w_img, h_img = img.shape[1], img.shape[0] kf_F = make_F(kf_F, dt) kf_Q = make_Q(kf_Q, dt) kf_x, kf_P = batch_kf_predict( kf_F, kf_x, kf_P, kf_Q) bboxes_f = x2bbox(kf_x) det_t2 = det_latest bboxes_t2 = np.array([results_raw[det_t2]]) t1 = perf_counter() n = len(bboxes_t2) if n: updated = False if len(kf_x): order1, order2, n_matched12, tracks, tkidx = iou_assoc( bboxes_f, tracks, tkidx, bboxes_t2, opts.match_iou_th, no_unmatched1=True, ) if n_matched12: kf_x = kf_x[order1] kf_P = kf_P[order1] kf_x, kf_P = batch_kf_update( bbox2z( bboxes_t2[order2[:n_matched12]]), kf_x, kf_P, kf_R, ) kf_x_new = bbox2x( bboxes_t2[order2[n_matched12:]]) n_unmatched2 = len(bboxes_t2) - n_matched12 kf_P_new = kf_P_init.expand( n_unmatched2, -1, -1) kf_x = torch.cat((kf_x, kf_x_new)) kf_P = torch.cat((kf_P, kf_P_new)) updated = True if not updated: # start from scratch kf_x = bbox2x(bboxes_t2) kf_P = kf_P_init.expand(len(bboxes_t2), -1, -1) if not given_tracks: tracks = np.arange(tkidx, tkidx + n, dtype=np.uint32) tkidx += n t2 = perf_counter() t_assoc.append(t2 - t1) t3 = perf_counter() if len(kf_x): dt = ii - ifidx w_img, h_img = img.shape[1], img.shape[0] # PyTorch small matrix multiplication is slow # use numpy instead kf_x_np = kf_x[:, :, 0].numpy() bboxes_t3 = kf_x_np[:n_matched12, : 4] + dt * kf_x_np[:n_matched12, 4:] if n_matched12 < len(kf_x): bboxes_t3 = np.concatenate( (bboxes_t3, kf_x_np[n_matched12:, :4])) bboxes_t3, keep = extrap_clean_up(bboxes_t3, w_img, h_img, lt=True) t4 = perf_counter() t_forecast.append(t4 - t3) if len(bboxes_t3): results_ccf.append(list(bboxes_t3[0])) else: results_ccf.append(list(bboxes_t2[0])) miss += 1 # ou_path=os.path.join(opts.out_dir) # if not os.path.isdir(ou_path): # os.makedirs(ou_path) # result_path = os.path.join(ou_path, '{}.txt'.format(seq)) with open(result_path, 'w') as f: for x in results_ccf: f.write(','.join([str(i) for i in x]) + '\n') print("Evaluate with forcasting seq: {} ({}/{}), done!,miss {}".format( seq, sid + 1, len(seqs), miss))
import argparse from os import scandir from os.path import join, isfile from tqdm import tqdm import numpy as np from PIL import Image, ImageFont, ImageDraw import sys sys.path.insert(0, '..') sys.path.insert(0, '.') from util import mkdir2 from vis.make_videos_numbered import worker_func as make_video_func data_dir = '/data2/mengtial' out_dir = mkdir2(join(data_dir, 'Exp/ArgoVerse1.1/vid/compute')) text = [ ['Single GPU Final Solution', 'Optimized and predictive (AP 17.8)'], ['Infinite GPU Final Solution', 'More responsive (AP 20.3)'], ] dirs = [ join( data_dir, 'Exp/ArgoVerse1.1/visf-th0.5/pps_mrcnn50_nm_ds_s0.75_kf_fba_iou_lin_pkt/val' ), join( data_dir, 'Exp/ArgoVerse1.1/visf-th0.5/pps_mrcnn50_nm_inf_s0.75_kf_fba_iou_lin_pkt/val' ),
import numpy as np from html4vision import Col, imagetile import sys sys.path.insert(0, '..') sys.path.insert(0, '.') from util import mkdir2 data_dir = '/data2/mengtial' data_root = join(data_dir, 'ArgoVerse1.1/tracking') split = 'val' data_cfg = 'ArgoVerse1.1' out_dir = mkdir2(join(data_dir, 'Exp', data_cfg, 'visf')) out_name = 'best_single_all_seq_mask.html' title = 'Best Single GPU (Mask)' link_video = True n_show = 100 np.random.seed(0) folder = join(data_dir, 'Exp', data_cfg, 'visf-th0.5', 'pps_mrcnn50_ds_s0.5_kf_fba_iou_lin_pkt', split) print(f'python vis/make_videos.py "{folder}" --fps 30') srv_dir = data_dir srv_port = 40001 # host_name = socket.gethostname() host_name = 'trinity.vision.cs.cmu.edu'
def main(): opts = parse_args() mkdir2(opts.out_dir) vis_out = bool(opts.vis_dir) if vis_out: mkdir2(opts.vis_dir) db = COCO(opts.annot_path) class_names = [c['name'] for c in db.dataset['categories']] seqs = db.dataset['sequences'] seq_dirs = db.dataset['seq_dirs'] box_ccf = pickle.load(open(opts.box_ccf_path, 'rb')) mask_ccf = pickle.load(open(opts.mask_ccf_path, 'rb')) box_end_idx = 0 mask_end_idx = 0 results_ccf = [] # instance based for iid, img in tqdm(db.imgs.items()): img_name = img['name'] sid = img['sid'] seq_name = seqs[sid] box_start_idx = box_end_idx while box_start_idx < len(box_ccf) and box_ccf[box_start_idx]['image_id'] < img['id']: box_start_idx += 1 box_end_idx = box_start_idx while box_end_idx < len(box_ccf) and box_ccf[box_end_idx]['image_id'] == img['id']: box_end_idx += 1 box_dets = box_ccf[box_start_idx:box_end_idx] mask_start_idx = mask_end_idx while mask_start_idx < len(mask_ccf) and mask_ccf[mask_start_idx]['image_id'] < img['id']: mask_start_idx += 1 mask_end_idx = mask_start_idx while mask_end_idx < len(mask_ccf) and mask_ccf[mask_end_idx]['image_id'] == img['id']: mask_end_idx += 1 mask_dets = mask_ccf[mask_start_idx:mask_end_idx] if len(box_dets) == 0: bboxes1, scores1, labels1, masks1 = [], [], [], None elif len(mask_dets) == 0: bboxes1 = np.array([d['bbox'] for d in box_dets]) scores1 = np.array([d['score'] for d in box_dets]) labels1 = np.array([d['category_id'] for d in box_dets]) masks1 = np.array([d['segmentation'] for d in box_dets]) else: # the slow version, but works with out of order ccf results # dets = [r for r in box_ccf if r['image_id'] == img['id']] bboxes1 = np.array([d['bbox'] for d in box_dets]) scores1 = np.array([d['score'] for d in box_dets]) labels1 = np.array([d['category_id'] for d in box_dets]) masks1 = np.array([d['segmentation'] for d in box_dets]) bboxes2 = np.array([d['bbox'] for d in mask_dets]) scores2 = np.array([d['score'] for d in mask_dets]) labels2 = np.array([d['category_id'] for d in mask_dets]) masks2 = np.array([d['segmentation'] for d in mask_dets]) score_argsort = np.argsort(scores1)[::-1] bboxes1 = bboxes1[score_argsort] scores1 = scores1[score_argsort] labels1 = labels1[score_argsort] masks1 = masks1[score_argsort] score_argsort = np.argsort(scores2)[::-1] bboxes2 = bboxes2[score_argsort] scores2 = scores2[score_argsort] labels2 = labels2[score_argsort] masks2 = masks2[score_argsort] order1, order2, n_matched12, _, _ = iou_assoc( bboxes1, labels1, np.arange(len(bboxes1)), 0, bboxes2, labels2, opts.match_iou_th, no_unmatched1=False, ) bboxes1 = bboxes1[order1] scores1 = scores1[order1] labels1 = labels1[order1] masks1 = masks1[order1] bboxes2 = bboxes2[order2] scores2 = scores2[order2] labels2 = labels2[order2] masks2 = masks2[order2] mask1_fix = warp_mask_to_box( masks2[:n_matched12], bboxes2[:n_matched12], bboxes1[:n_matched12] ) masks1 = np.concatenate((mask1_fix, masks1[n_matched12:])) if vis_out: img_path = join(opts.data_root, seq_dirs[sid], img_name) I = imread(img_path) vis_path = join(opts.vis_dir, seq_name, img_name[:-3] + 'jpg') bboxes = ltwh2ltrb(bboxes1) if len(bboxes1) else [] if opts.overwrite or not isfile(vis_path): vis_det( I, bboxes, labels1, class_names, masks1, scores1, out_scale=opts.vis_scale, out_file=vis_path ) n = len(bboxes1) for i in range(n): result_dict = { 'image_id': iid, 'bbox': bboxes1[i], 'score': scores1[i], 'category_id': labels1[i], } if masks1 is not None: result_dict['segmentation'] = masks1[i] results_ccf.append(result_dict) out_path = join(opts.out_dir, 'results_ccf.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(results_ccf, open(out_path, 'wb')) if not opts.no_eval: eval_summary = eval_ccf(db, results_ccf) out_path = join(opts.out_dir, 'eval_summary.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if opts.eval_mask: print('Evaluating instance segmentation') eval_summary = eval_ccf(db, results_ccf, iou_type='segm') out_path = join(opts.out_dir, 'eval_summary_mask.pkl') if opts.overwrite or not isfile(out_path): pickle.dump(eval_summary, open(out_path, 'wb')) if vis_out: print(f'python vis/make_videos.py "{opts.vis_dir}"')