def _construct_seq_dict(self, root_dir, subset): # load task information task_file = osp.join(root_dir, 'tasks/{}.csv'.format(subset)) task = np.loadtxt(task_file, delimiter=',', dtype=str) # load dev annotations if subset == 'dev': dev_anno_file = osp.join(root_dir, 'annotations/dev.csv') dev_anno = np.loadtxt(dev_anno_file, delimiter=',', dtype=str) # construct seq_dict seq_dict = {} for s, line in enumerate(task): # parse task information vid_id, obj_id = line[:2] init_frame, last_frame = line[2:4].astype(int) init_anno = line[4:8].astype(np.float32) # log information seq_name = '_'.join([vid_id, obj_id]) if s % 50 == 0 or s + 1 == len(task): ops.sys_print('Processing sequence [%d/%d]: %s...' % (s + 1, len(task), seq_name)) # parse annotations seq_dir = osp.join(root_dir, 'images', subset, vid_id) img0 = ops.read_image(seq_dir + '/000000.jpeg') h, w = img0.shape[:2] meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'target_num': 1 } # parse and rescale initial annotations anno = np.expand_dims(init_anno[[0, 2, 1, 3]], axis=0) anno[:, [0, 2]] *= w anno[:, [1, 3]] *= h # image paths frames = np.arange(init_frame, last_frame + 1, self.frame_stride) img_files = [osp.join(seq_dir, '%06d.jpeg' % f) for f in frames] # update meta information meta.update({ 'frame_num': len(img_files), 'total_instances': len(img_files), 'frames': frames }) # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta } } return seq_dict
def _construct_seq_dict(self, root_dir): # image and annotation paths anno_files = sorted(glob.glob( osp.join(root_dir, '*/groundtruth_rect.txt'))) seq_dirs = [osp.dirname(f) for f in anno_files] seq_names = [osp.basename(d) for d in seq_dirs] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): img_files = sorted(glob.glob( osp.join(seq_dirs[s], 'img/*.jpg'))) anno = np.loadtxt(anno_files[s], delimiter=',') anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1 # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len} # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta}} return seq_dict
def forward_test(self, img_files, init_bbox, visualize=False, return_all=False): # state variables frame_num = len(img_files) if not return_all: bboxes = np.zeros((frame_num, 4)) else: bboxes = np.zeros((frame_num, 100, 4)) bboxes[0] = init_bbox times = np.zeros(frame_num) for f, img_file in enumerate(img_files): if self.input_type == 'image': img = ops.read_image(img_file, self.color_fmt) elif self.input_type == 'file': img = img_file begin = time.time() if not return_all: if f == 0: self.init(img, init_bbox) else: bboxes[f, :] = self.update(img, return_all=return_all) else: if f == 0: self.init(img, init_bbox) else: bboxes[f, ::] = self.update(img, return_all=return_all) times[f] = time.time() - begin if visualize: ops.show_image(img, bboxes[f, :4]) return bboxes, times
def _construct_seq_dict(self, root_dir, subset): # image and annotation paths seq_dirs = [] anno_files = [] if 'train' in subset: _seq_dirs = sorted( glob.glob( osp.join(root_dir, 'VisDrone2018-VID-train/sequences/*_v'))) _anno_files = [ osp.join(root_dir, 'VisDrone2018-VID-train/annotations', osp.basename(s) + '.txt') for s in _seq_dirs ] seq_dirs += _seq_dirs anno_files += _anno_files if 'val' in subset: _seq_dirs = sorted( glob.glob( osp.join(root_dir, 'VisDrone2018-VID-val/sequences/*_v'))) _anno_files = [ osp.join(root_dir, 'VisDrone2018-VID-val/annotations', osp.basename(s) + '.txt') for s in _seq_dirs ] seq_dirs += _seq_dirs anno_files += _anno_files seq_names = [osp.basename(s) for s in seq_dirs] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): if s % 10 == 0 or s + 1 == len(seq_names): ops.sys_print('Processing [%d/%d]: %s' % (s + 1, len(seq_names), seq_name)) img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg'))) anno_s = np.loadtxt(anno_files[s], delimiter=',', dtype=np.float32) anno_s = self._format(anno_s) # meta information img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': len(img_files), 'target_num': len(set(anno_s[:, 1])), 'total_instances': len(anno_s) } # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno_s, 'meta': meta } } return seq_dict
def _construct_seq_dict(self, root_dir, version): # image and annotation paths valid_seqs = self.__version_dict[version] anno_files = sorted(list(chain.from_iterable(glob.glob( osp.join(root_dir, s, 'groundtruth*.txt')) for s in valid_seqs))) # remove empty annotation files # (e.g., groundtruth_rect.1.txt of Human4) anno_files = self._filter_files(anno_files) seq_dirs = [osp.dirname(f) for f in anno_files] seq_names = [osp.basename(d) for d in seq_dirs] # rename repeated sequence names # (e.g., Jogging and Skating2) seq_names = self._rename_seqs(seq_names) # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): img_files = sorted(glob.glob( osp.join(seq_dirs[s], 'img/*.jpg'))) # special sequences # (visit http://cvlab.hanyang.ac.kr/tracker_benchmark/index.html for detail) if seq_name.lower() == 'david': img_files = img_files[300-1:770] elif seq_name.lower() == 'football1': img_files = img_files[:74] elif seq_name.lower() == 'freeman3': img_files = img_files[:460] elif seq_name.lower() == 'freeman4': img_files = img_files[:283] elif seq_name.lower() == 'diving': img_files = img_files[:215] # load annotations (to deal with different delimeters) with open(anno_files[s], 'r') as f: anno = np.loadtxt(io.StringIO(f.read().replace(',', ' '))) anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1 # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len} # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta}} return seq_dict
def _construct_seq_dict(self, root_dir, version): # sequence meta information meta_file = osp.join(osp.dirname(__file__), 'uav123.json') with open(meta_file) as f: seq_metas = json.load(f) # image and annotation paths anno_files = sorted( glob.glob(osp.join(root_dir, 'anno/%s/*.txt' % version))) seq_names = [osp.basename(f)[:-4] for f in anno_files] seq_dirs = [osp.join( root_dir, 'data_seq/UAV123/%s' % \ seq_metas[version][n]['folder_name']) for n in seq_names] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): # valid frame range start_frame = seq_metas[version][seq_names[s]]['start_frame'] end_frame = seq_metas[version][seq_names[s]]['end_frame'] img_files = [ osp.join(seq_dirs[s], '%06d.jpg' % f) for f in range(start_frame, end_frame + 1) ] # load annotations anno = np.loadtxt(anno_files[s], delimiter=',') anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1 # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len } # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta } } return seq_dict
def show(self, tracker_names, seq_names=None, play_speed=1, visualize=True, save=False, save_dir='screenshots'): if seq_names is None: seq_names = self.dataset.seq_names elif isinstance(seq_names, str): seq_names = [seq_names] assert isinstance(tracker_names, (list, tuple)) assert isinstance(seq_names, (list, tuple)) play_speed = int(round(play_speed)) assert play_speed > 0 for s, seq_name in enumerate(seq_names): ops.sys_print('[%d/%d] Showing results on %s...' % (s + 1, len(seq_names), seq_name)) # mkdir if required to save screenshots if save: out_dir = osp.join(save_dir, seq_name) if not osp.exists(out_dir): os.makedirs(out_dir) # load all tracking results records = {} for name in tracker_names: record_file = osp.join(self.result_dir, name, '%s.txt' % seq_name) records[name] = np.loadtxt(record_file, delimiter=',') # loop over the sequence and display results img_files, target = self.dataset[seq_name][:2] img_files = img_files[::self.frame_stride] target['anno'] = target['anno'][self.frame_stride] for f, img_file in enumerate(img_files): if not f % play_speed == 0: continue img = ops.read_image(img_file) bboxes = [records[name][f] for name in tracker_names] if len(target['anno']) > f: bboxes = [target['anno'][f]] + bboxes img = ops.show_image(img, bboxes, visualize=visualize) # save screenshot if required if save: out_file = osp.join(out_dir, '%08d.jpg' % (f + 1)) cv2.imwrite(out_file, img)
def _construct_seq_dict(self, root_dir, fps): # image and annotation paths anno_files = sorted(glob.glob(osp.join(root_dir, '*/%d/*.txt' % fps))) seq_names = [osp.basename(f)[:-4] for f in anno_files] seq_dirs = [ osp.join(osp.dirname(f), n) for f, n in zip(anno_files, seq_names) ] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): if s % 50 == 0 or s + 1 == len(seq_names): ops.sys_print('Processing sequence [%d/%d]: %s...' % (s + 1, len(seq_names), seq_name)) img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg'))) anno = np.loadtxt(anno_files[s], dtype=str) anno = anno[:, 1:5].astype(np.float32) # handle inconsistent lengths if not len(img_files) == len(anno): if abs(len(anno) / len(img_files) - 8) < 1: anno = anno[0::8, :] diff = abs(len(img_files) - len(anno)) if diff > 0 and diff <= 1: n = min(len(img_files), len(anno)) anno = anno[:n] img_files = img_files[:n] assert len(img_files) == len(anno) # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len } # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta } } return seq_dict
def _construct_seq_dict(self, root_dir, subset): # load subset sequence names split_file = osp.join(osp.dirname(__file__), 'lasot.json') with open(split_file, 'r') as f: splits = json.load(f) seq_names = splits[subset] # image and annotation paths seq_dirs = [ osp.join(root_dir, n[:n.rfind('-')], n, 'img') for n in seq_names ] anno_files = [ osp.join(root_dir, n[:n.rfind('-')], n, 'groundtruth.txt') for n in seq_names ] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): if s % 100 == 0 or s + 1 == len(seq_names): ops.sys_print('Processing sequence [%d/%d]: %s...' % (s + 1, len(seq_names), seq_name)) img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg'))) anno = np.loadtxt(anno_files[s], delimiter=',', dtype=np.float32) anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1 # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = self._fetch_meta(seq_dirs[s]) meta.update({ 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len }) # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta } } return seq_dict
def _construct_seq_dict(self, root_dir, list_file): # image and annotation paths with open(list_file, 'r') as f: seq_names = f.read().strip().split('\n') seq_dirs = [osp.join(root_dir, s, 'color') for s in seq_names] if os.path.exists( osp.join(root_dir, seq_names[0], 'annotations', 'groundtruth.txt')): anno_files = [ osp.join(root_dir, s, 'annotations', 'groundtruth.txt') for s in seq_names ] else: anno_files = [ osp.join(root_dir, s, 'annotation', 'groundtruth.txt') for s in seq_names ] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg'))) anno = np.loadtxt(anno_files[s], delimiter=',') anno = self._format(anno) # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = self._fetch_meta(seq_dirs[s], seq_len) meta.update({ 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len }) # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta } } return seq_dict
def _construct_seq_dict(self, root_dir, subset_dirs): # image and annotation paths anno_files = [ glob.glob(osp.join(root_dir, c, 'anno/*.txt')) for c in subset_dirs ] anno_files = sorted(sum(anno_files, [])) seq_dirs = [ osp.join(osp.dirname(osp.dirname(f)), 'frames', osp.basename(f)[:-4]) for f in anno_files ] seq_names = [osp.basename(d) for d in seq_dirs] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): if s % 100 == 0 or s + 1 == len(seq_names): ops.sys_print('Processing sequence [%d/%d]: %s...' % (s + 1, len(seq_names), seq_name)) img_files = glob.glob(osp.join(seq_dirs[s], '*.jpg')) img_files = sorted(img_files, key=lambda f: int(osp.basename(f)[:-4])) anno = np.loadtxt(anno_files[s], delimiter=',') if anno.ndim == 1: anno = np.expand_dims(anno, axis=0) anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1 # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len } # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta } } return seq_dict
def _construct_seq_dict(self, root_dir, version, subset): # image and annotation paths seq_dirs = sorted(glob.glob( osp.join(root_dir, subset, '*/img1'))) parent_dirs = [osp.dirname(d) for d in seq_dirs] seq_names = [osp.basename(d) for d in parent_dirs] det_files = [osp.join( d, 'det/det.txt') for d in parent_dirs] if subset == 'train': gt_files = [osp.join( d, 'gt/gt.txt') for d in parent_dirs] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): img_files = sorted(glob.glob( osp.join(seq_dirs[s], '*.jpg'))) det = np.loadtxt( det_files[s], delimiter=',', dtype=np.float32) det = self._format_det(det) target = {'det': det} if subset == 'train': gt = np.loadtxt( gt_files[s], delimiter=',', dtype=np.float32) gt = self._format_gt(gt) target.update({'anno': gt}) # meta information img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': len(img_files), 'target_num': -1 if subset != 'train' else len(set(gt[:, 1])), 'total_instances': -1 if subset != 'train' else len(gt)} target.update({'meta': meta}) # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': target} return seq_dict
def __getitem__(self, index): r""" Args: index (integer): Index of an image. Returns: img, target (tuple): where ``target`` is dict of annotations. """ img_name = self.img_names[index] img_info = self.img_dict[img_name] # parse image and annotations img = ops.read_image(img_info['img_file']) target = img_info['target'] if hasattr(self, 'transforms') and self.transforms is not None: img, target = self.transforms(img, target) return img, target
def _construct_seq_dict(self, root_dir, subset): # image and annotation paths if subset == 'test': anno_files = sorted( glob.glob(osp.join(root_dir, 'initialization/*_s.txt'))) else: anno_files = sorted( glob.glob(osp.join(root_dir, 'annotations/*_s.txt'))) seq_names = [osp.basename(f)[:-4] for f in anno_files] seq_dirs = [ osp.join(root_dir, 'sequences/{}'.format(n)) for n in seq_names ] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): img_files = sorted(glob.glob(osp.join(seq_dirs[s], 'img*.jpg'))) anno = np.loadtxt(anno_files[s], delimiter=',') if anno.ndim == 1: assert anno.size == 4 anno = anno[np.newaxis, :] anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1 # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len } # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta } } return seq_dict
def _construct_seq_dict(self, root_dir): # image and annotation paths seq_dirs = sorted(glob.glob( osp.join(root_dir, '*/*_*/'))) seq_dirs = [d[:-1] for d in seq_dirs] seq_names = [osp.basename(d) for d in seq_dirs] anno_files = [osp.join( root_dir, 'annotation/annotation/{}_gt_points.txt'.format(n)) for n in seq_names] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): if s % 50 == 0 or s + 1 == len(seq_names): ops.sys_print('Processing sequence [%d/%d]: %s...' % ( s + 1, len(seq_names), seq_name)) img_files = sorted(glob.glob( osp.join(seq_dirs[s], '*.jpg'))) anno = np.loadtxt(anno_files[s]) n = min(len(img_files), len(anno)) assert n > 0 img_files = img_files[:n] anno = anno[:n] # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len} # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta}} return seq_dict
def _construct_seq_dict(self, root_dir): # image and annotation paths anno_files = sorted(glob.glob( osp.join(root_dir, '*/*_gt.txt'))) seq_dirs = [osp.dirname(f) for f in anno_files] seq_names = [osp.basename(d) for d in seq_dirs] # valid frame range for each sequence range_files = [glob.glob( osp.join(d, '*_frames.txt'))[0] for d in seq_dirs] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): # load valid frame range frames = np.loadtxt( range_files[s], dtype=int, delimiter=',') img_files = [osp.join( seq_dirs[s], 'img/%04d.jpg' % f) for f in range(frames[0], frames[1] + 1)] # load annotations anno = np.loadtxt(anno_files[s], delimiter=',') anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1 # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len} # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta}} return seq_dict
def _construct_seq_dict(self, root_dir, subset, list_file): # image and annotation paths with open(list_file, 'r') as f: seq_names = f.read().strip().split('\n') seq_dirs = [osp.join(root_dir, subset, s) for s in seq_names] anno_files = [osp.join(d, 'groundtruth.txt') for d in seq_dirs] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): if s % 100 == 0 or s + 1 == len(seq_names): ops.sys_print('Processing sequence [%d/%d]: %s...' % (s + 1, len(seq_names), seq_name)) img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg'))) anno = np.loadtxt(anno_files[s], delimiter=',', dtype=np.float32) if anno.ndim == 1: assert anno.size == 4 anno = anno[np.newaxis, :] anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1 # meta information seq_len = len(img_files) img0 = ops.read_image(img_files[0]) meta = self._fetch_meta(seq_dirs[s]) meta.update({ 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': seq_len, 'target_num': 1, 'total_instances': seq_len }) # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno, 'meta': meta } } return seq_dict
def forward_test(self, img_files, init_bbox, visualize=False): frame_num = len(img_files) times = np.zeros(frame_num) preds = [{ 'present': True, 'score': 1.0, 'xmin': init_bbox[0], 'xmax': init_bbox[2], 'ymin': init_bbox[1], 'ymax': init_bbox[3] }] for f, img_file in enumerate(img_files): if self.input_type == 'image': img = ops.read_image(img_file, self.color_fmt) elif self.input_type == 'file': img = img_file begin = time.time() if f == 0: self.init(img, init_bbox) else: bbox, score, present = self.update(img) preds.append({ 'present': present, 'score': score, 'xmin': bbox[0], 'xmax': bbox[2], 'ymin': bbox[1], 'ymax': bbox[3] }) times[f] = time.time() - begin if visualize: ops.show_image(img, bboxes[f, :]) # update the preds as one-per-second frame_stride = 30 preds = {f * frame_stride: pred for f, pred in enumerate(preds)} return preds, times
def _check_dataset(self, dataset): seq_num = len(dataset) if seq_num == 0: ops.sys_print('Warning: {} dataset is empty, ' 'skip test...'.format(dataset.name)) return self.assertGreater(seq_num, 0) ops.sys_print('{} contains {} sequences'.format(dataset.name, seq_num)) # sanity check inds = random.sample(range(seq_num), min(seq_num, 10)) for i in inds: img_files, target = dataset[i] anno, meta = target['anno'], target['meta'] if anno.shape[0] == 1: continue # test sets if anno.shape[1] in [4, 8]: self.assertEqual(len(img_files), len(anno)) else: self.assertGreaterEqual(len(img_files) - 1, anno[:, 0].max()) # visualization if self.visualize: img_files, target = random.choice(dataset) anno = target['anno'] for f, img_file in enumerate(img_files): if f >= anno.shape[0]: break # test sets if anno.shape[1] == 9: bboxes = anno[anno[:, 0] == f, 2:6] elif anno.shape[1] == 8: break # TODO: points are not supported yet else: bboxes = anno[f, :] img = ops.read_image(img_file) ops.show_image(img, bboxes, delay=1)
def _construct_seq_dict(self, root_dir, subset): # image and annotation paths seq_dirs = [] anno_dirs = [] if 'train' in subset: _seq_dirs = sorted( glob.glob(osp.join(root_dir, 'Data/VID/train/ILSVRC*/ILSVRC*'))) _anno_dirs = [ osp.join(root_dir, 'Annotations/VID/train', *s.split('/')[-2:]) for s in _seq_dirs ] seq_dirs += _seq_dirs anno_dirs += _anno_dirs if 'val' in subset: _seq_dirs = sorted( glob.glob(osp.join(root_dir, 'Data/VID/val/ILSVRC2015_val_*'))) _anno_dirs = [ osp.join(root_dir, 'Annotations/VID/val', s.split('/')[-1]) for s in _seq_dirs ] seq_dirs += _seq_dirs anno_dirs += _anno_dirs seq_names = [osp.basename(s) for s in seq_dirs] # construct seq_dict seq_dict = {} for s, seq_name in enumerate(seq_names): if s % 100 == 0 or s + 1 == len(seq_names): ops.sys_print('Processing sequence [%d/%d]: %s...' % (s + 1, len(seq_names), seq_name)) # parse XML annotation files anno_files = sorted(glob.glob(osp.join(anno_dirs[s], '*.xml'))) objects = [ ET.ElementTree(file=f).findall('object') for f in anno_files ] anno_s = [] for f, group in enumerate(objects): for obj in group: anno_obj = [ f, # frame_id int(obj.find('trackid').text), # target_id int(obj.find('bndbox/xmin').text), # x1 int(obj.find('bndbox/ymin').text), # y1 int(obj.find('bndbox/xmax').text), # x2 int(obj.find('bndbox/ymax').text), # y2 1, # score/keep self.WORDNET_IDs.index( obj.find('name').text), # class_id int(obj.find('occluded').text) ] # occlusion anno_s.append(anno_obj) anno_s = np.array(anno_s, dtype=np.float32) # meta information img_files = [ osp.join(seq_dirs[s], '%06d.JPEG' % f) for f in range(len(objects)) ] img0 = ops.read_image(img_files[0]) meta = { 'width': img0.shape[1], 'height': img0.shape[0], 'frame_num': len(img_files), 'target_num': len(set(anno_s[:, 1])), 'total_instances': len(anno_s) } # update seq_dict seq_dict[seq_name] = { 'img_files': img_files, 'target': { 'anno': anno_s, 'meta': meta } } return seq_dict
def __getitem__(self, index): if index > len(self): raise IndexError('Index out of range') index %= len(self.seqs) # get image files and annotations img_files, target = self.seqs[index] anno, meta = target['anno'], target['meta'] # filter noisy annotations valid_indices = self._filter(anno, meta) if len(valid_indices) < 2: return self._random_next() # ramdomly sample a pair rand_z, rand_x = self._sample_pair(anno, valid_indices) if rand_z < 0 or rand_x < 0: return self._random_next() # annotations pair if anno.shape[1] in [4, 8]: # SOT-based pairs bboxes_z = np.expand_dims(anno[rand_z], axis=0) bboxes_x = np.expand_dims(anno[rand_x], axis=0) elif anno.shape[1] == 9: # MOT/VID-based pairs # find consistent track_ids mask_z = (anno[:, 0] == rand_z) mask_x = (anno[:, 0] == rand_x) z_ids, x_ids = anno[mask_z, 1], anno[mask_x, 1] join_ids = sorted(set(z_ids) & set(x_ids)) # extract bounding boxes (ignore the class label) bboxes_z, bboxes_x = [], [] for track_id in join_ids: mask_id = (anno[:, 1] == track_id) mask_id_z = (mask_z & mask_id) mask_id_x = (mask_x & mask_id) if mask_id_z.sum() != 1 or mask_id_x.sum() != 1: ops.sys_print('Warning: found repeated ID for', self.seqs.seq_names[index]) return self._random_next() bboxes_z += [anno[mask_id_z, 2:6]] bboxes_x += [anno[mask_id_x, 2:6]] bboxes_z = np.concatenate(bboxes_z, axis=0) bboxes_x = np.concatenate(bboxes_x, axis=0) assert len(bboxes_z) == len(bboxes_x) == len(join_ids) # image pair img_z = ops.read_image(img_files[rand_z]) img_x = ops.read_image(img_files[rand_x]) # bound annotations by image boundaries h, w = img_z.shape[:2] bboxes_z = ops.bound_bboxes(bboxes_z, [w, h]) bboxes_x = ops.bound_bboxes(bboxes_x, [w, h]) # build target target = {'bboxes_z': bboxes_z, 'bboxes_x': bboxes_x} # apply transforms if applicable if self.transforms is not None: return self.transforms(img_z, img_x, target) else: return img_z, img_x, target
def run_realtime(self, tracker, visualize=False): ops.sys_print('Running real-time experiment...') # loop over the complete dataset for s, (img_files, target) in enumerate(self.dataset): seq_name = self.dataset.seq_names[s] ops.sys_print('--Sequence %d/%d: %s' % (s + 1, len(self.dataset), seq_name)) # skip if results exist record_file = os.path.join(self.result_dir, tracker.name, 'realtime', seq_name, '%s_001.txt' % seq_name) if os.path.exists(record_file): ops.sys_print(' Found results, skipping %s' % seq_name) continue # rectangular bounding boxes anno = target['anno'] anno_rects = anno.copy() if anno_rects.shape[1] == 8: anno_rects = self.dataset._corner2rect(anno_rects) # state variables bboxes = [] times = [] next_start = 0 failure = False failed_frame = -1 total_time = 0.0 grace = 3 - 1 offset = 0 # tracking loop for f, img_file in enumerate(img_files): img = ops.read_image(img_file) if tracker.input_type == 'image': frame = img elif tracker.input_type == 'file': frame = img_file start_time = time.time() if f == next_start: # during initial frames tracker.init(frame, anno_rects[f]) bboxes.append([1]) # reset state variables failure = False failed_frame = -1 total_time = 0.0 grace = 3 - 1 offset = f elif not failure: # during success frames # calculate current frame if grace > 0: total_time += 1000.0 / 25 grace -= 1 else: total_time += max(1000.0 / 25, last_time * 1000.0) current = offset + int( np.round(np.floor(total_time * 25) / 1000.0)) # delayed/tracked bounding box if f < current: bbox = bboxes[-1] elif f == current: bbox = tracker.update(frame) iou = ops.poly_iou(anno[f], bbox, bound=img.shape[1::-1]) if iou <= 0.0: # tracking failure failure = True failed_frame = f next_start = current + self.skip_initialize bboxes.append([2]) else: # tracking succeed bboxes.append(bbox) else: # during failure frames if f < current: # skipping frame due to slow speed bboxes.append([0]) start_time = np.NaN elif f == current: # current frame bbox = tracker.update(frame) iou = ops.poly_iou(anno[f], bbox, bound=img.shape[1::-1]) if iou <= 0.0: # tracking failure bboxes.append([2]) bboxes[failed_frame] = [0] times[failed_frame] = np.NaN else: # tracking succeed bboxes.append(bbox) elif f < next_start: # skipping frame due to failure bboxes.append([0]) start_time = np.NaN # store elapsed time last_time = time.time() - start_time times.append(last_time) # visualize if required if visualize: if len(bboxes[-1]) == 4: ops.show_image(img, bboxes[-1]) else: ops.show_image(img) # record results self._record(record_file, bboxes, times)
def run_supervised(self, tracker, visualize=False): ops.sys_print('Running supervised experiment...') # loop over the complete dataset for s, (img_files, target) in enumerate(self.dataset): seq_name = self.dataset.seq_names[s] ops.sys_print('--Sequence %d/%d: %s' % (s + 1, len(self.dataset), seq_name)) # rectangular bounding boxes anno = target['anno'] anno_rects = anno.copy() if anno_rects.shape[1] == 8: anno_rects = self.dataset._corner2rect(anno_rects) # run multiple repetitions for each sequence for r in range(self.repetitions): # check if the tracker is deterministic if r > 0 and tracker.is_deterministic: break elif r == 3 and self._check_deterministic( 'baseline', tracker.name, seq_name): ops.sys_print(' Detected a deterministic tracker, ' 'skipping remaining trials.') break ops.sys_print(' Repetition: %d' % (r + 1)) # skip if results exist record_file = os.path.join(self.result_dir, tracker.name, 'baseline', seq_name, '%s_%03d.txt' % (seq_name, r + 1)) if os.path.exists(record_file): ops.sys_print(' Found results, skipping %s' % seq_name) continue # state variables bboxes = [] times = [] failure = False next_start = -1 # tracking loop for f, img_file in enumerate(img_files): img = ops.read_image(img_file) if tracker.input_type == 'image': frame = img elif tracker.input_type == 'file': frame = img_file start_time = time.time() if f == 0: # initial frame tracker.init(frame, anno_rects[0]) bboxes.append([1]) elif failure: # during failure frames if f == next_start: if np.all(anno_rects[f] <= 0): next_start += 1 start_time = np.NaN bboxes.append([0]) else: failure = False tracker.init(frame, anno_rects[f]) bboxes.append([1]) else: start_time = np.NaN bboxes.append([0]) else: # during success frames bbox = tracker.update(frame) iou = ops.poly_iou(anno[f], bbox, bound=img.shape[1::-1]) if iou <= 0.0: # tracking failure failure = True next_start = f + self.skip_initialize bboxes.append([2]) else: # tracking succeed bboxes.append(bbox) # store elapsed time times.append(time.time() - start_time) # visualize if required if visualize: if len(bboxes[-1]) == 4: ops.show_image(img, bboxes[-1]) else: ops.show_image(img) # record results self._record(record_file, bboxes, times)
def show(self, tracker_names, seq_names=None, play_speed=1, experiment='supervised', visualize=True, save=False, save_dir='screenshots'): if seq_names is None: seq_names = self.dataset.seq_names elif isinstance(seq_names, str): seq_names = [seq_names] assert isinstance(tracker_names, (list, tuple)) assert isinstance(seq_names, (list, tuple)) assert experiment in ['supervised', 'unsupervised', 'realtime'] play_speed = int(round(play_speed)) assert play_speed > 0 # "supervised" experiment results are stored in "baseline" folder if experiment == 'supervised': experiment = 'baseline' # function for loading results def read_record(filename): with open(filename) as f: record = f.read().strip().split('\n') record = [[float(t) for t in line.split(',')] for line in record] for i, r in enumerate(record): if len(r) == 4: record[i] = np.array(r) elif len(r) == 8: r = np.array(r)[np.newaxis, :] r = self.dataset._corner2rect(r) record[i] = r[0] else: record[i] = np.zeros(4) return record for s, seq_name in enumerate(seq_names): ops.sys_print('[%d/%d] Showing results on %s...' % (s + 1, len(seq_names), seq_name)) # mkdir if required to save screenshots if save: out_dir = os.path.join(save_dir, seq_name) if not os.path.exists(out_dir): os.makedirs(out_dir) # load all tracking results records = {} for name in tracker_names: record_file = os.path.join(self.result_dir, name, experiment, seq_name, '%s_001.txt' % seq_name) records[name] = read_record(record_file) # loop over the sequence and display results img_files, target = self.dataset[seq_name][:2] anno = target['anno'] if anno.shape[1] == 8: anno = self.dataset._corner2rect(anno) for f, img_file in enumerate(img_files): if not f % play_speed == 0: continue img = ops.read_image(img_file) bboxes = [anno[f] ] + [records[name][f] for name in tracker_names] img = ops.show_image(img, bboxes, visualize=visualize) # save screenshot if required if save: out_file = os.path.join(out_dir, '%08d.jpg' % (f + 1)) cv2.imwrite(out_file, img)
def report(self, tracker_names, plot_curves=False): if isinstance(tracker_names, str): tracker_names = [tracker_names] assert isinstance(tracker_names, (list, tuple)) # function for loading results def read_record(filename): with open(filename) as f: record = f.read().strip().split('\n') record = [[float(t) for t in line.split(',')] for line in record] return record # assume tracker_names[0] is your tracker report_dir = os.path.join(self.report_dir, tracker_names[0]) if not os.path.exists(report_dir): os.makedirs(report_dir) report_file = os.path.join(report_dir, 'performance.json') performance = {} for name in tracker_names: ops.sys_print('Evaluating %s' % name) ious = {} ious_full = {} failures = {} times = {} masks = {} # frame masks for attribute tags for s, (img_files, target) in enumerate(self.dataset): anno, meta = target['anno'], target['meta'] seq_name = self.dataset.seq_names[s] # initialize frames scores frame_num = len(img_files) ious[seq_name] = np.full((self.repetitions, frame_num), np.nan, dtype=float) ious_full[seq_name] = np.full((self.repetitions, frame_num), np.nan, dtype=float) failures[seq_name] = np.full((self.repetitions, frame_num), np.nan, dtype=float) times[seq_name] = np.full((self.repetitions, frame_num), np.nan, dtype=float) # read results of all repetitions record_files = sorted( glob.glob( os.path.join(self.result_dir, name, 'baseline', seq_name, '%s_[0-9]*.txt' % seq_name))) bboxes = [read_record(f) for f in record_files] assert all([len(b) == len(anno) for b in bboxes]) # calculate frame ious with burnin bound = ops.read_image(img_files[0]).shape[1::-1] seq_ious = [ self._calc_iou(b, anno, bound, burnin=True) for b in bboxes ] ious[seq_name][:len(seq_ious), :] = seq_ious # calculate frame ious without burnin seq_ious_full = [ self._calc_iou(b, anno, bound) for b in bboxes ] ious_full[seq_name][:len(seq_ious_full), :] = seq_ious_full # calculate frame failures seq_failures = [[ len(b) == 1 and b[0] == 2 for b in bboxes_per_rep ] for bboxes_per_rep in bboxes] failures[seq_name][:len(seq_failures), :] = seq_failures # collect frame runtimes time_file = os.path.join(self.result_dir, name, 'baseline', seq_name, '%s_time.txt' % seq_name) if os.path.exists(time_file): seq_times = np.loadtxt(time_file, delimiter=',').T times[seq_name][:len(seq_times), :] = seq_times # collect attribute masks tag_num = len(self.tags) masks[seq_name] = np.zeros((tag_num, frame_num), bool) for i, tag in enumerate(self.tags): if tag in meta: masks[seq_name][i, :] = meta[tag] # frames with no tags if 'empty' in self.tags: tag_frames = np.array([ v for k, v in meta.items() if isinstance(v, np.ndarray) and \ not 'practical' in k], dtype=bool) ind = self.tags.index('empty') masks[seq_name][ind, :] = \ ~np.logical_or.reduce(tag_frames, axis=0) # concatenate frames seq_names = self.dataset.seq_names masks = np.concatenate([masks[s] for s in seq_names], axis=1) ious = np.concatenate([ious[s] for s in seq_names], axis=1) failures = np.concatenate([failures[s] for s in seq_names], axis=1) with warnings.catch_warnings(): # average over repetitions warnings.simplefilter('ignore', category=RuntimeWarning) ious = np.nanmean(ious, axis=0) failures = np.nanmean(failures, axis=0) # calculate average overlaps and failures for each tag tag_ious = np.array([np.nanmean(ious[m]) for m in masks]) tag_failures = np.array( [np.nansum(failures[m]) for m in masks]) tag_frames = masks.sum(axis=1) # remove nan values tag_ious[np.isnan(tag_ious)] = 0.0 tag_weights = tag_frames / tag_frames.sum() # calculate weighted accuracy and robustness accuracy = np.sum(tag_ious * tag_weights) robustness = np.sum(tag_failures * tag_weights) # calculate tracking speed times = np.concatenate([t.reshape(-1) for t in times.values()]) # remove invalid values times = times[~np.isnan(times)] times = times[times > 0] if len(times) > 0: speed = np.mean(1. / times) else: speed = -1 performance.update({ name: { 'accuracy': accuracy, 'robustness': robustness, 'speed_fps': speed } }) # save performance with open(report_file, 'w') as f: json.dump(performance, f, indent=4) ops.sys_print('Performance saved at %s' % report_file) return performance