def _construct_seq_dict(self, root_dir, subset):
# load task information
task_file = osp.join(root_dir, 'tasks/{}.csv'.format(subset))
task = np.loadtxt(task_file, delimiter=',', dtype=str)
# load dev annotations
if subset == 'dev':
dev_anno_file = osp.join(root_dir, 'annotations/dev.csv')
dev_anno = np.loadtxt(dev_anno_file, delimiter=',', dtype=str)
# construct seq_dict
seq_dict = {}
for s, line in enumerate(task):
# parse task information
vid_id, obj_id = line[:2]
init_frame, last_frame = line[2:4].astype(int)
init_anno = line[4:8].astype(np.float32)
# log information
seq_name = '_'.join([vid_id, obj_id])
if s % 50 == 0 or s + 1 == len(task):
ops.sys_print('Processing sequence [%d/%d]: %s...' %
(s + 1, len(task), seq_name))
# parse annotations
seq_dir = osp.join(root_dir, 'images', subset, vid_id)
img0 = ops.read_image(seq_dir + '/000000.jpeg')
h, w = img0.shape[:2]
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'target_num': 1
}
# parse and rescale initial annotations
anno = np.expand_dims(init_anno[[0, 2, 1, 3]], axis=0)
anno[:, [0, 2]] *= w
anno[:, [1, 3]] *= h
# image paths
frames = np.arange(init_frame, last_frame + 1, self.frame_stride)
img_files = [osp.join(seq_dir, '%06d.jpeg' % f) for f in frames]
# update meta information
meta.update({
'frame_num': len(img_files),
'total_instances': len(img_files),
'frames': frames
})
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta
}
}
return seq_dict
def _construct_seq_dict(self, root_dir):
# image and annotation paths
anno_files = sorted(glob.glob(
osp.join(root_dir, '*/groundtruth_rect.txt')))
seq_dirs = [osp.dirname(f) for f in anno_files]
seq_names = [osp.basename(d) for d in seq_dirs]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
img_files = sorted(glob.glob(
osp.join(seq_dirs[s], 'img/*.jpg')))
anno = np.loadtxt(anno_files[s], delimiter=',')
anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta}}
return seq_dict
def forward_test(self, img_files, init_bbox, visualize=False, return_all=False):
# state variables
frame_num = len(img_files)
if not return_all:
bboxes = np.zeros((frame_num, 4))
else:
bboxes = np.zeros((frame_num, 100, 4))
bboxes[0] = init_bbox
times = np.zeros(frame_num)
for f, img_file in enumerate(img_files):
if self.input_type == 'image':
img = ops.read_image(img_file, self.color_fmt)
elif self.input_type == 'file':
img = img_file
begin = time.time()
if not return_all:
if f == 0:
self.init(img, init_bbox)
else:
bboxes[f, :] = self.update(img, return_all=return_all)
else:
if f == 0:
self.init(img, init_bbox)
else:
bboxes[f, ::] = self.update(img, return_all=return_all)
times[f] = time.time() - begin
if visualize:
ops.show_image(img, bboxes[f, :4])
return bboxes, times
def _construct_seq_dict(self, root_dir, subset):
# image and annotation paths
seq_dirs = []
anno_files = []
if 'train' in subset:
_seq_dirs = sorted(
glob.glob(
osp.join(root_dir,
'VisDrone2018-VID-train/sequences/*_v')))
_anno_files = [
osp.join(root_dir, 'VisDrone2018-VID-train/annotations',
osp.basename(s) + '.txt') for s in _seq_dirs
]
seq_dirs += _seq_dirs
anno_files += _anno_files
if 'val' in subset:
_seq_dirs = sorted(
glob.glob(
osp.join(root_dir, 'VisDrone2018-VID-val/sequences/*_v')))
_anno_files = [
osp.join(root_dir, 'VisDrone2018-VID-val/annotations',
osp.basename(s) + '.txt') for s in _seq_dirs
]
seq_dirs += _seq_dirs
anno_files += _anno_files
seq_names = [osp.basename(s) for s in seq_dirs]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
if s % 10 == 0 or s + 1 == len(seq_names):
ops.sys_print('Processing [%d/%d]: %s' %
(s + 1, len(seq_names), seq_name))
img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg')))
anno_s = np.loadtxt(anno_files[s], delimiter=',', dtype=np.float32)
anno_s = self._format(anno_s)
# meta information
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': len(img_files),
'target_num': len(set(anno_s[:, 1])),
'total_instances': len(anno_s)
}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno_s,
'meta': meta
}
}
return seq_dict
def _construct_seq_dict(self, root_dir, version):
# image and annotation paths
valid_seqs = self.__version_dict[version]
anno_files = sorted(list(chain.from_iterable(glob.glob(
osp.join(root_dir, s, 'groundtruth*.txt')) for s in valid_seqs)))
# remove empty annotation files
# (e.g., groundtruth_rect.1.txt of Human4)
anno_files = self._filter_files(anno_files)
seq_dirs = [osp.dirname(f) for f in anno_files]
seq_names = [osp.basename(d) for d in seq_dirs]
# rename repeated sequence names
# (e.g., Jogging and Skating2)
seq_names = self._rename_seqs(seq_names)
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
img_files = sorted(glob.glob(
osp.join(seq_dirs[s], 'img/*.jpg')))
# special sequences
# (visit http://cvlab.hanyang.ac.kr/tracker_benchmark/index.html for detail)
if seq_name.lower() == 'david':
img_files = img_files[300-1:770]
elif seq_name.lower() == 'football1':
img_files = img_files[:74]
elif seq_name.lower() == 'freeman3':
img_files = img_files[:460]
elif seq_name.lower() == 'freeman4':
img_files = img_files[:283]
elif seq_name.lower() == 'diving':
img_files = img_files[:215]
# load annotations (to deal with different delimeters)
with open(anno_files[s], 'r') as f:
anno = np.loadtxt(io.StringIO(f.read().replace(',', ' ')))
anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta}}
return seq_dict
def _construct_seq_dict(self, root_dir, version):
# sequence meta information
meta_file = osp.join(osp.dirname(__file__), 'uav123.json')
with open(meta_file) as f:
seq_metas = json.load(f)
# image and annotation paths
anno_files = sorted(
glob.glob(osp.join(root_dir, 'anno/%s/*.txt' % version)))
seq_names = [osp.basename(f)[:-4] for f in anno_files]
seq_dirs = [osp.join(
root_dir, 'data_seq/UAV123/%s' % \
seq_metas[version][n]['folder_name'])
for n in seq_names]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
# valid frame range
start_frame = seq_metas[version][seq_names[s]]['start_frame']
end_frame = seq_metas[version][seq_names[s]]['end_frame']
img_files = [
osp.join(seq_dirs[s], '%06d.jpg' % f)
for f in range(start_frame, end_frame + 1)
]
# load annotations
anno = np.loadtxt(anno_files[s], delimiter=',')
anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len
}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta
}
}
return seq_dict
def show(self,
tracker_names,
seq_names=None,
play_speed=1,
visualize=True,
save=False,
save_dir='screenshots'):
if seq_names is None:
seq_names = self.dataset.seq_names
elif isinstance(seq_names, str):
seq_names = [seq_names]
assert isinstance(tracker_names, (list, tuple))
assert isinstance(seq_names, (list, tuple))
play_speed = int(round(play_speed))
assert play_speed > 0
for s, seq_name in enumerate(seq_names):
ops.sys_print('[%d/%d] Showing results on %s...' %
(s + 1, len(seq_names), seq_name))
# mkdir if required to save screenshots
if save:
out_dir = osp.join(save_dir, seq_name)
if not osp.exists(out_dir):
os.makedirs(out_dir)
# load all tracking results
records = {}
for name in tracker_names:
record_file = osp.join(self.result_dir, name,
'%s.txt' % seq_name)
records[name] = np.loadtxt(record_file, delimiter=',')
# loop over the sequence and display results
img_files, target = self.dataset[seq_name][:2]
img_files = img_files[::self.frame_stride]
target['anno'] = target['anno'][self.frame_stride]
for f, img_file in enumerate(img_files):
if not f % play_speed == 0:
continue
img = ops.read_image(img_file)
bboxes = [records[name][f] for name in tracker_names]
if len(target['anno']) > f:
bboxes = [target['anno'][f]] + bboxes
img = ops.show_image(img, bboxes, visualize=visualize)
# save screenshot if required
if save:
out_file = osp.join(out_dir, '%08d.jpg' % (f + 1))
cv2.imwrite(out_file, img)
def _construct_seq_dict(self, root_dir, fps):
# image and annotation paths
anno_files = sorted(glob.glob(osp.join(root_dir, '*/%d/*.txt' % fps)))
seq_names = [osp.basename(f)[:-4] for f in anno_files]
seq_dirs = [
osp.join(osp.dirname(f), n) for f, n in zip(anno_files, seq_names)
]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
if s % 50 == 0 or s + 1 == len(seq_names):
ops.sys_print('Processing sequence [%d/%d]: %s...' %
(s + 1, len(seq_names), seq_name))
img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg')))
anno = np.loadtxt(anno_files[s], dtype=str)
anno = anno[:, 1:5].astype(np.float32)
# handle inconsistent lengths
if not len(img_files) == len(anno):
if abs(len(anno) / len(img_files) - 8) < 1:
anno = anno[0::8, :]
diff = abs(len(img_files) - len(anno))
if diff > 0 and diff <= 1:
n = min(len(img_files), len(anno))
anno = anno[:n]
img_files = img_files[:n]
assert len(img_files) == len(anno)
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len
}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta
}
}
return seq_dict
def _construct_seq_dict(self, root_dir, subset):
# load subset sequence names
split_file = osp.join(osp.dirname(__file__), 'lasot.json')
with open(split_file, 'r') as f:
splits = json.load(f)
seq_names = splits[subset]
# image and annotation paths
seq_dirs = [
osp.join(root_dir, n[:n.rfind('-')], n, 'img') for n in seq_names
]
anno_files = [
osp.join(root_dir, n[:n.rfind('-')], n, 'groundtruth.txt')
for n in seq_names
]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
if s % 100 == 0 or s + 1 == len(seq_names):
ops.sys_print('Processing sequence [%d/%d]: %s...' %
(s + 1, len(seq_names), seq_name))
img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg')))
anno = np.loadtxt(anno_files[s], delimiter=',', dtype=np.float32)
anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = self._fetch_meta(seq_dirs[s])
meta.update({
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len
})
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta
}
}
return seq_dict
def _construct_seq_dict(self, root_dir, list_file):
# image and annotation paths
with open(list_file, 'r') as f:
seq_names = f.read().strip().split('\n')
seq_dirs = [osp.join(root_dir, s, 'color') for s in seq_names]
if os.path.exists(
osp.join(root_dir, seq_names[0], 'annotations',
'groundtruth.txt')):
anno_files = [
osp.join(root_dir, s, 'annotations', 'groundtruth.txt')
for s in seq_names
]
else:
anno_files = [
osp.join(root_dir, s, 'annotation', 'groundtruth.txt')
for s in seq_names
]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg')))
anno = np.loadtxt(anno_files[s], delimiter=',')
anno = self._format(anno)
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = self._fetch_meta(seq_dirs[s], seq_len)
meta.update({
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len
})
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta
}
}
return seq_dict
def _construct_seq_dict(self, root_dir, subset_dirs):
# image and annotation paths
anno_files = [
glob.glob(osp.join(root_dir, c, 'anno/*.txt')) for c in subset_dirs
]
anno_files = sorted(sum(anno_files, []))
seq_dirs = [
osp.join(osp.dirname(osp.dirname(f)), 'frames',
osp.basename(f)[:-4]) for f in anno_files
]
seq_names = [osp.basename(d) for d in seq_dirs]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
if s % 100 == 0 or s + 1 == len(seq_names):
ops.sys_print('Processing sequence [%d/%d]: %s...' %
(s + 1, len(seq_names), seq_name))
img_files = glob.glob(osp.join(seq_dirs[s], '*.jpg'))
img_files = sorted(img_files,
key=lambda f: int(osp.basename(f)[:-4]))
anno = np.loadtxt(anno_files[s], delimiter=',')
if anno.ndim == 1:
anno = np.expand_dims(anno, axis=0)
anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len
}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta
}
}
return seq_dict
def _construct_seq_dict(self, root_dir, version, subset):
# image and annotation paths
seq_dirs = sorted(glob.glob(
osp.join(root_dir, subset, '*/img1')))
parent_dirs = [osp.dirname(d) for d in seq_dirs]
seq_names = [osp.basename(d) for d in parent_dirs]
det_files = [osp.join(
d, 'det/det.txt') for d in parent_dirs]
if subset == 'train':
gt_files = [osp.join(
d, 'gt/gt.txt') for d in parent_dirs]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
img_files = sorted(glob.glob(
osp.join(seq_dirs[s], '*.jpg')))
det = np.loadtxt(
det_files[s], delimiter=',', dtype=np.float32)
det = self._format_det(det)
target = {'det': det}
if subset == 'train':
gt = np.loadtxt(
gt_files[s], delimiter=',', dtype=np.float32)
gt = self._format_gt(gt)
target.update({'anno': gt})
# meta information
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': len(img_files),
'target_num': -1 if subset != 'train' else len(set(gt[:, 1])),
'total_instances': -1 if subset != 'train' else len(gt)}
target.update({'meta': meta})
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': target}
return seq_dict
def __getitem__(self, index):
r"""
Args:
index (integer): Index of an image.
Returns:
img, target (tuple): where ``target`` is dict of annotations.
"""
img_name = self.img_names[index]
img_info = self.img_dict[img_name]
# parse image and annotations
img = ops.read_image(img_info['img_file'])
target = img_info['target']
if hasattr(self, 'transforms') and self.transforms is not None:
img, target = self.transforms(img, target)
return img, target
def _construct_seq_dict(self, root_dir, subset):
# image and annotation paths
if subset == 'test':
anno_files = sorted(
glob.glob(osp.join(root_dir, 'initialization/*_s.txt')))
else:
anno_files = sorted(
glob.glob(osp.join(root_dir, 'annotations/*_s.txt')))
seq_names = [osp.basename(f)[:-4] for f in anno_files]
seq_dirs = [
osp.join(root_dir, 'sequences/{}'.format(n)) for n in seq_names
]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
img_files = sorted(glob.glob(osp.join(seq_dirs[s], 'img*.jpg')))
anno = np.loadtxt(anno_files[s], delimiter=',')
if anno.ndim == 1:
assert anno.size == 4
anno = anno[np.newaxis, :]
anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len
}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta
}
}
return seq_dict
def _construct_seq_dict(self, root_dir):
# image and annotation paths
seq_dirs = sorted(glob.glob(
osp.join(root_dir, '*/*_*/')))
seq_dirs = [d[:-1] for d in seq_dirs]
seq_names = [osp.basename(d) for d in seq_dirs]
anno_files = [osp.join(
root_dir, 'annotation/annotation/{}_gt_points.txt'.format(n))
for n in seq_names]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
if s % 50 == 0 or s + 1 == len(seq_names):
ops.sys_print('Processing sequence [%d/%d]: %s...' % (
s + 1, len(seq_names), seq_name))
img_files = sorted(glob.glob(
osp.join(seq_dirs[s], '*.jpg')))
anno = np.loadtxt(anno_files[s])
n = min(len(img_files), len(anno))
assert n > 0
img_files = img_files[:n]
anno = anno[:n]
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta}}
return seq_dict
def _construct_seq_dict(self, root_dir):
# image and annotation paths
anno_files = sorted(glob.glob(
osp.join(root_dir, '*/*_gt.txt')))
seq_dirs = [osp.dirname(f) for f in anno_files]
seq_names = [osp.basename(d) for d in seq_dirs]
# valid frame range for each sequence
range_files = [glob.glob(
osp.join(d, '*_frames.txt'))[0]
for d in seq_dirs]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
# load valid frame range
frames = np.loadtxt(
range_files[s], dtype=int, delimiter=',')
img_files = [osp.join(
seq_dirs[s], 'img/%04d.jpg' % f)
for f in range(frames[0], frames[1] + 1)]
# load annotations
anno = np.loadtxt(anno_files[s], delimiter=',')
anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta}}
return seq_dict
def _construct_seq_dict(self, root_dir, subset, list_file):
# image and annotation paths
with open(list_file, 'r') as f:
seq_names = f.read().strip().split('\n')
seq_dirs = [osp.join(root_dir, subset, s) for s in seq_names]
anno_files = [osp.join(d, 'groundtruth.txt') for d in seq_dirs]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
if s % 100 == 0 or s + 1 == len(seq_names):
ops.sys_print('Processing sequence [%d/%d]: %s...' %
(s + 1, len(seq_names), seq_name))
img_files = sorted(glob.glob(osp.join(seq_dirs[s], '*.jpg')))
anno = np.loadtxt(anno_files[s], delimiter=',', dtype=np.float32)
if anno.ndim == 1:
assert anno.size == 4
anno = anno[np.newaxis, :]
anno[:, 2:] = anno[:, :2] + anno[:, 2:] - 1
# meta information
seq_len = len(img_files)
img0 = ops.read_image(img_files[0])
meta = self._fetch_meta(seq_dirs[s])
meta.update({
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': seq_len,
'target_num': 1,
'total_instances': seq_len
})
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno,
'meta': meta
}
}
return seq_dict
def forward_test(self, img_files, init_bbox, visualize=False):
frame_num = len(img_files)
times = np.zeros(frame_num)
preds = [{
'present': True,
'score': 1.0,
'xmin': init_bbox[0],
'xmax': init_bbox[2],
'ymin': init_bbox[1],
'ymax': init_bbox[3]
}]
for f, img_file in enumerate(img_files):
if self.input_type == 'image':
img = ops.read_image(img_file, self.color_fmt)
elif self.input_type == 'file':
img = img_file
begin = time.time()
if f == 0:
self.init(img, init_bbox)
else:
bbox, score, present = self.update(img)
preds.append({
'present': present,
'score': score,
'xmin': bbox[0],
'xmax': bbox[2],
'ymin': bbox[1],
'ymax': bbox[3]
})
times[f] = time.time() - begin
if visualize:
ops.show_image(img, bboxes[f, :])
# update the preds as one-per-second
frame_stride = 30
preds = {f * frame_stride: pred for f, pred in enumerate(preds)}
return preds, times
def _check_dataset(self, dataset):
seq_num = len(dataset)
if seq_num == 0:
ops.sys_print('Warning: {} dataset is empty, '
'skip test...'.format(dataset.name))
return
self.assertGreater(seq_num, 0)
ops.sys_print('{} contains {} sequences'.format(dataset.name, seq_num))
# sanity check
inds = random.sample(range(seq_num), min(seq_num, 10))
for i in inds:
img_files, target = dataset[i]
anno, meta = target['anno'], target['meta']
if anno.shape[0] == 1:
continue # test sets
if anno.shape[1] in [4, 8]:
self.assertEqual(len(img_files), len(anno))
else:
self.assertGreaterEqual(len(img_files) - 1, anno[:, 0].max())
# visualization
if self.visualize:
img_files, target = random.choice(dataset)
anno = target['anno']
for f, img_file in enumerate(img_files):
if f >= anno.shape[0]:
break # test sets
if anno.shape[1] == 9:
bboxes = anno[anno[:, 0] == f, 2:6]
elif anno.shape[1] == 8:
break # TODO: points are not supported yet
else:
bboxes = anno[f, :]
img = ops.read_image(img_file)
ops.show_image(img, bboxes, delay=1)
def _construct_seq_dict(self, root_dir, subset):
# image and annotation paths
seq_dirs = []
anno_dirs = []
if 'train' in subset:
_seq_dirs = sorted(
glob.glob(osp.join(root_dir,
'Data/VID/train/ILSVRC*/ILSVRC*')))
_anno_dirs = [
osp.join(root_dir, 'Annotations/VID/train',
*s.split('/')[-2:]) for s in _seq_dirs
]
seq_dirs += _seq_dirs
anno_dirs += _anno_dirs
if 'val' in subset:
_seq_dirs = sorted(
glob.glob(osp.join(root_dir, 'Data/VID/val/ILSVRC2015_val_*')))
_anno_dirs = [
osp.join(root_dir, 'Annotations/VID/val',
s.split('/')[-1]) for s in _seq_dirs
]
seq_dirs += _seq_dirs
anno_dirs += _anno_dirs
seq_names = [osp.basename(s) for s in seq_dirs]
# construct seq_dict
seq_dict = {}
for s, seq_name in enumerate(seq_names):
if s % 100 == 0 or s + 1 == len(seq_names):
ops.sys_print('Processing sequence [%d/%d]: %s...' %
(s + 1, len(seq_names), seq_name))
# parse XML annotation files
anno_files = sorted(glob.glob(osp.join(anno_dirs[s], '*.xml')))
objects = [
ET.ElementTree(file=f).findall('object') for f in anno_files
]
anno_s = []
for f, group in enumerate(objects):
for obj in group:
anno_obj = [
f, # frame_id
int(obj.find('trackid').text), # target_id
int(obj.find('bndbox/xmin').text), # x1
int(obj.find('bndbox/ymin').text), # y1
int(obj.find('bndbox/xmax').text), # x2
int(obj.find('bndbox/ymax').text), # y2
1, # score/keep
self.WORDNET_IDs.index(
obj.find('name').text), # class_id
int(obj.find('occluded').text)
] # occlusion
anno_s.append(anno_obj)
anno_s = np.array(anno_s, dtype=np.float32)
# meta information
img_files = [
osp.join(seq_dirs[s], '%06d.JPEG' % f)
for f in range(len(objects))
]
img0 = ops.read_image(img_files[0])
meta = {
'width': img0.shape[1],
'height': img0.shape[0],
'frame_num': len(img_files),
'target_num': len(set(anno_s[:, 1])),
'total_instances': len(anno_s)
}
# update seq_dict
seq_dict[seq_name] = {
'img_files': img_files,
'target': {
'anno': anno_s,
'meta': meta
}
}
return seq_dict
def __getitem__(self, index):
if index > len(self):
raise IndexError('Index out of range')
index %= len(self.seqs)
# get image files and annotations
img_files, target = self.seqs[index]
anno, meta = target['anno'], target['meta']
# filter noisy annotations
valid_indices = self._filter(anno, meta)
if len(valid_indices) < 2:
return self._random_next()
# ramdomly sample a pair
rand_z, rand_x = self._sample_pair(anno, valid_indices)
if rand_z < 0 or rand_x < 0:
return self._random_next()
# annotations pair
if anno.shape[1] in [4, 8]:
# SOT-based pairs
bboxes_z = np.expand_dims(anno[rand_z], axis=0)
bboxes_x = np.expand_dims(anno[rand_x], axis=0)
elif anno.shape[1] == 9:
# MOT/VID-based pairs
# find consistent track_ids
mask_z = (anno[:, 0] == rand_z)
mask_x = (anno[:, 0] == rand_x)
z_ids, x_ids = anno[mask_z, 1], anno[mask_x, 1]
join_ids = sorted(set(z_ids) & set(x_ids))
# extract bounding boxes (ignore the class label)
bboxes_z, bboxes_x = [], []
for track_id in join_ids:
mask_id = (anno[:, 1] == track_id)
mask_id_z = (mask_z & mask_id)
mask_id_x = (mask_x & mask_id)
if mask_id_z.sum() != 1 or mask_id_x.sum() != 1:
ops.sys_print('Warning: found repeated ID for',
self.seqs.seq_names[index])
return self._random_next()
bboxes_z += [anno[mask_id_z, 2:6]]
bboxes_x += [anno[mask_id_x, 2:6]]
bboxes_z = np.concatenate(bboxes_z, axis=0)
bboxes_x = np.concatenate(bboxes_x, axis=0)
assert len(bboxes_z) == len(bboxes_x) == len(join_ids)
# image pair
img_z = ops.read_image(img_files[rand_z])
img_x = ops.read_image(img_files[rand_x])
# bound annotations by image boundaries
h, w = img_z.shape[:2]
bboxes_z = ops.bound_bboxes(bboxes_z, [w, h])
bboxes_x = ops.bound_bboxes(bboxes_x, [w, h])
# build target
target = {'bboxes_z': bboxes_z, 'bboxes_x': bboxes_x}
# apply transforms if applicable
if self.transforms is not None:
return self.transforms(img_z, img_x, target)
else:
return img_z, img_x, target
def run_realtime(self, tracker, visualize=False):
ops.sys_print('Running real-time experiment...')
# loop over the complete dataset
for s, (img_files, target) in enumerate(self.dataset):
seq_name = self.dataset.seq_names[s]
ops.sys_print('--Sequence %d/%d: %s' %
(s + 1, len(self.dataset), seq_name))
# skip if results exist
record_file = os.path.join(self.result_dir, tracker.name,
'realtime', seq_name,
'%s_001.txt' % seq_name)
if os.path.exists(record_file):
ops.sys_print(' Found results, skipping %s' % seq_name)
continue
# rectangular bounding boxes
anno = target['anno']
anno_rects = anno.copy()
if anno_rects.shape[1] == 8:
anno_rects = self.dataset._corner2rect(anno_rects)
# state variables
bboxes = []
times = []
next_start = 0
failure = False
failed_frame = -1
total_time = 0.0
grace = 3 - 1
offset = 0
# tracking loop
for f, img_file in enumerate(img_files):
img = ops.read_image(img_file)
if tracker.input_type == 'image':
frame = img
elif tracker.input_type == 'file':
frame = img_file
start_time = time.time()
if f == next_start:
# during initial frames
tracker.init(frame, anno_rects[f])
bboxes.append([1])
# reset state variables
failure = False
failed_frame = -1
total_time = 0.0
grace = 3 - 1
offset = f
elif not failure:
# during success frames
# calculate current frame
if grace > 0:
total_time += 1000.0 / 25
grace -= 1
else:
total_time += max(1000.0 / 25, last_time * 1000.0)
current = offset + int(
np.round(np.floor(total_time * 25) / 1000.0))
# delayed/tracked bounding box
if f < current:
bbox = bboxes[-1]
elif f == current:
bbox = tracker.update(frame)
iou = ops.poly_iou(anno[f], bbox, bound=img.shape[1::-1])
if iou <= 0.0:
# tracking failure
failure = True
failed_frame = f
next_start = current + self.skip_initialize
bboxes.append([2])
else:
# tracking succeed
bboxes.append(bbox)
else:
# during failure frames
if f < current:
# skipping frame due to slow speed
bboxes.append([0])
start_time = np.NaN
elif f == current:
# current frame
bbox = tracker.update(frame)
iou = ops.poly_iou(anno[f],
bbox,
bound=img.shape[1::-1])
if iou <= 0.0:
# tracking failure
bboxes.append([2])
bboxes[failed_frame] = [0]
times[failed_frame] = np.NaN
else:
# tracking succeed
bboxes.append(bbox)
elif f < next_start:
# skipping frame due to failure
bboxes.append([0])
start_time = np.NaN
# store elapsed time
last_time = time.time() - start_time
times.append(last_time)
# visualize if required
if visualize:
if len(bboxes[-1]) == 4:
ops.show_image(img, bboxes[-1])
else:
ops.show_image(img)
# record results
self._record(record_file, bboxes, times)
def run_supervised(self, tracker, visualize=False):
ops.sys_print('Running supervised experiment...')
# loop over the complete dataset
for s, (img_files, target) in enumerate(self.dataset):
seq_name = self.dataset.seq_names[s]
ops.sys_print('--Sequence %d/%d: %s' %
(s + 1, len(self.dataset), seq_name))
# rectangular bounding boxes
anno = target['anno']
anno_rects = anno.copy()
if anno_rects.shape[1] == 8:
anno_rects = self.dataset._corner2rect(anno_rects)
# run multiple repetitions for each sequence
for r in range(self.repetitions):
# check if the tracker is deterministic
if r > 0 and tracker.is_deterministic:
break
elif r == 3 and self._check_deterministic(
'baseline', tracker.name, seq_name):
ops.sys_print(' Detected a deterministic tracker, '
'skipping remaining trials.')
break
ops.sys_print(' Repetition: %d' % (r + 1))
# skip if results exist
record_file = os.path.join(self.result_dir, tracker.name,
'baseline', seq_name,
'%s_%03d.txt' % (seq_name, r + 1))
if os.path.exists(record_file):
ops.sys_print(' Found results, skipping %s' % seq_name)
continue
# state variables
bboxes = []
times = []
failure = False
next_start = -1
# tracking loop
for f, img_file in enumerate(img_files):
img = ops.read_image(img_file)
if tracker.input_type == 'image':
frame = img
elif tracker.input_type == 'file':
frame = img_file
start_time = time.time()
if f == 0:
# initial frame
tracker.init(frame, anno_rects[0])
bboxes.append([1])
elif failure:
# during failure frames
if f == next_start:
if np.all(anno_rects[f] <= 0):
next_start += 1
start_time = np.NaN
bboxes.append([0])
else:
failure = False
tracker.init(frame, anno_rects[f])
bboxes.append([1])
else:
start_time = np.NaN
bboxes.append([0])
else:
# during success frames
bbox = tracker.update(frame)
iou = ops.poly_iou(anno[f],
bbox,
bound=img.shape[1::-1])
if iou <= 0.0:
# tracking failure
failure = True
next_start = f + self.skip_initialize
bboxes.append([2])
else:
# tracking succeed
bboxes.append(bbox)
# store elapsed time
times.append(time.time() - start_time)
# visualize if required
if visualize:
if len(bboxes[-1]) == 4:
ops.show_image(img, bboxes[-1])
else:
ops.show_image(img)
# record results
self._record(record_file, bboxes, times)
def show(self,
tracker_names,
seq_names=None,
play_speed=1,
experiment='supervised',
visualize=True,
save=False,
save_dir='screenshots'):
if seq_names is None:
seq_names = self.dataset.seq_names
elif isinstance(seq_names, str):
seq_names = [seq_names]
assert isinstance(tracker_names, (list, tuple))
assert isinstance(seq_names, (list, tuple))
assert experiment in ['supervised', 'unsupervised', 'realtime']
play_speed = int(round(play_speed))
assert play_speed > 0
# "supervised" experiment results are stored in "baseline" folder
if experiment == 'supervised':
experiment = 'baseline'
# function for loading results
def read_record(filename):
with open(filename) as f:
record = f.read().strip().split('\n')
record = [[float(t) for t in line.split(',')] for line in record]
for i, r in enumerate(record):
if len(r) == 4:
record[i] = np.array(r)
elif len(r) == 8:
r = np.array(r)[np.newaxis, :]
r = self.dataset._corner2rect(r)
record[i] = r[0]
else:
record[i] = np.zeros(4)
return record
for s, seq_name in enumerate(seq_names):
ops.sys_print('[%d/%d] Showing results on %s...' %
(s + 1, len(seq_names), seq_name))
# mkdir if required to save screenshots
if save:
out_dir = os.path.join(save_dir, seq_name)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# load all tracking results
records = {}
for name in tracker_names:
record_file = os.path.join(self.result_dir, name, experiment,
seq_name, '%s_001.txt' % seq_name)
records[name] = read_record(record_file)
# loop over the sequence and display results
img_files, target = self.dataset[seq_name][:2]
anno = target['anno']
if anno.shape[1] == 8:
anno = self.dataset._corner2rect(anno)
for f, img_file in enumerate(img_files):
if not f % play_speed == 0:
continue
img = ops.read_image(img_file)
bboxes = [anno[f]
] + [records[name][f] for name in tracker_names]
img = ops.show_image(img, bboxes, visualize=visualize)
# save screenshot if required
if save:
out_file = os.path.join(out_dir, '%08d.jpg' % (f + 1))
cv2.imwrite(out_file, img)
def report(self, tracker_names, plot_curves=False):
if isinstance(tracker_names, str):
tracker_names = [tracker_names]
assert isinstance(tracker_names, (list, tuple))
# function for loading results
def read_record(filename):
with open(filename) as f:
record = f.read().strip().split('\n')
record = [[float(t) for t in line.split(',')] for line in record]
return record
# assume tracker_names[0] is your tracker
report_dir = os.path.join(self.report_dir, tracker_names[0])
if not os.path.exists(report_dir):
os.makedirs(report_dir)
report_file = os.path.join(report_dir, 'performance.json')
performance = {}
for name in tracker_names:
ops.sys_print('Evaluating %s' % name)
ious = {}
ious_full = {}
failures = {}
times = {}
masks = {} # frame masks for attribute tags
for s, (img_files, target) in enumerate(self.dataset):
anno, meta = target['anno'], target['meta']
seq_name = self.dataset.seq_names[s]
# initialize frames scores
frame_num = len(img_files)
ious[seq_name] = np.full((self.repetitions, frame_num),
np.nan,
dtype=float)
ious_full[seq_name] = np.full((self.repetitions, frame_num),
np.nan,
dtype=float)
failures[seq_name] = np.full((self.repetitions, frame_num),
np.nan,
dtype=float)
times[seq_name] = np.full((self.repetitions, frame_num),
np.nan,
dtype=float)
# read results of all repetitions
record_files = sorted(
glob.glob(
os.path.join(self.result_dir, name, 'baseline',
seq_name, '%s_[0-9]*.txt' % seq_name)))
bboxes = [read_record(f) for f in record_files]
assert all([len(b) == len(anno) for b in bboxes])
# calculate frame ious with burnin
bound = ops.read_image(img_files[0]).shape[1::-1]
seq_ious = [
self._calc_iou(b, anno, bound, burnin=True) for b in bboxes
]
ious[seq_name][:len(seq_ious), :] = seq_ious
# calculate frame ious without burnin
seq_ious_full = [
self._calc_iou(b, anno, bound) for b in bboxes
]
ious_full[seq_name][:len(seq_ious_full), :] = seq_ious_full
# calculate frame failures
seq_failures = [[
len(b) == 1 and b[0] == 2 for b in bboxes_per_rep
] for bboxes_per_rep in bboxes]
failures[seq_name][:len(seq_failures), :] = seq_failures
# collect frame runtimes
time_file = os.path.join(self.result_dir, name, 'baseline',
seq_name, '%s_time.txt' % seq_name)
if os.path.exists(time_file):
seq_times = np.loadtxt(time_file, delimiter=',').T
times[seq_name][:len(seq_times), :] = seq_times
# collect attribute masks
tag_num = len(self.tags)
masks[seq_name] = np.zeros((tag_num, frame_num), bool)
for i, tag in enumerate(self.tags):
if tag in meta:
masks[seq_name][i, :] = meta[tag]
# frames with no tags
if 'empty' in self.tags:
tag_frames = np.array([
v for k, v in meta.items()
if isinstance(v, np.ndarray) and \
not 'practical' in k], dtype=bool)
ind = self.tags.index('empty')
masks[seq_name][ind, :] = \
~np.logical_or.reduce(tag_frames, axis=0)
# concatenate frames
seq_names = self.dataset.seq_names
masks = np.concatenate([masks[s] for s in seq_names], axis=1)
ious = np.concatenate([ious[s] for s in seq_names], axis=1)
failures = np.concatenate([failures[s] for s in seq_names], axis=1)
with warnings.catch_warnings():
# average over repetitions
warnings.simplefilter('ignore', category=RuntimeWarning)
ious = np.nanmean(ious, axis=0)
failures = np.nanmean(failures, axis=0)
# calculate average overlaps and failures for each tag
tag_ious = np.array([np.nanmean(ious[m]) for m in masks])
tag_failures = np.array(
[np.nansum(failures[m]) for m in masks])
tag_frames = masks.sum(axis=1)
# remove nan values
tag_ious[np.isnan(tag_ious)] = 0.0
tag_weights = tag_frames / tag_frames.sum()
# calculate weighted accuracy and robustness
accuracy = np.sum(tag_ious * tag_weights)
robustness = np.sum(tag_failures * tag_weights)
# calculate tracking speed
times = np.concatenate([t.reshape(-1) for t in times.values()])
# remove invalid values
times = times[~np.isnan(times)]
times = times[times > 0]
if len(times) > 0:
speed = np.mean(1. / times)
else:
speed = -1
performance.update({
name: {
'accuracy': accuracy,
'robustness': robustness,
'speed_fps': speed
}
})
# save performance
with open(report_file, 'w') as f:
json.dump(performance, f, indent=4)
ops.sys_print('Performance saved at %s' % report_file)
return performance
