def frame_result_visualization(frame_result, pc):
visualizer = Visualizer2D(figsize=(12, 12))
bbox0, bbox1 = frame_result['bbox0'], frame_result['bbox1']
gt_bbox0, gt_bbox1 = frame_result['gt_bbox0'], frame_result['gt_bbox1']
bbox1, gt_bbox1 = BBox.array2bbox(bbox1), BBox.array2bbox(gt_bbox1)
visualizer.handler_box(bbox1, color='light_blue')
visualizer.handler_box(gt_bbox1, color='red')
vis_pc = utils.pc_in_box_2D(gt_bbox1, pc, 4.0)
visualizer.handler_pc(vis_pc)
visualizer.show()
visualizer.close()
def main(bench_list, result_folder, name, data_folder, token, process):
pred_shape_folder = os.path.join(result_folder, name, 'shapes')
results = list()
for tracklet_index, tracklet_info in enumerate(bench_list):
if tracklet_index % process != token:
continue
print('{:} {:} / {:}'.format(HARDNESS, tracklet_index + 1,
len(bench_list)))
id = tracklet_info['id']
segment_name = tracklet_info['segment_name']
frame_range = tracklet_info['frame_range']
tracklet_results = json.load(
open(
os.path.join(result_folder, name, 'summary',
'{:}.json'.format(id))))
frame_keys = list(tracklet_results.keys())
frame_keys = sorted(str_list_to_int(frame_keys))
preds = list()
for key in frame_keys:
bbox = tracklet_results[str(key)]['bbox1']
bbox = BBox.array2bbox(bbox)
preds.append(bbox)
preds.insert(
0, BBox.array2bbox(tracklet_results[str(frame_keys[0])]['bbox0']))
shape_file_path = os.path.join(pred_shape_folder, '{:}.npz'.format(id))
if os.path.exists(shape_file_path):
shape = np.load(shape_file_path, allow_pickle=True)['shape']
else:
scene_pcs = np.load(os.path.join(data_folder, 'pc', 'raw_pc',
'{:}.npz'.format(segment_name)),
allow_pickle=True)
ego_infos = np.load(os.path.join(data_folder, 'ego_info',
'{:}.npz'.format(segment_name)),
allow_pickle=True)
pcs = [
pc2world(ego_infos[str(i)], scene_pcs[str(i)])
for i in range(frame_range[0], frame_range[1] + 1)
]
shape = create_shapes(pcs, preds)
np.savez_compressed(shape_file_path, shape=shape)
gt_shape = np.load(os.path.join(data_folder, 'gt_shapes',
'{:}.npz'.format(id)),
allow_pickle=True)['pc']
cd = compute_cd_loss(shape, gt_shape)
results.append(cd)
return results
def find_gt_bboxes(id, data_folder, segment_name, start_frame, end_frame):
""" This function is for finding the gt bboxes
Args:
id (str): id of the tracklet
gt_folder (str): root for data storage
segment_name (str): the segment to look at
start_frame (int): which frame to search
Return
list of bboxes
"""
gt_info = np.load(os.path.join(data_folder, 'gt_info',
'{:}.npz'.format(segment_name)),
allow_pickle=True)
ego_info = np.load(os.path.join(data_folder, 'ego_info',
'{:}.npz'.format(segment_name)),
allow_pickle=True)
bboxes, ids = gt_info['bboxes'][start_frame], gt_info['ids'][start_frame]
index = ids.index(id)
gts = list()
for i in range(start_frame + 1, end_frame + 1):
# first frame needs no evaluation
# so begin from start_frame + 1
frame_bboxes = gt_info['bboxes'][i]
frame_ids = gt_info['ids'][i]
index = frame_ids.index(id)
bbox = frame_bboxes[index]
bbox = BBox.array2bbox(bbox)
ego_matrix = ego_info[str(i)]
bbox = BBox.bbox2world(ego_matrix, bbox)
gts.append(bbox)
return gts
def sequence_eval(obj_list, result_folder, name, data_folder, iou):
""" evaluate and return the tracklet-level information in tracklets
"""
results = list()
for tracklet_info in obj_list:
tracklet_results = json.load(
open(
os.path.join(result_folder, name, 'summary',
'{:}.json'.format(tracklet_info['id']))))
frame_keys = list(tracklet_results.keys())
frame_keys = sorted(str_list_to_int(frame_keys))
if iou:
# iou has been computed
ious = list()
for key in frame_keys:
ious.append(tracklet_results[str(key)]['iou3d'])
tracklet_metrics = metrics_from_ious(ious)
else:
gts = find_gt_bboxes(id=tracklet_info['id'],
data_folder=data_folder,
segment_name=tracklet_info['segment_name'],
start_frame=tracklet_info['frame_range'][0] +
1,
end_frame=tracklet_info['frame_range'][1])
preds = list()
for key in frame_keys:
bbox = tracklet_results[str(key)]['bbox1']
bbox = BBox.array2bbox(bbox)
preds.append(bbox)
tracklet_metrics = metrics_from_bboxes(preds, gts)
results.append(tracklet_metrics)
return results
def find_bboxes(id, data_folder, segment_name, start_frame, end_frame):
""" In the SOT, the beginning frame is a GT BBox
This function is for finding the gt bbox
Args:
id (str): id of the tracklet
data_folder (str): root for data storage
segment_name (str): the segment to look at
start_frame (int): which frame to search
Return
BBox (numpy array): [x, y, z, h, l, w, h]
"""
gt_info = np.load(os.path.join(data_folder, 'gt_info',
'{:}.npz'.format(segment_name)),
allow_pickle=True)
ego_info = np.load(os.path.join(data_folder, 'ego_info',
'{:}.npz'.format(segment_name)),
allow_pickle=True)
bboxes, ids = gt_info['bboxes'][start_frame], gt_info['ids'][start_frame]
index = ids.index(id)
start_bbox = bboxes[index]
gts = list()
for i in range(start_frame, end_frame + 1):
frame_bboxes = gt_info['bboxes'][i]
frame_ids = gt_info['ids'][i]
index = frame_ids.index(id)
bbox = frame_bboxes[index]
bbox = BBox.array2bbox(bbox)
ego_matrix = ego_info[str(i)]
bbox = BBox.bbox2world(ego_matrix, bbox)
gts.append(bbox)
return start_bbox, gts
def find_bboxes(id, data_folder, segment_name, start_frame, end_frame):
""" This function is for finding the gt bboxes
"""
gt_info = np.load(os.path.join(data_folder, 'gt_info', '{:}.npz'.format(segment_name)),
allow_pickle=True)
bboxes, ids = gt_info['bboxes'][start_frame], gt_info['ids'][start_frame]
index = ids.index(id)
gts = list()
for i in range(start_frame, end_frame + 1):
frame_bboxes = gt_info['bboxes'][i]
frame_ids = gt_info['ids'][i]
index = frame_ids.index(id)
bbox = frame_bboxes[index]
bbox = BBox.array2bbox(bbox)
gts.append(bbox)
return gts
def __init__(self, ego_info, pc, start_bbox=None, terrain=None, dets=None):
""" the input of a frame should contain several fields, some compulsory, some optional
Args:
ego_info (4 * 4 matrix): ego matrix
pc (N * 3 array): point cloud
terrain (N * 3 pc terrain map, optional): ground point cloud. Defaults to None.
dets (BBoxes, optional): detection bboxes. Defaults to None.
"""
self.ego = ego_info
self.pc = pc
self.terrain = terrain
self.dets = None
if dets is not None:
self.dets = [BBox.array2bbox(det) for det in dets]
self.start_bbox = start_bbox
self.preprocess()
return
def tracker_api(configs,
id,
start_bbox,
start_frame,
data_loader,
track_len,
gts=None,
visualize=False):
""" api for the tracker
Args:
configs: model configuration read from config.yaml
id (str): each tracklet has an id
start_bbox ([x, y, z, yaw, l, w, h]): the beginning location of this id
data_loader (an iterator): iterator returning data of each incoming frame
Return:
{
frame_number0: pred_bbox0,
frame_number1: pred_bbox1,
...
frame_numberN: pred_bboxN
}
"""
tracker = sot_3d.Tracker(id=id,
configs=configs,
start_bbox=start_bbox,
start_frame=start_frame,
track_len=track_len)
tracklet_result = dict()
for frame_index in range(track_len):
print('////////////////////////////////////////')
print('Processing {:} {:} / {:}'.format(id, frame_index + 1,
track_len))
# initialize a tracker
frame_data = next(data_loader)
# if the first frame, add the start_bbox
input_bbox = None
if frame_index == 0:
input_bbox = BBox.bbox2world(frame_data['ego'],
BBox.array2bbox(start_bbox))
input_data = sot_3d.FrameData(ego_info=frame_data['ego'],
pc=frame_data['pc'],
start_bbox=input_bbox,
terrain=frame_data['terrain'],
dets=frame_data['dets'])
# run the frame level tracking
frame_output = tracker.track(input_data)
# the frame 0 may produce no output
if not frame_output:
continue
# if gt is not None, we may compare our prediction with gt
frame_result = compare_to_gt(frame_index, frame_output, gts)
max_frame_key = max(list(frame_result.keys()))
for i in range(max_frame_key + 1):
print('BBox0 : {:}'.format(frame_result[i]['bbox0']))
print('BBox1 : {:}'.format(frame_result[i]['bbox1']))
print('Motion : {:}'.format(frame_result[i]['motion']))
if gts:
print('GT BBox0 : {:}'.format(frame_result[i]['gt_bbox0']))
print('GT BBox1 : {:}'.format(frame_result[i]['gt_bbox1']))
print('GT Motion: {:}'.format(frame_result[i]['gt_motion']))
print('IOUS : {:} {:}'.format(frame_result[i]['iou2d'],
frame_result[i]['iou3d']))
print('\n')
tracklet_result[frame_index +
start_frame] = frame_result[max_frame_key]
if visualize:
frame_result_visualization(frame_result[max_frame_key],
tracker.input_data.pc)
return tracklet_result