def main():
"""main function"""
root_dir_default = osp.join(_init_paths.root_dir, 'data', 'kitti', 'KITTI-Object')
splits_file_default = osp.join(_init_paths.root_dir, 'data', 'kitti', 'splits', 'trainval.txt')
all_categories = ['Car', 'Van', 'Truck', 'Pedestrian', 'Person_sitting', 'Cyclist', 'Tram']
parser = argparse.ArgumentParser()
parser.add_argument("-r", "--root_dir", default=root_dir_default, help="Path to KITTI Object directory")
parser.add_argument("-s", "--split_file", default=splits_file_default, help="Path to split file")
parser.add_argument("-c", "--categories", type=str, nargs='+', default=['Car'], choices=all_categories, help="Object type (category)")
args = parser.parse_args()
print "------------- Config ------------------"
for arg in vars(args):
print "{} \t= {}".format(arg, getattr(args, arg))
assert osp.exists(args.root_dir), 'KITTI Object dir "{}" does not exist'.format(args.root_dir)
assert osp.exists(args.split_file), 'Path to split file does not exist: {}'.format(args.split_file)
image_names = [x.rstrip() for x in open(args.split_file)]
num_of_images = len(image_names)
print 'Using Split {} with {} images'.format(osp.basename(args.split_file), num_of_images)
root_dir = osp.join(args.root_dir, 'training')
label_dir = osp.join(root_dir, 'label_2_updated')
image_dir = osp.join(root_dir, 'image_2')
calib_dir = osp.join(root_dir, 'calib')
assert osp.exists(root_dir)
assert osp.exists(label_dir)
assert osp.exists(image_dir)
assert osp.exists(calib_dir)
dataset_name = 'kitti_' + osp.splitext(osp.basename(args.split_file))[0]
dataset = ImageDataset(dataset_name)
dataset.set_rootdir(root_dir)
# Using a slight harder settings thank standard kitti hardness
min_height = 20 # minimum height for evaluated groundtruth/detections
max_occlusion = 2 # maximum occlusion level of the groundtruth used for evaluation
max_truncation = 0.7 # maximum truncation level of the groundtruth used for evaluation
total_num_of_objects = 0
print 'Creating ImageDataset. May take long time'
for image_name in tqdm(image_names):
image_file_path = osp.join(image_dir, image_name + '.png')
label_file_path = osp.join(label_dir, image_name + '.txt')
calib_file_path = osp.join(calib_dir, image_name + '.txt')
assert osp.exists(image_file_path)
assert osp.exists(label_file_path)
assert osp.exists(calib_file_path)
objects = read_kitti_object_labels(label_file_path)
# filter the objects based on kitti hardness criteria
filtered_objects = {}
for obj_id, obj in enumerate(objects):
if obj['type'] not in args.categories:
continue
bbx = np.asarray(obj['bbox'])
too_hard = False
if (bbx[3] - bbx[1]) < min_height:
too_hard = True
if obj['occlusion'] > max_occlusion:
too_hard = True
if obj['truncation'] > max_truncation:
too_hard = True
if not too_hard:
filtered_objects[obj_id] = obj
if not filtered_objects:
continue
total_num_of_objects += len(filtered_objects)
image = cv2.imread(image_file_path)
W = image.shape[1]
H = image.shape[0]
calib_data = read_kitti_calib_file(calib_file_path)
P0 = calib_data['P0'].reshape((3, 4))
P2 = calib_data['P2'].reshape((3, 4))
K = P0[:3, :3]
assert np.all(P2[:3, :3] == K)
cam2_center = -np.linalg.inv(K).dot(P2[:, 3])
velo_T_cam0 = get_kitti_cam0_to_velo(calib_data)
velo_T_cam2 = velo_T_cam0 * Pose(t=cam2_center)
cam2_T_velo = get_kitti_velo_to_cam(calib_data, cam2_center)
assert np.allclose(velo_T_cam2.inverse().matrix(), cam2_T_velo.matrix())
annotation = OrderedDict()
annotation['image_file'] = osp.relpath(image_file_path, root_dir)
annotation['image_size'] = NoIndent([W, H])
annotation['image_intrinsic'] = NoIndent(K.astype(np.float).tolist())
obj_infos = []
for obj_id in sorted(filtered_objects):
obj = filtered_objects[obj_id]
obj_pose_cam2 = get_kitti_object_pose(obj, velo_T_cam0, cam2_center)
obj_pose_cam0 = get_kitti_object_pose(obj, velo_T_cam0, np.zeros(3))
assert np.allclose(obj_pose_cam0.t - obj_pose_cam2.t, cam2_center)
bbx_visible = np.array(obj['bbox'])
bbx_amodal = get_kitti_amodal_bbx(obj, K, obj_pose_cam2)
obj_origin_proj = project_point(K, obj_pose_cam2.t)
distance = np.linalg.norm(obj_pose_cam2.t)
delta_rot = rotation_from_two_vectors(obj_pose_cam2.t, np.array([0., 0., 1.]))
obj_rel_rot = np.matmul(delta_rot, obj_pose_cam2.R)
assert np.allclose(delta_rot.dot(obj_pose_cam2.t), np.array([0., 0., distance]))
viewpoint = viewpoint_from_rotation(obj_rel_rot)
R_vp = rotation_from_viewpoint(viewpoint)
assert np.allclose(R_vp, obj_rel_rot, rtol=1e-03), "R_vp = \n{}\nobj_rel_rot = \n{}\n".format(R_vp, obj_rel_rot)
assert np.allclose(np.matmul(delta_rot.T, R_vp), obj_pose_cam2.R, rtol=1e-04)
pred_alpha = get_kitti_alpha_from_object_pose(obj_pose_cam2, velo_T_cam2)
alpha_diff = wrap_to_pi(pred_alpha - obj['alpha'])
assert np.abs(alpha_diff) < 0.011, "{} vs {}. alpha_diff={}".format(pred_alpha, obj['alpha'], alpha_diff)
obj_info = OrderedDict()
obj_info['id'] = obj_id
obj_info['category'] = obj['type'].lower()
obj_info['dimension'] = NoIndent(obj['dimension'][::-1]) # [length, width, height]
obj_info['bbx_visible'] = NoIndent(bbx_visible.tolist())
obj_info['bbx_amodal'] = NoIndent(np.around(bbx_amodal, decimals=6).tolist())
obj_info['viewpoint'] = NoIndent(np.around(viewpoint, decimals=6).tolist())
obj_info['center_proj'] = NoIndent(np.around(obj_origin_proj, decimals=6).tolist())
obj_info['center_dist'] = round(float(distance), 6)
obj_infos.append(obj_info)
annotation['object_infos'] = obj_infos
dataset.add_image_info(annotation)
print 'Finished creating dataset with {} images and {} objects.'.format(dataset.num_of_images(), total_num_of_objects)
metainfo = OrderedDict()
metainfo['total_num_of_objects'] = total_num_of_objects
metainfo['categories'] = NoIndent([x.lower() for x in args.categories])
metainfo['min_height'] = min_height
metainfo['max_occlusion'] = max_occlusion
metainfo['max_truncation'] = max_truncation
dataset.set_metainfo(metainfo)
out_json_filename = dataset_name + '.json'
print 'Saving annotations to {}'.format(out_json_filename)
dataset.write_data_to_json(out_json_filename)
def main():
"""main function"""
root_dir_default = osp.join(_init_paths.root_dir, 'data', 'pascal3D',
'Pascal3D-Dataset')
split_choices = ['train', 'val', 'trainval', 'test']
sub_dataset_choices = ['imagenet', 'pascal']
category_choices = ['car', 'motorbike', 'bicycle', 'bus']
parser = argparse.ArgumentParser()
parser.add_argument("-r",
"--root_dir",
default=root_dir_default,
help="Path to Pascal3d Object directory")
parser.add_argument("-s",
"--split",
default='trainval',
choices=split_choices,
help="Split type")
parser.add_argument("-d",
"--sub_dataset",
default='imagenet',
choices=sub_dataset_choices,
help="Sub dataset type")
parser.add_argument("-c",
"--category",
type=str,
default='car',
choices=category_choices,
help="Object type (category)")
parser.add_argument("-n",
"--dataset_name",
type=str,
help="Optional output dataset name")
parser.add_argument('--no-truncated',
dest='keep_truncated',
action='store_false',
help="use this to remove truncated objects")
parser.set_defaults(keep_truncated=True)
parser.add_argument('--no-occluded',
dest='keep_occluded',
action='store_false',
help="use this to remove occluded objects")
parser.set_defaults(keep_occluded=True)
parser.add_argument('--no-difficult',
dest='keep_difficult',
action='store_false',
help="use this to remove difficult objects")
parser.set_defaults(keep_difficult=True)
args = parser.parse_args()
assert osp.exists(args.root_dir), "Directory '{}' do not exist".format(
args.root_dir)
anno_dir = osp.join(args.root_dir, 'AnnotationsFixed',
'{}_{}'.format(args.category, args.sub_dataset))
image_dir = osp.join(args.root_dir, 'Images',
'{}_{}'.format(args.category, args.sub_dataset))
assert osp.exists(anno_dir), "Directory '{}' do not exist".format(anno_dir)
assert osp.exists(image_dir), "Directory '{}' do not exist".format(
image_dir)
split_file = osp.join(_init_paths.root_dir, 'data', 'pascal3D', 'splits',
'{}_{}.txt'.format(args.sub_dataset, args.split))
assert osp.exists(split_file), "Split file '{}' do not exist".format(
split_file)
print "split = {}".format(args.split)
print "sub_dataset = {}".format(args.sub_dataset)
print "category = {}".format(args.category)
print "anno_dir = {}".format(anno_dir)
print "image_dir = {}".format(image_dir)
print "keep_truncated = {}".format(args.keep_truncated)
print "keep_occluded = {}".format(args.keep_occluded)
print "keep_difficult = {}".format(args.keep_difficult)
image_names = [x.rstrip() for x in open(split_file)]
num_of_images = len(image_names)
print 'Using split {} with {} images'.format(args.split, num_of_images)
# imagenet uses JPEG while pascal images are in jpg format
image_ext = '.JPEG' if args.sub_dataset == 'imagenet' else '.jpg'
if args.dataset_name:
dataset_name = args.dataset_name
else:
dataset_name = 'pascal3d_{}_{}_{}'.format(args.sub_dataset, args.split,
args.category)
dataset = ImageDataset(dataset_name)
dataset.set_rootdir(args.root_dir)
print "Importing dataset ..."
for image_name in tqdm(image_names):
anno_file = osp.join(anno_dir, image_name + '.mat')
image_file = osp.join(image_dir, image_name + image_ext)
if not osp.exists(anno_file):
continue
assert osp.exists(image_file), "Image file '{}' do not exist".format(
image_file)
image_info = OrderedDict()
image_info['image_file'] = osp.relpath(image_file, args.root_dir)
image = cv2.imread(image_file)
assert image.size, "image loaded from '{}' is empty".format(image_file)
W = image.shape[1]
H = image.shape[0]
image_info['image_size'] = NoIndent([W, H])
record = sio.loadmat(anno_file)['record'].flatten()[0]
assert record['filename'][
0] == image_name + image_ext, "{} vs {}".format(
record['filename'][0], image_name + image_ext)
record_objects = record['objects'].flatten()
obj_infos = []
for obj_id in xrange(len(record_objects)):
rec_obj = record_objects[obj_id]
category = rec_obj['class'].flatten()[0]
if category != args.category:
continue
occluded = bool(rec_obj['occluded'].flatten()[0])
truncated = bool(rec_obj['truncated'].flatten()[0])
difficult = bool(rec_obj['difficult'].flatten()[0])
if not args.keep_truncated and truncated:
continue
if not args.keep_occluded and occluded:
continue
if not args.keep_difficult and difficult:
continue
rec_vp = rec_obj['viewpoint'].flatten()[0]
distance = rec_vp['distance'].flatten()[0]
if distance == 0.0:
continue
azimuth = math.radians(rec_vp['azimuth'][0, 0])
elevation = math.radians(rec_vp['elevation'][0, 0])
tilt = math.radians(rec_vp['theta'][0, 0])
if azimuth == 0.0 and elevation == 0.0 and tilt == 0.0:
continue
viewpoint = np.around(np.array([azimuth, elevation, tilt],
dtype=np.float),
decimals=6)
viewpoint = wrap_to_pi_array(viewpoint)
assert_viewpoint(viewpoint)
assert rec_vp['focal'][
0,
0] == 1, "rec_vp['focal'] is expected to be 1 but got {}".format(
rec_vp['focal'][0, 0])
center_proj = np.array([rec_vp['px'][0, 0], rec_vp['py'][0, 0]],
dtype=np.float)
assert_coord2D(center_proj)
vbbx = rec_obj['bbox'].flatten()
assert_bbx(vbbx)
vbbx = clip_bbx_by_image_size(vbbx, W, H)
if np.any(vbbx[:2] >= vbbx[2:]):
continue
obj_info = OrderedDict()
obj_info['id'] = obj_id
obj_info['category'] = category
# since we dont have precise measure, use an approximate measure
obj_info['occlusion'] = 0.5 if occluded else 0.0
obj_info['truncation'] = 0.5 if truncated else 0.0
obj_info['difficulty'] = 0.5 if difficult else 0.0
vbbx = np.around(vbbx, decimals=6)
assert_bbx(vbbx)
obj_info['bbx_visible'] = NoIndent(vbbx.tolist())
if 'abbx' in rec_obj.dtype.names:
abbx = rec_obj['abbx'].flatten()
if abbx.shape == (4, ):
assert_bbx(abbx)
obj_info['bbx_amodal'] = NoIndent(
np.around(abbx, decimals=6).tolist())
obj_info['viewpoint'] = NoIndent(viewpoint.tolist())
obj_info['center_proj'] = NoIndent(
np.around(center_proj, decimals=6).tolist())
obj_infos.append(obj_info)
# only add if we have atleast 1 object
if obj_infos:
image_info['object_infos'] = obj_infos
dataset.add_image_info(image_info)
total_num_of_objects = sum(
[len(img_info['object_infos']) for img_info in dataset.image_infos()])
print 'Finished creating dataset with {} images and {} objects.'.format(
dataset.num_of_images(), total_num_of_objects)
num_of_objects_with_abbx = sum([
len([
obj_info for obj_info in img_info['object_infos']
if 'bbx_amodal' in obj_info
]) for img_info in dataset.image_infos()
])
print "Number of objects with bbx_amodal information = {}".format(
num_of_objects_with_abbx)
metainfo = OrderedDict()
metainfo['total_num_of_objects'] = total_num_of_objects
metainfo['categories'] = NoIndent([args.category])
dataset.set_metainfo(metainfo)
out_json_filename = dataset_name + '.json'
dataset.write_data_to_json(out_json_filename)
def test_single_weights_file(weights_file, net, input_dataset):
"""Test already initalized net with a new set of weights"""
net.copy_from(weights_file)
net.layers[0].generate_datum_ids()
input_num_of_objects = sum([len(image_info['object_infos']) for image_info in input_dataset.image_infos()])
assert net.layers[0].curr_data_ids_idx == 0
assert net.layers[0].number_of_datapoints() == input_num_of_objects
assert net.layers[0].data_ids == range(input_num_of_objects)
data_samples = net.layers[0].data_samples
num_of_data_samples = len(data_samples)
batch_size = net.layers[0].batch_size
num_of_batches = int(np.ceil(num_of_data_samples / float(batch_size)))
assert len(net.layers[0].image_loader) == input_dataset.num_of_images()
# Create Result dataset
result_dataset = ImageDataset(input_dataset.name())
result_dataset.set_rootdir(input_dataset.rootdir())
result_dataset.set_metainfo(input_dataset.metainfo().copy())
# Add weight file and its md5 checksum to metainfo
result_dataset.metainfo()['weights_file'] = weights_file
result_dataset.metainfo()['weights_file_md5'] = md5(open(weights_file, 'rb').read()).hexdigest()
# Set the image level fields
for input_im_info in input_dataset.image_infos():
result_im_info = OrderedDict()
result_im_info['image_file'] = input_im_info['image_file']
result_im_info['image_size'] = NoIndent(input_im_info['image_size'])
result_im_info['image_intrinsic'] = NoIndent(input_im_info['image_intrinsic'])
result_im_info['object_infos'] = []
result_dataset.add_image_info(result_im_info)
assert result_dataset.num_of_images() == input_dataset.num_of_images()
assert_funcs = {
"viewpoint": assert_viewpoint,
"bbx_visible": assert_bbx,
"bbx_amodal": assert_bbx,
"center_proj": assert_coord2D,
}
performance_metric = {}
print 'Evaluating for {} batches with {} imaes per batch.'.format(num_of_batches, batch_size)
for b in tqdm.trange(num_of_batches):
start_idx = batch_size * b
end_idx = min(batch_size * (b + 1), num_of_data_samples)
# print 'Working on batch: %d/%d (Image# %d - %d)' % (b, num_of_batches, start_idx, end_idx)
output = net.forward()
# store all accuracy outputs
for key in [key for key in output if any(x in key for x in ["accuracy", "iou", "error"])]:
assert np.squeeze(output[key]).shape == (), "Expects {} output to be scalar but got {}".format(key, output[key].shape)
current_batch_accuracy = float(np.squeeze(output[key]))
if key in performance_metric:
performance_metric[key].append(current_batch_accuracy)
else:
performance_metric[key] = [current_batch_accuracy]
for i in xrange(start_idx, end_idx):
image_id = data_samples[i]['image_id']
image_info = result_dataset.image_infos()[image_id]
object_info = OrderedDict()
# since we are not changing cetegory orid it is directly copied
object_info['id'] = data_samples[i]['id']
object_info['category'] = data_samples[i]['category']
# since we are not predicting bbx_visible, it is directly copied
object_info['bbx_visible'] = NoIndent(data_samples[i]['bbx_visible'].tolist())
for info in ["bbx_amodal", "viewpoint", "center_proj"]:
pred_info = "pred_" + info
if pred_info in net.blobs:
prediction = np.squeeze(net.blobs[pred_info].data[i - start_idx, ...])
assert_funcs[info](prediction)
object_info[info] = NoIndent(prediction.tolist())
image_info['object_infos'].append(object_info)
for key in sorted(performance_metric):
performance_metric[key] = np.mean(performance_metric[key])
print 'Test set {}: {:.4f}'.format(key, performance_metric[key])
regex = re.compile('iter_([0-9]*).caffemodel')
performance_metric['iter'] = int(regex.findall(weights_file)[0])
result_num_of_objects = sum([len(image_info['object_infos']) for image_info in result_dataset.image_infos()])
assert result_num_of_objects == num_of_data_samples
return result_dataset, performance_metric
def run_inference(weights_file, net, input_dataset):
"""Run inference with already initalized net with a new set of weights"""
net.copy_from(weights_file)
net.layers[0].generate_datum_ids()
num_of_images = input_dataset.num_of_images()
assert net.layers[0].curr_data_ids_idx == 0
assert net.layers[0].number_of_datapoints() == num_of_images
assert net.layers[0].data_ids == range(num_of_images)
assert len(net.layers[0].image_loader) == num_of_images
assert len(net.layers[0].data_samples) == num_of_images
assert net.layers[
0].rois_per_image < 0, "rois_per_image need to be dynamic for testing"
assert net.layers[
0].imgs_per_batch == 1, "We only support one image per batch while testing"
assert net.layers[0].flip_ratio < 0, "No flipping while testing"
assert net.layers[0].jitter_iou_min > 1, "No jittering"
# Create Result dataset
result_dataset = ImageDataset(input_dataset.name())
result_dataset.set_rootdir(input_dataset.rootdir())
result_dataset.set_metainfo(input_dataset.metainfo().copy())
# Add weight file and its md5 checksum to metainfo
result_dataset.metainfo()['weights_file'] = weights_file
result_dataset.metainfo()['weights_file_md5'] = md5(
open(weights_file, 'rb').read()).hexdigest()
# Set the image level fields
for input_im_info in input_dataset.image_infos():
result_im_info = OrderedDict()
result_im_info['image_file'] = input_im_info['image_file']
result_im_info['image_size'] = input_im_info['image_size']
if 'image_intrinsic' in input_im_info:
result_im_info['image_intrinsic'] = input_im_info[
'image_intrinsic']
obj_infos = []
for input_obj_info in input_im_info['object_infos']:
obj_info = OrderedDict()
for field in ['id', 'category', 'score', 'bbx_visible']:
if field in input_obj_info:
obj_info[field] = input_obj_info[field]
obj_infos.append(obj_info)
result_im_info['object_infos'] = obj_infos
assert len(result_im_info['object_infos']) == len(
input_im_info['object_infos'])
result_dataset.add_image_info(result_im_info)
assert result_dataset.num_of_images() == num_of_images
assert len(net.layers[0].data_samples) == num_of_images
for result_img_info, layer_img_info in zip(result_dataset.image_infos(),
net.layers[0].data_samples):
assert len(result_img_info['object_infos']) == len(
layer_img_info['object_infos'])
assert_funcs = {
"viewpoint": assert_viewpoint,
"bbx_visible": assert_bbx,
"bbx_amodal": assert_bbx,
"center_proj": assert_coord2D,
}
print 'Running inference for {} images.'.format(num_of_images)
for image_id in tqdm.trange(num_of_images):
# Run forward pass
_ = net.forward()
img_info = result_dataset.image_infos()[image_id]
expected_num_of_rois = len(img_info['object_infos'])
assert net.blobs['rois'].data.shape == (
expected_num_of_rois,
5), "{}_{}".format(net.blobs['rois'].data.shape,
expected_num_of_rois)
for info in ["bbx_amodal", "viewpoint", "center_proj"]:
pred_info = "pred_" + info
if pred_info in net.blobs:
assert net.blobs[pred_info].data.shape[
0] == expected_num_of_rois
for i, obj_info in enumerate(img_info['object_infos']):
for info in ["bbx_amodal", "viewpoint", "center_proj"]:
pred_info = "pred_" + info
if pred_info in net.blobs:
prediction = np.squeeze(net.blobs[pred_info].data[i, ...])
assert_funcs[info](prediction)
obj_info[info] = prediction.tolist()
return result_dataset
