def __init__(self, experiment_config_path, planes_dir, calib_dir):
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
self.dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split_dir = 'testing'
# These two lines below are necessary for KittiUtils to function properly
self.cluster_split = self.dataset_config.cluster_split
self.config = dataset_config
self.data_split = self.config.data_split
self.name = self.config.name
self.dataset_dir = os.path.expanduser(self.config.dataset_dir)
self.has_labels = self.config.has_labels
self.cluster_split = self.config.cluster_split
self.classes = list(self.config.classes)
self.num_classes = len(self.classes)
self.num_clusters = np.asarray(self.config.num_clusters)
self.model_config = config_builder.proto_to_obj(model_config)
self.paths_config = self.model_config.paths_config
self.checkpoint_dir = self.paths_config.checkpoint_dir
# Build the dataset object
self.dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
self.dataset.train_val_test = "test"
# Score threshold for drawing bounding boxes
self.avod_score_threshold = 0.25
def set_up_model(pipeline_config, data_split):
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config, is_training=False)
dataset_config = config_builder.proto_to_obj(dataset_config)
train_val_test = data_split
# Always run in test mode
dataset_config.data_split = 'test'
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset_config.aug_list = []
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_ssd_model':
model = AvodSSDModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model
def main(_):
parser = argparse.ArgumentParser()
# Example usage
# --checkpoint_name='avod_cars_example'
# --data_split='test'
# --ckpt_indices=50 100 112
# Optional arg:
# --device=0
parser.add_argument('--checkpoint_name',
type=str,
dest='checkpoint_name',
required=True,
help='Checkpoint name must be specified as a str\
and must match the experiment config file name.')
parser.add_argument('--data_split',
type=str,
dest='data_split',
required=True,
help='Data split must be specified e.g. val or test')
parser.add_argument('--ckpt_indices',
type=int,
nargs='+',
dest='ckpt_indices',
required=True,
help='Checkpoint indices must be a set of \
integers with space in between -> 0 10 20 etc')
parser.add_argument('--device',
type=str,
dest='device',
default='0',
help='CUDA device id')
args = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
experiment_config = args.checkpoint_name + '.config'
# Read the config from the experiment folder
experiment_config_path = avod.root_dir() + '/data/outputs/' +\
args.checkpoint_name + '/' + experiment_config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
inference(model_config, eval_config, dataset_config, args.data_split,
args.ckpt_indices)
# Visualize
show_predictions_2d_jhuang.main(args.checkpoint_name)
def test():
parser = argparse.ArgumentParser()
parser.add_argument('--avod_config_path',
type=str,
dest='avod_config_path',
required=True,
help='avod_config_path')
parser.add_argument('--sample_idx',
type=str,
dest='sample_idx',
required=True,
help='sample id')
args = parser.parse_args()
_, _, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.avod_config_path, is_training=False)
dataset = get_dataset(dataset_config, 'val')
idx = np.argwhere(dataset.sample_names==args.sample_idx).squeeze()
# print(idx)
kitti_samples = dataset.load_samples([idx])
sample = kitti_samples[0]
label_mask = np.equal(sample[constants.KEY_LABEL_CLASSES], g_type2onehotclass['Car']+1)
gt_cls = sample[constants.KEY_LABEL_CLASSES][label_mask]
gt_boxes_3d = sample[constants.KEY_LABEL_BOXES_3D][label_mask]
gt_boxes_bev = []
for i in range(len(gt_cls)):
gt_obj = box_3d_encoder.box_3d_to_object_label(gt_boxes_3d[i], gt_cls[i])
gt_corner_3d = compute_box_3d(gt_obj)
gt_boxes_bev.append(gt_corner_3d[:4, [0,2]])
print(gt_boxes_bev)
rpn_out = pickle.load(open("rpn_out/%s" % sample[constants.KEY_SAMPLE_NAME], "rb"))
pos_prop = []
for prop in rpn_out['proposals_and_scores']:
corners = compute_box_3d(box_3d_encoder.box_3d_to_object_label(prop[:7]))
label_idx, iou = find_match_label(corners[:4, [0,2]], gt_boxes_bev)
if iou > 0.65:
pos_prop.append(corners)
pc = sample[constants.KEY_POINT_CLOUD].T
import mayavi.mlab as mlab
from viz_util import draw_lidar, draw_gt_boxes3d
fig = draw_lidar(pc)
fig = draw_gt_boxes3d(pos_prop, fig, draw_text=False, color=(1, 1, 1))
input()
# visualize_rpn_out(sample, rpn_out['proposals_and_scores'])
prediction = pickle.load(open("%s"%sample[constants.KEY_SAMPLE_NAME], "rb"))
print(prediction)
visualize(dataset, sample, prediction)
def main(_):
experiment_config = 'avod_cars_example.config'
# Read the config from the experiment folder
experiment_config_path = avod.root_dir() + '/data/outputs/' +\
'avod_cars_example' + '/' + experiment_config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
test(model_config, eval_config, dataset_config, 'val', -1)
def setUp(self):
tf.test.TestCase.setUp(self)
test_pipeline_config_path = avod.root_dir() + \
'/configs/unittest_pipeline.config'
self.model_config, self.train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# Generate dataset
self.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST, use_defaults=False,
new_cfg=dataset_config)
def main(_):
parser = argparse.ArgumentParser()
default_pipeline_config_path = avod.root_dir() + \
'/configs/avod_cars_example.config'
default_output_dir = '/data/kitti_avod/object/outputs'
parser.add_argument('--pipeline_config',
type=str,
dest='pipeline_config_path',
default=default_pipeline_config_path,
help='Path to the pipeline config')
parser.add_argument('--data_split',
type=str,
dest='data_split',
default='val',
help='Data split for evaluation')
parser.add_argument('--device',
type=str,
dest='device',
default=None,
help='CUDA device id')
parser.add_argument('--output_dir',
type=str,
dest='output_dir',
default=default_output_dir,
help='output dir to save checkpoints')
args = parser.parse_args()
# Parse pipeline config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.pipeline_config_path,
is_training=False,
output_dir=args.output_dir)
# Overwrite data split
dataset_config.data_split = args.data_split
# Set CUDA device id
if args.device:
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
evaluate(model_config, eval_config, dataset_config,
output_dir=args.output_dir)
def main(_):
parser = argparse.ArgumentParser()
# Defaults
default_pipeline_config_path = avod.root_dir() + \
'/configs/avod_cars_example.config'
default_data_split = 'train'
default_device = '1'
parser.add_argument('--pipeline_config',
type=str,
dest='pipeline_config_path',
default=default_pipeline_config_path,
help='Path to the pipeline config')
parser.add_argument('--data_split',
type=str,
dest='data_split',
default=default_data_split,
help='Data split for training')
parser.add_argument('--device',
type=str,
dest='device',
default=default_device,
help='CUDA device id')
parser.add_argument('--output_dir',
type=str,
help='out dir')
args = parser.parse_args()
# Parse pipeline config
output = args.output_dir
is_training=True
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.pipeline_config_path, is_training, output)
# Overwrite data split
dataset_config.data_split = args.data_split
# Set CUDA device id
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
train(model_config, train_config, dataset_config)
def main(args):
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.experiment_config_path, is_training=False,
output_dir=args.output_dir)
assert eval_config.do_eval_sin or eval_config.do_eval_ain, \
"This code only supports repeated evaluation in eval_sin or eval_ain modes"
if eval_config.do_eval_sin and eval_config.do_eval_ain:
raise ValueError('only one mode can be executed: (eval_sin) or (eval_ain)')
print('..Working on {}'.format(args.experiment_config_path))
calc_avg_scores(args.output_dir,model_config,eval_config,dataset_config,args.data_split)
print('..Working on {} (iou05)'.format(args.experiment_config_path))
calc_avg_scores(args.output_dir,model_config,eval_config,dataset_config,args.data_split,iou05=True)
def set_up_model_train_mode(pipeline_config_path, data_split):
"""Returns the model and its train_op."""
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=True)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=data_split,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=data_split,
dataset=dataset)
elif model_name == 'avod_ssd_model':
model = AvodSSDModel(model_config,
train_val_test=data_split,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
prediction_dict = model.build()
losses_dict, total_loss = model.loss(prediction_dict)
# These parameters are required to set up the optimizer
global_summaries = set([])
global_step_tensor = tf.Variable(0, trainable=False)
training_optimizer = optimizer_builder.build(train_config.optimizer,
global_summaries,
global_step_tensor)
# Set up the train op
train_op = slim.learning.create_train_op(total_loss, training_optimizer)
return model, train_op
def main(_):
parser = argparse.ArgumentParser()
default_pipeline_config_path = avod.root_dir() + \
'/configs/avod_cars_example.config'
parser.add_argument('--pipeline_config',
type=str,
dest='pipeline_config_path',
default=default_pipeline_config_path,
help='Path to the pipeline config')
parser.add_argument('--data_split',
type=str,
dest='data_split',
default='val',
help='Data split for evaluation')
parser.add_argument('--device',
type=str,
dest='device',
default='0',
help='CUDA device id')
args = parser.parse_args()
# Parse pipeline config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.pipeline_config_path,
is_training=False)
# Overwrite data split
dataset_config.data_split = args.data_split
# Set CUDA device id
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
evaluate(model_config, eval_config, dataset_config)
def set_up_model_test_mode(pipeline_config_path, data_split):
"""Returns the model and its config in test mode."""
model_config, _, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=False)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
# Use the validation set
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
# Remove augmentation when in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config, train_val_test='test', dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config, train_val_test='test', dataset=dataset)
elif model_name == 'avod_ssd_model':
model = AvodSSDModel(model_config,
train_val_test='test',
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model, model_config
def main():
parser = argparse.ArgumentParser()
# Example usage
# --checkpoint_name='avod_exp_example'
# --base_dir='/home/<username>/GTAData/'
parser.add_argument('--checkpoint_name',
type=str,
dest='checkpoint_name',
required=True,
help='Checkpoint name must be specified as a str\
and must match the experiment config file name.')
parser.add_argument('--base_dir',
type=str,
dest='base_dir',
required=True,
help='Base data directory must be specified')
args = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
experiment_config = args.checkpoint_name + '.config'
# Read the config from the experiment folder
experiment_config_path = avod.root_dir() + '/data/outputs/' +\
args.checkpoint_name + '/' + experiment_config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
filter_gt_labels(dataset_config, args.base_dir)
def infer_main(checkpoint_name,
ckpt_indices,
additional_cls,
start_perspective=0):
experiment_config = checkpoint_name + '.config'
# Read the config from the experiment folder
experiment_config_path = avod.root_dir() + '/data/outputs/' +\
checkpoint_name + '/' + experiment_config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
base_dir = os.path.split(cfg.DATASET_DIR)[0] + '/'
os.environ['CUDA_VISIBLE_DEVICES'] = cfg.CUDA_DEVICE
inference(model_config,
eval_config,
dataset_config,
base_dir,
ckpt_indices,
additional_cls,
start_perspective=start_perspective)
def main():
"""This demo shows RPN proposals and AVOD predictions in the
3D point cloud.
Keys:
F1: Toggle proposals
F2: Toggle predictions
F3: Toggle 3D voxel grid
F4: Toggle point cloud
F5: Toggle easy ground truth objects (Green)
F6: Toggle medium ground truth objects (Orange)
F7: Toggle hard ground truth objects (Red)
F8: Toggle all ground truth objects (default off)
F9: Toggle ground slice filter (default off)
F10: Toggle offset slice filter (default off)
"""
##############################
# Options
##############################
rpn_score_threshold = 0.1
avod_score_threshold = 0.1
proposals_line_width = 1.0
predictions_line_width = 3.0
show_orientations = True
point_cloud_source = 'depth'
# Config file folder, default (<avod_root>/data/outputs/<checkpoint_name>)
config_dir = None
checkpoint_name = 'pyramid_cars_with_aug_example'
global_step = None # Latest checkpoint
global_step = 83000
#data_split = 'val_half'
data_split = 'val'
# data_split = 'test'
# Show 3D iou text
draw_ious_3d = True
name_list =[]
#name_file = '/media/wavelab/d3cd89ab-7705-4996-94f3-01da25ba8f50/moosey/val.txt'
#with open(name_file) as f:
#for line in f:
#newline = line.replace("\n","")
#name_list.append(newline)
#name_list =['0000000003','0000000009','0000000016','0000000233','0000000234','0000000236','0000000422','0000000473','0000000490','0000000494','0000000547','0000000655',\
#'0000000679','0000000690','0000000692','0000000781']
name_list =['0000000004']
for names in name_list:
sample_name = names
#sample_name = None
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# val split
# sample_name = '000181'
# sample_name = '000751'
# sample_name = '000843'
# sample_name = '000944'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# val split
# sample_name = '000015'
# sample_name = '000048'
# sample_name = '000058'
# sample_name = '000076' # Medium, few ped, 1 cyc
# sample_name = '000108'
# sample_name = '000118'
# sample_name = '000145'
# sample_name = '000153'
# sample_name = '000186'
# sample_name = '000195'
# sample_name = '000199'
# sample_name = '000397'
# sample_name = '004425'
# sample_name = '004474' # Hard, many ped, 1 cyc
# sample_name = '004657' # Hard, Few cycl, few ped
# sample_name = '006071'
# sample_name = '006828' # Hard, Few cycl, few ped
# sample_name = '006908' # Hard, Few cycl, few ped
# sample_name = '007412'
# sample_name = '007318' # Hard, Few cycl, few ped
##############################
# End of Options
##############################
if data_split == 'test':
draw_ious_3d = False
if config_dir is None:
config_dir = avod.root_dir() + '/data/outputs/' + checkpoint_name
# Parse experiment config
pipeline_config_file = \
config_dir + '/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
dataset_config.data_split = data_split
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_names[sample_idx]
##############################
# Setup Paths
##############################
img_idx = int(sample_name)
# Text files directory
proposals_and_scores_dir = avod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = avod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_predictions_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
# Output images directory
img_out_dir = avod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/predictions/images_3d/{}/{}/{}'.format(dataset.data_split,
global_step,
rpn_score_threshold)
if not os.path.exists(img_out_dir):
os.makedirs(img_out_dir)
##############################
# Proposals
##############################
# Load proposals from files
proposals_and_scores = np.loadtxt(proposals_and_scores_dir +
"/{}/{}.txt".format(global_step,
sample_name))
proposals = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
rpn_score_mask = proposal_scores > rpn_score_threshold
proposals = proposals[rpn_score_mask]
proposal_scores = proposal_scores[rpn_score_mask]
print('Proposals:', len(proposal_scores), proposal_scores)
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposals]
##############################
# Predictions
##############################
# Load proposals from files
predictions_and_scores = np.loadtxt(predictions_and_scores_dir +
"/{}/{}.txt".format(
global_step,
sample_name)).reshape(-1, 9)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_types = np.asarray(predictions_and_scores[:, 8], dtype=np.int32)
avod_score_mask = prediction_scores >= avod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
prediction_scores = prediction_scores[avod_score_mask]
print('Predictions: ', len(prediction_scores), prediction_scores)
final_predictions = np.copy(prediction_boxes_3d)
# # Swap l, w for predictions where w > l
# swapped_indices = predictions[:, 4] > predictions[:, 3]
# final_predictions[swapped_indices, 3] = predictions[swapped_indices, 4]
# final_predictions[swapped_indices, 4] = predictions[swapped_indices, 3]
prediction_objs = []
for pred_idx in range(len(final_predictions)):
prediction_box_3d = final_predictions[pred_idx]
prediction_type = dataset.classes[prediction_types[pred_idx]]
prediction_obj = box_3d_encoder.box_3d_to_object_label(
prediction_box_3d, obj_type=prediction_type)
prediction_objs.append(prediction_obj)
##############################
# Ground Truth
##############################
if dataset.has_labels:
# Get ground truth labels
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset, img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# 3D IoU
##############################
if draw_ious_3d:
# Convert to box_3d
all_gt_boxes_3d = [box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in all_gt_objs]
pred_boxes_3d = [box_3d_encoder.object_label_to_box_3d(pred_obj)
for pred_obj in prediction_objs]
max_ious_3d = demo_utils.get_max_ious_3d(all_gt_boxes_3d,
pred_boxes_3d)
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
print("***************")
print(point_cloud_source)
print(img_idx)
print(image.shape)
point_cloud = dataset.kitti_utils.get_point_cloud(point_cloud_source,
img_idx,
image_shape=image.shape)
print("This is the shape of the point_cloud")
print(point_cloud.shape)
point_cloud = np.asarray(point_cloud)
# Filter point cloud to extents
area_extents = np.asarray([[-40, 40], [-5, 3], [0, 70]])
bev_extents = area_extents[[0, 2]]
points = point_cloud.T
point_filter = obj_utils.get_point_filter(point_cloud, area_extents)
points = points[point_filter]
point_colours = vis_utils.project_img_to_point_cloud(points,
image,
dataset.calib_dir,
img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1,
create_leaf_layout=False)
# Ground plane
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
vtk_axes = vtk.vtkAxesActor()
vtk_axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for proposal boxes
vtk_proposal_boxes = VtkBoxes()
vtk_proposal_boxes.set_line_width(proposals_line_width)
vtk_proposal_boxes.set_objects(proposal_objs,
COLOUR_SCHEME_PREDICTIONS)
# Create VtkBoxes for prediction boxes
vtk_prediction_boxes = VtkPyramidBoxes()
vtk_prediction_boxes.set_line_width(predictions_line_width)
vtk_prediction_boxes.set_objects(prediction_objs,
COLOUR_SCHEME_PREDICTIONS,
show_orientations)
# Create VtkBoxes for ground truth
vtk_hard_gt_boxes = VtkBoxes()
vtk_medium_gt_boxes = VtkBoxes()
vtk_easy_gt_boxes = VtkBoxes()
vtk_all_gt_boxes = VtkBoxes()
vtk_hard_gt_boxes.set_objects(hard_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_medium_gt_boxes.set_objects(medium_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_easy_gt_boxes.set_objects(easy_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_all_gt_boxes.set_objects(all_gt_objs, VtkBoxes.COLOUR_SCHEME_KITTI,
show_orientations)
# Create VtkTextLabels for 3D ious
vtk_text_labels = VtkTextLabels()
if draw_ious_3d and len(all_gt_boxes_3d) > 0:
gt_positions_3d = np.asarray(all_gt_boxes_3d)[:, 0:3]
vtk_text_labels.set_text_labels(
gt_positions_3d,
['{:0.3f}'.format(iou_3d) for iou_3d in max_ious_3d])
# Create VtkGroundPlane
vtk_ground_plane = VtkGroundPlane()
vtk_slice_bot_plane = VtkGroundPlane()
vtk_slice_top_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, bev_extents)
vtk_slice_bot_plane.set_plane(ground_plane + [0, 0, 0, -0.2], bev_extents)
vtk_slice_top_plane.set_plane(ground_plane + [0, 0, 0, -2.0], bev_extents)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_prediction_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_text_labels.vtk_actor)
# Add ground plane and slice planes
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_bot_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_top_plane.vtk_actor)
vtk_renderer.AddActor(vtk_axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(3)
vtk_proposal_boxes.vtk_actor.SetVisibility(0)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.SetVisibility(0)
vtk_slice_bot_plane.vtk_actor.SetVisibility(0)
vtk_slice_top_plane.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_bot_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_top_plane.vtk_actor.GetProperty().SetOpacity(0.9)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(140.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
avod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
# Add custom interactor to toggle actor visibilities
custom_interactor = vis_utils.CameraInfoInteractorStyle([
vtk_proposal_boxes.vtk_actor,
vtk_prediction_boxes.vtk_actor,
vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor,
vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor,
vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor,
vtk_ground_plane.vtk_actor,
vtk_slice_bot_plane.vtk_actor,
vtk_slice_top_plane.vtk_actor,
vtk_text_labels.vtk_actor,
])
vtk_render_window_interactor.SetInteractorStyle(custom_interactor)
# Render in VTK
vtk_render_window.Render()
# Take a screenshot
window_to_image_filter = vtk.vtkWindowToImageFilter()
window_to_image_filter.SetInput(vtk_render_window)
window_to_image_filter.Update()
png_writer = vtk.vtkPNGWriter()
file_name = img_out_dir + "/{}.png".format(sample_name)
png_writer.SetFileName(file_name)
png_writer.SetInputData(window_to_image_filter.GetOutput())
png_writer.Write()
print('Screenshot saved to ', file_name)
#vtk_render_window_interactor.Start() # Blocking
vtk_render_window_interactor.Initialize() # Non-Blocking
def main():
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
##############################
# Options
##############################
checkpoint_name = 'fpn_people_dual_NHSP_train'#'fpn_people_dual_SHPL_train'#'pyramid_people_with_NHSP_example_train'#'pyramid_people_with_NHSP_example'
data_split = 'val'#'test'#'val'
global_steps = None
# global_steps = [28000, 19000, 33000, 34000]
score_threshold = 0.1
save_2d = False # Save 2D predictions
save_3d = True # Save 2D and 3D predictions together
save_alphas = True # Save alphas (observation angles)
# Checkpoints below this are skipped
min_step = 20000
##############################
# End of Options
##############################
# Parse experiment config
pipeline_config_file = \
avod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
# Overwrite defaults
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.aug_list = []
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Get available prediction folders
predictions_root_dir = avod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
final_predictions_root_dir = predictions_root_dir + \
'/final_predictions_and_scores/' + dataset.data_split
print('Converting detections from', final_predictions_root_dir)
if not global_steps:
global_steps = os.listdir(final_predictions_root_dir)
global_steps.sort(key=int)
print('Checkpoints found ', global_steps)
for step_idx in range(len(global_steps)):
global_step = global_steps[step_idx]
# Skip first checkpoint
if int(global_step) < min_step:
continue
final_predictions_dir = final_predictions_root_dir + \
'/' + str(global_step)
# 2D and 3D prediction directories
kitti_predictions_2d_dir = predictions_root_dir + \
'/kitti_predictions_2d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
kitti_predictions_3d_dir = predictions_root_dir + \
'/kitti_predictions_3d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
if save_2d and not os.path.exists(kitti_predictions_2d_dir):
os.makedirs(kitti_predictions_2d_dir)
if save_3d and not os.path.exists(kitti_predictions_3d_dir):
os.makedirs(kitti_predictions_3d_dir)
# Do conversion
num_samples = dataset.num_samples
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting detections from:', final_predictions_dir)
if save_2d:
print('2D Detections saved to:', kitti_predictions_2d_dir)
if save_3d:
print('3D Detections saved to:', kitti_predictions_3d_dir)
for sample_idx in range(num_samples):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
prediction_file = sample_name + '.txt'
kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \
'/' + prediction_file
kitti_predictions_3d_file_path = kitti_predictions_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
# # Swap l, w for predictions where w > l
# swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
# fixed_predictions = np.copy(all_predictions)
# fixed_predictions[swapped_indices, 3] = all_predictions[
# swapped_indices, 4]
# fixed_predictions[swapped_indices, 4] = all_predictions[
# swapped_indices, 3]
score_filter = all_predictions[:, 7] >= score_threshold
all_predictions = all_predictions[score_filter]
# If no predictions, skip to next file
if len(all_predictions) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(dataset.get_rgb_image_path(sample_name))
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, stereo_calib_p2,
truncate=True, image_size=image.size)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
else:
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Get object types
all_pred_classes = all_predictions[:, 8].astype(np.int32)
obj_types = [dataset.classes[class_idx]
for class_idx in all_pred_classes]
# Truncation and Occlusion are always empty (see below)
# Alpha
if not save_alphas:
kitti_predictions[:, 3] = -10 * \
np.ones((len(kitti_predictions)), dtype=np.int32)
else:
alphas = all_predictions[:, 6] - \
np.arctan2(all_predictions[:, 0], all_predictions[:, 2])
kitti_predictions[:, 3] = alphas
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones((len(kitti_predictions), 2),
dtype=np.int32)
# Empty 3D (x, y, z)
kitti_empty_2 = -1 * np.ones((len(kitti_predictions), 3),
dtype=np.int32)
# Empty 3D (h, w, l)
kitti_empty_3 = -1000 * np.ones((len(kitti_predictions), 3),
dtype=np.int32)
# Empty 3D (ry)
kitti_empty_4 = -10 * np.ones((len(kitti_predictions), 1),
dtype=np.int32)
# Stack 2D predictions text
kitti_text_2d = np.column_stack([obj_types,
kitti_empty_1,
kitti_predictions[:, 3:8],
kitti_empty_2,
kitti_empty_3,
kitti_empty_4,
kitti_predictions[:, 15]])
# Stack 3D predictions text
kitti_text_3d = np.column_stack([obj_types,
kitti_empty_1,
kitti_predictions[:, 3:16]])
# Save to text files
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, kitti_text_2d,
newline='\r\n', fmt='%s')
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d,
newline='\r\n', fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
def inference(rpn_model_path, detect_model_path, avod_config_path):
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
avod_config_path, is_training=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
dataset = get_dataset(dataset_config, 'val')
# run avod proposal network
rpn_endpoints, sess1, rpn_model = get_proposal_network(model_config, dataset, rpn_model_path)
end_points, sess2 = get_detection_network(detect_model_path)
all_prediction = []
all_id_list = None
all_2d_boxes = []
for idx in range(3769):
feed_dict1 = rpn_model.create_feed_dict()
kitti_samples = dataset.load_samples([idx])
sample = kitti_samples[0]
'''
if sample[constants.KEY_SAMPLE_NAME] < '001100':
continue
if sample[constants.KEY_SAMPLE_NAME] > '001200':
break
'''
start_time = time.time()
rpn_predictions = sess1.run(rpn_endpoints, feed_dict=feed_dict1)
top_anchors = rpn_predictions[RpnModel.PRED_TOP_ANCHORS]
top_proposals = box_3d_encoder.anchors_to_box_3d(top_anchors)
softmax_scores = rpn_predictions[RpnModel.PRED_TOP_OBJECTNESS_SOFTMAX]
proposals_and_scores = np.column_stack((top_proposals,
softmax_scores))
top_img_roi = rpn_predictions[RpnModel.PRED_TOP_IMG_ROI]
top_bev_roi = rpn_predictions[RpnModel.PRED_TOP_BEV_ROI]
roi_num = len(top_img_roi)
top_img_roi = np.reshape(top_img_roi, (roi_num, -1))
top_bev_roi = np.reshape(top_bev_roi, (roi_num, -1))
roi_features = np.column_stack((top_img_roi, top_bev_roi))
'''
# save proposal
if os.path.exists(os.path.join('/data/ssd/public/jlliu/Kitti/object/training/proposal', '%s.txt'%(sample[constants.KEY_SAMPLE_NAME]))):
continue
np.savetxt(os.path.join('./proposals_and_scores/', '%s.txt'%sample[constants.KEY_SAMPLE_NAME]), proposals_and_scores, fmt='%.3f')
np.savetxt(os.path.join('./roi_features/', '%s_roi.txt'%sample[constants.KEY_SAMPLE_NAME]), roi_features, fmt='%.5f')
print('save ' + sample[constants.KEY_SAMPLE_NAME])
'''
# run frustum_pointnets_v2
point_clouds, feature_vec, rot_angle_list, prop_cls_labels = get_pointnet_input(sample, proposals_and_scores, roi_features)
try:
prediction = detect_batch(sess2, end_points, point_clouds, feature_vec, rot_angle_list, prop_cls_labels)
except:
traceback.print_exc()
continue
elapsed_time = time.time() - start_time
print(sample[constants.KEY_SAMPLE_NAME], elapsed_time)
# concat all predictions for kitti eval
id_list = np.ones((len(prediction),)) * int(sample[constants.KEY_SAMPLE_NAME])
if all_id_list is None:
all_id_list = id_list
else:
all_id_list = np.concatenate((all_id_list, id_list), axis=0)
for pred in prediction:
obj = box_3d_encoder.box_3d_to_object_label(np.array(pred[0:7]), obj_type=type_whitelist[pred[8]])
corners = compute_box_3d(obj)
projected = calib_utils.project_to_image(corners.T, sample[constants.KEY_STEREO_CALIB_P2])
x1 = np.amin(projected[0])
y1 = np.amin(projected[1])
x2 = np.amax(projected[0])
y2 = np.amax(projected[1])
all_2d_boxes.append([x1, y1, x2, y2])
all_prediction += prediction
# save result
pickle.dump({'proposals_and_scores': proposals_and_scores, 'roi_features': roi_features}, open("rpn_out/%s"%sample[constants.KEY_SAMPLE_NAME], "wb"))
pickle.dump(prediction, open('final_out/%s' % sample[constants.KEY_SAMPLE_NAME], 'wb'))
visualize(dataset, sample, prediction)
# for kitti eval
write_detection_results('./detection_results', all_prediction, all_id_list, all_2d_boxes)
def main(_):
parser = argparse.ArgumentParser()
# Example usage
# --checkpoint_name='avod_cars_example'
# --data_split='test'
# --ckpt_indices=50 100 112
# Optional arg:
# --device=0
default_output_dir = '/data/kitti_avod/object/outputs'
# parser.add_argument('--checkpoint_name',
# type=str,
# dest='checkpoint_name',
# required=True,
# help='Checkpoint name must be specified as a str\
# and must match the experiment config file name.')
parser.add_argument('--experiment_config',
type=str,
required=True,
dest='experiment_config_path',
help='Path to the experiment config must be specified')
parser.add_argument('--data_split',
type=str,
dest='data_split',
required=True,
help='Data split must be specified e.g. val or test')
parser.add_argument('--ckpt_indices',
type=int,
nargs='+',
dest='ckpt_indices',
required=True,
help='Checkpoint indices must be a set of \
integers with space in between -> 0 10 20 etc')
parser.add_argument('--device',
type=str,
dest='device',
default=None,
help='CUDA device id')
parser.add_argument('--output_dir',
type=str,
dest='output_dir',
default=default_output_dir,
help='output dir to save checkpoints')
parser.add_argument('--force_sin_input_name',
type=str,
dest='force_sin_input_name',
default=None,
help='force sin_input_name to run only for the given input')
args = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
# experiment_config = args.checkpoint_name + '.config'
# # Read the config from the experiment folder
# experiment_config_path = avod.root_dir() + '/data/outputs/' +\
# args.checkpoint_name + '/' + experiment_config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.experiment_config_path, is_training=False,
output_dir=args.output_dir)
if args.device:
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
inference(model_config, eval_config,
dataset_config, args.data_split,
args.ckpt_indices,
output_dir=args.output_dir,
force_sin_input_name=args.force_sin_input_name)
def main(_):
parser = argparse.ArgumentParser()
# Example usage
# --checkpoint_name='avod_exp_example'
# --data_split='test'
# --ckpt_indices=50 100 112
# Optional arg:
# --device=0
# --additional_cls=False
parser.add_argument('--checkpoint_name',
type=str,
dest='checkpoint_name',
required=True,
help='Checkpoint name must be specified as a str\
and must match the experiment config file name.')
parser.add_argument('--base_dir',
type=str,
dest='base_dir',
required=True,
help='Base data directory must be specified')
parser.add_argument('--ckpt_indices',
type=int,
nargs='+',
dest='ckpt_indices',
required=True,
help='Checkpoint indices must be a set of \
integers with space in between -> 0 10 20 etc')
parser.add_argument('--device',
type=str,
dest='device',
default='0',
help='CUDA device id')
parser.add_argument('--additional_cls',
dest='additional_cls',
action='store_true',
default=False,
help='If detections are from an additional class\
i.e. another class has already been pre-processed.')
args = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
experiment_config = args.checkpoint_name + '.config'
# Read the config from the experiment folder
experiment_config_path = avod.root_dir() + '/data/outputs/' +\
args.checkpoint_name + '/' + experiment_config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
inference(model_config, eval_config, dataset_config, args.base_dir,
args.ckpt_indices, args.additional_cls)
