def main():
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST)
dataset_config.data_split = "trainval"
unittest_dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
gen_label_clusters.main(unittest_dataset)
gen_mini_batches.main(unittest_dataset)
def get_fake_dataset(self, data_split, directory):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
# Overwrite config values
dataset_config.data_split = data_split
dataset_config.dataset_dir = directory
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
return dataset
def setUpClass(cls):
# Initialize the Kitti dataset
test_dir = tests.test_path()
# Get the unittest-kitti dataset
dataset_builder = DatasetBuilder()
cls.dataset = dataset_builder.build_kitti_dataset(
dataset_builder.KITTI_UNITTEST)
cls.log_dir = test_dir + '/logs'
cls.bev_vgg_cls = vgg.BevVggClassification()
def test_project_to_image_space_tensors(self):
anchors = np.asarray([[0, 0, 3, 2, 0, 6], [3, 0, 3, 2, 0, 2]],
dtype=np.float64)
img_idx = int('000217')
img_shape = [375, 1242]
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
dataset_config.data_split = 'train'
dataset_config.dataset_dir = tests.test_path() + \
"/datasets/Kitti/object"
dataset = DatasetBuilder().build_kitti_dataset(dataset_config)
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
# Project the 3D points in numpy space
img_corners, img_corners_norm = anchor_projector.project_to_image_space(
anchors, stereo_calib_p2, img_shape)
# convert the required params to tensors
tf_stereo_calib_p2 = tf.convert_to_tensor(stereo_calib_p2,
dtype=tf.float32)
tf_anchors = tf.convert_to_tensor(anchors, dtype=tf.float32)
tf_img_shape = tf.convert_to_tensor(img_shape, dtype=tf.float32)
# Project the 3D points in tensor space
img_corners_tensor, img_corners_norm_tensor = \
anchor_projector.tf_project_to_image_space(tf_anchors,
tf_stereo_calib_p2,
tf_img_shape)
sess = tf.Session()
with sess.as_default():
img_corners_out = img_corners_tensor.eval()
img_corners_norm_out = img_corners_norm_tensor.eval()
np.testing.assert_allclose(img_corners,
img_corners_out,
atol=1e-04,
err_msg='Incorrect corner projection')
np.testing.assert_allclose(
img_corners_norm,
img_corners_norm_out,
atol=1e-04,
err_msg='Incorrect normalized corner projection')
def train(model_config, train_config, dataset_config):
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
train_val_test = 'train'
model_name = model_config.model_name
with tf.Graph().as_default():
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'bev_only_rpn_model':
model = BevOnlyRpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'bev_only_avod_model':
model = BevOnlyAvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
trainer.train(model, train_config)
def __init__(self, experiment_config_path, planes_dir, calib_dir):
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
self.dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split_dir = 'testing'
# These two lines below are necessary for KittiUtils to function properly
self.cluster_split = self.dataset_config.cluster_split
self.config = dataset_config
self.data_split = self.config.data_split
self.name = self.config.name
self.dataset_dir = os.path.expanduser(self.config.dataset_dir)
self.has_labels = self.config.has_labels
self.cluster_split = self.config.cluster_split
self.classes = list(self.config.classes)
self.num_classes = len(self.classes)
self.num_clusters = np.asarray(self.config.num_clusters)
self.model_config = config_builder.proto_to_obj(model_config)
self.paths_config = self.model_config.paths_config
self.checkpoint_dir = self.paths_config.checkpoint_dir
# Build the dataset object
self.dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
self.dataset.train_val_test = "test"
# Score threshold for drawing bounding boxes
self.avod_score_threshold = 0.25
def set_up_model(pipeline_config, data_split):
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config, is_training=False)
dataset_config = config_builder.proto_to_obj(dataset_config)
train_val_test = data_split
# Always run in test mode
dataset_config.data_split = 'test'
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset_config.aug_list = []
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_ssd_model':
model = AvodSSDModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model
def train(model_config, train_config, dataset_config):
# 读取config文件里面的详细内容，包括：
# model_config：模型参数
# train_config：训练参数
# dataset_config：数据集参数
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
train_val_test = 'train'
# 包括avod_model 和 rpn_model
model_name = model_config.model_name
with tf.Graph().as_default():
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
trainer.train(model, train_config)
def test_data_splits(self):
bad_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST)
# Test invalid splits
bad_config.data_split = "bad"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "train"
bad_config.data_split = "training"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "val"
bad_config.data_split = "validation"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "test"
bad_config.data_split = "testing"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Train split
train_dataset = self.get_fake_dataset('train', self.fake_kitti_dir)
self.assertEqual(train_dataset.num_samples, 7)
# Validation split
validation_dataset = self.get_fake_dataset('val', self.fake_kitti_dir)
self.assertEqual(validation_dataset.num_samples, 6)
# Train + validation split
trainval_dataset = self.get_fake_dataset('trainval',
self.fake_kitti_dir)
self.assertEqual(trainval_dataset.num_samples, 13)
# Test split
test_dataset = self.get_fake_dataset('test', self.fake_kitti_dir)
self.assertEqual(test_dataset.num_samples, 10)
def train(model_config, train_config, dataset_config):
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
train_val_test = 'train'
model_name = model_config.model_name
with tf.Graph().as_default():
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'retinanet_model':
model = RetinanetModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
#import pdb
#pdb.set_trace()
trainer.train(model, train_config)
def inference(model_config, eval_config, dataset_config, data_split,
ckpt_indices):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
eval_config.eval_mode = 'test'
eval_config.evaluate_repeatedly = False
dataset_config.has_labels = False
# Enable this to see the actually memory being used
eval_config.allow_gpu_mem_growth = True
eval_config = config_builder.proto_to_obj(eval_config)
# Grab the checkpoint indices to evaluate
eval_config.ckpt_indices = ckpt_indices
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'bev_only_rpn_model':
model = BevOnlyRpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'bev_only_avod_model':
model = BevOnlyAvodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
model_evaluator.run_latest_checkpoints()
def setUpClass(cls):
pipeline_config = pipeline_pb2.NetworkPipelineConfig()
dataset_config = pipeline_config.dataset_config
config_path = avod.root_dir() + '/configs/unittest_model.config'
cls.model_config = config_build.get_model_config_from_file(config_path)
dataset_config.MergeFrom(DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
def filter_gt_labels(dataset_config, base_dir):
"""Plots detection errors for xyz, lwh, ry, and shows 3D IoU with
ground truth boxes
"""
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_dir = base_dir
dataset_config.dataset_dir = base_dir
dataset_config.data_split = 'train'
dataset_config.data_split_dir = 'training'
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
dataset.classes = ['Car', 'Bus', 'Pedestrian', 'Cyclist', 'Truck']
dataset.aug_sample_list = []
difficulty = 2
out_dir = base_dir + dataset_config.data_split_dir + '/label_filtered_2/'
if not os.path.exists(out_dir):
os.makedirs(out_dir)
print("Saving filtered labels to: ", out_dir)
# TODO read in augmented labels to filter by 3D point count
# Also changes synthetic to true in read_labels
# dataset.label_dir = base_dir + '/' + dataset_config.data_split_dir + '/label_aug_2/'
dataset.label_dir = base_dir + '/' + dataset_config.data_split_dir + '/label_2/'
files = os.listdir(dataset.label_dir)
num_files = len(files)
sample_idx = 0
for file in files:
sys.stdout.flush()
sys.stdout.write('\r{} / {}'.format(sample_idx + 1, num_files))
sample_idx = sample_idx + 1
filepath = dataset.label_dir + '/' + file
idx = int(os.path.splitext(file)[0])
# Get filtered ground truth
all_gt_objs = obj_utils.read_labels(dataset.label_dir,
idx,
synthetic=False)
if all_gt_objs != None:
all_gt_objs = dataset.kitti_utils.filter_labels(
all_gt_objs,
max_forward=dataset.kitti_utils.area_extents[2, 1],
max_side=dataset.kitti_utils.area_extents[0, 1])
# Save gt to output file
save_filtered_objs(all_gt_objs, idx, out_dir)
def inferPerspective(model_config, eval_config, dataset_config,
additional_cls):
model_name = model_config.model_name
entity_perspect_dir = dataset_config.dataset_dir + dataset_config.data_split_dir + '/'
logging.debug("Inferring perspective: %s\n %s\n %s",
dataset_config.data_split, entity_perspect_dir,
dataset_config.dataset_dir)
files_in_range = create_split.create_split(dataset_config.dataset_dir,
entity_perspect_dir,
dataset_config.data_split)
# If there are no files within the range cfg.MIN_IDX, cfg.MAX_IDX
# then skip this perspective
if not files_in_range:
logging.debug(
"No files within the range cfg.MIN_IDX, cfg.MAX_IDX, skipping perspective"
)
return
if not additional_cls:
estimate_ground_planes.estimate_ground_planes(entity_perspect_dir,
dataset_config, 0)
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
#Switch inference output directory
model_config.paths_config.pred_dir = entity_perspect_dir + '/{}/'.format(
cfg.AVOD_OUTPUT_DIR)
logging.debug("Prediction directory: %s",
model_config.paths_config.pred_dir)
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
model_evaluator.run_latest_checkpoints()
save_kitti_predictions.convertPredictionsToKitti(
dataset, model_config.paths_config.pred_dir, additional_cls)
def set_up_model_test_mode(pipeline_config_path, data_split):
"""Returns the model and its config in test mode."""
model_config, _, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=False)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
# Use the validation set
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
# Remove augmentation when in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config, train_val_test='test', dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config, train_val_test='test', dataset=dataset)
elif model_name == 'avod_ssd_model':
model = AvodSSDModel(model_config,
train_val_test='test',
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model, model_config
def setUp(self):
tf.test.TestCase.setUp(self)
test_pipeline_config_path = avod.root_dir() + \
'/configs/unittest_pipeline.config'
self.model_config, self.train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# Generate dataset
self.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST, use_defaults=False,
new_cfg=dataset_config)
def main(dataset=None):
if not dataset:
dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_TRAIN)
label_cluster_utils = dataset.kitti_utils.label_cluster_utils
print("Generating clusters in {}/{}".format(label_cluster_utils.data_dir,
dataset.data_split))
clusters, std_devs = dataset.get_cluster_info()
print("Clusters generated")
print("classes: {}".format(dataset.classes))
print("num_clusters: {}".format(dataset.num_clusters))
print("all_clusters:\n {}".format(clusters))
print("all_std_devs:\n {}".format(std_devs))
def get_dataset(dataset_config, data_split):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = data_split in ['train', 'val']
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
return dataset
def train(model_config, train_config, dataset_config):
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
train_val_test = 'train'
model_name = model_config.model_name
with tf.Graph().as_default():
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
trainer.train(model, train_config)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
trainer.train(model, train_config)
elif model_name == 'avod_moe_model':
model = AvodMoeModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
trainer_moe.train(model, train_config)
elif model_name == 'epbrm':
model = epBRM(model_config, dataset=dataset)
epbrm_trainer.train(model, train_config)
elif model_name == 'avod_model_new_bev':
model = AvodModelBEV(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_model_double_fusion_new_bev':
model = AvodModelDoubleFusionBEV(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
if model_name == 'avod_model_new_bev' or model_name == 'avod_model_double_fusion_new_bev':
trainer_new_bev.train(model, train_config)
else:
trainer.train(model, train_config)
def set_up_model_train_mode(pipeline_config_path, data_split):
"""Returns the model and its train_op."""
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=True)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=data_split,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=data_split,
dataset=dataset)
elif model_name == 'avod_ssd_model':
model = AvodSSDModel(model_config,
train_val_test=data_split,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
prediction_dict = model.build()
losses_dict, total_loss = model.loss(prediction_dict)
# These parameters are required to set up the optimizer
global_summaries = set([])
global_step_tensor = tf.Variable(0, trainable=False)
training_optimizer = optimizer_builder.build(train_config.optimizer,
global_summaries,
global_step_tensor)
# Set up the train op
train_op = slim.learning.create_train_op(total_loss, training_optimizer)
return model, train_op
def main(dataset=None):
"""Generates anchors info which is used for mini batch sampling.
Processing on 'Cars' can be split into multiple processes, see the Options
section for configuration.
Args:
dataset: KittiDataset (optional)
If dataset is provided, only generate info for that dataset.
If no dataset provided, generates info for all 3 classes.
"""
if dataset is not None:
do_preprocessing(dataset, None)
return
car_dataset_config_path = avod.root_dir() + \
'/configs/mb_preprocessing/rpn_cars.config'
ped_dataset_config_path = avod.root_dir() + \
'/configs/mb_preprocessing/rpn_pedestrians.config'
cyc_dataset_config_path = avod.root_dir() + \
'/configs/mb_preprocessing/rpn_cyclists.config'
ppl_dataset_config_path = avod.root_dir() + \
'/configs/mb_preprocessing/rpn_people.config'
##############################
# Options
##############################
# Serial vs parallel processing
in_parallel = True
process_car = True # Cars
process_ped = False # Pedestrians
process_cyc = False # Cyclists
process_ppl = True # People (Pedestrians + Cyclists)
# Number of child processes to fork, samples will
# be divided evenly amongst the processes (in_parallel must be True)
num_car_children = 8
num_ped_children = 8
num_cyc_children = 8
num_ppl_children = 8
##############################
# Dataset setup
##############################
if process_car:
car_dataset = DatasetBuilder.load_dataset_from_config(
car_dataset_config_path)
if process_ped:
ped_dataset = DatasetBuilder.load_dataset_from_config(
ped_dataset_config_path)
if process_cyc:
cyc_dataset = DatasetBuilder.load_dataset_from_config(
cyc_dataset_config_path)
if process_ppl:
ppl_dataset = DatasetBuilder.load_dataset_from_config(
ppl_dataset_config_path)
##############################
# Serial Processing
##############################
if not in_parallel:
if process_car:
do_preprocessing(car_dataset, None)
if process_ped:
do_preprocessing(ped_dataset, None)
if process_cyc:
do_preprocessing(cyc_dataset, None)
if process_ppl:
do_preprocessing(ppl_dataset, None)
print('All Done (Serial)')
##############################
# Parallel Processing
##############################
else:
# List of all child pids to wait on
all_child_pids = []
# Cars
if process_car:
car_indices_split = split_indices(car_dataset, num_car_children)
split_work(
all_child_pids,
car_dataset,
car_indices_split,
num_car_children)
# Pedestrians
if process_ped:
ped_indices_split = split_indices(ped_dataset, num_ped_children)
split_work(
all_child_pids,
ped_dataset,
ped_indices_split,
num_ped_children)
# Cyclists
if process_cyc:
cyc_indices_split = split_indices(cyc_dataset, num_cyc_children)
split_work(
all_child_pids,
cyc_dataset,
cyc_indices_split,
num_cyc_children)
# People (Pedestrians + Cyclists)
if process_ppl:
ppl_indices_split = split_indices(ppl_dataset, num_ppl_children)
split_work(
all_child_pids,
ppl_dataset,
ppl_indices_split,
num_ppl_children)
# Wait to child processes to finish
print('num children:', len(all_child_pids))
for i, child_pid in enumerate(all_child_pids):
os.waitpid(child_pid, 0)
print('All Done (Parallel)')
def main(checkpoint_name):
"""This demo shows RPN proposals and AVOD predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
# checkpoint_name =='pyramid_cars_with_aug_example'
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
##############################
# Options
##############################
dataset_config.data_split = 'val'
fig_size = (10, 6.1)
rpn_score_threshold = 0.1
avod_score_threshold = 0.002 #0.1 #0.009 # <== final threshold
sample_names = [90] + list(range(
98, 104)) + [138, 224, 270, 290, 310, 330, 520]
# Convert to string with correct format (don't change)
sample_names = ['{:06d}'.format(x) for x in sample_names]
# gt_classes = ['Car']
gt_classes = ['Pedestrian', 'Cyclist']
# gt_classes = ['Car', 'Pedestrian', 'Cyclist']
img_dir = '/home/jhuang/repo/avod/input'
# Overwrite this to select a specific checkpoint
global_step = None
# Drawing Toggles
draw_proposals_separate = False
draw_overlaid = False
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = False
draw_orientations_on_pred = False
# Draw 2D bounding boxes
draw_projected_2d_boxes = True
# Save images for samples with no detections
save_empty_images = True
draw_score = True
draw_iou = True
##############################
# End of Options
##############################
# Get the dataset
dataset = None
data_split = 'val'
# Setup Paths
predictions_dir = avod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir = predictions_dir + \
'/proposals_and_scores/' + data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
data_split, global_step, rpn_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
data_split, global_step, avod_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
data_split, global_step, avod_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
num_samples = len(sample_names)
for sample_idx in range(num_samples):
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s\n'.format(
sample_idx + 1, num_samples, avg_time,
est_time_left))
sys.stdout.flush()
sample_name = sample_names[sample_idx]
img_idx = int(sample_name)
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
print("scores=", prediction_scores)
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
avod_score_mask = prediction_scores >= avod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
print("len(prediction_boxes_3d)=", len(prediction_boxes_3d))
prediction_scores = prediction_scores[avod_score_mask]
prediction_class_indices = \
prediction_class_indices[avod_score_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
gt_objects = []
image_path = get_rgb_image_path(img_dir, img_idx, 'img_')
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
calib_dir = img_dir
stereo_calib = load_calib(calib_dir, img_idx, 'calib.txt')
calib_p2 = stereo_calib.p2
if draw_overlaid or draw_predictions_separate:
num_of_predictions = 0
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
calib_p2,
truncate=True,
image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
# else:
# if save_empty_images:
# pred_fig, pred_2d_axes, pred_3d_axes = \
# vis_utils.visualization(dataset.rgb_image_dir,
# img_idx,
# display=False,
# fig_size=fig_size)
# filename = pred_out_dir + '/' + sample_name + '.png'
# plt.savefig(filename)
# plt.close(pred_fig)
# continue
if draw_predictions_separate and num_of_predictions > 0:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization_jhuang(img_dir,
img_idx, 'img_',
display=False,
fig_size=fig_size)
draw_predictions([], calib_p2, num_of_predictions,
final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_2d_axes, pred_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
else:
pred_fig, pred_3d_axes = \
vis_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions([], calib_p2, num_of_predictions,
final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_3d_axes, draw_score, draw_iou,
gt_classes, draw_orientations_on_pred)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
print('\nDone')
def generate_fake_dataset():
return DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_UNITTEST)
def main():
"""This demo shows RPN proposals and AVOD predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
##############################
# Options
##############################
dataset_config.data_split = 'val'
fig_size = (10, 6.1)
rpn_score_threshold = 0.1
avod_score_threshold = 0.1
# gt_classes = ['Car']
gt_classes = ['Pedestrian', 'Cyclist']
# gt_classes = ['Car', 'Pedestrian', 'Cyclist']
# Overwrite this to select a specific checkpoint
global_step = None
checkpoint_name = sys.argv[1] #'pyramid_cars_with_aug_example'
# Drawing Toggles
draw_proposals_separate = False
draw_overlaid = False
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = False
draw_orientations_on_pred = False
# Draw 2D bounding boxes
draw_projected_2d_boxes = True
# Save images for samples with no detections
save_empty_images = True
draw_score = True
draw_iou = True
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
# Setup Paths
predictions_dir = avod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir = predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, rpn_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, avod_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step, avod_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
for sample_idx in range(dataset.num_samples):
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s'.format(
sample_idx + 1, dataset.num_samples, avg_time,
est_time_left))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
if not os.path.exists(proposals_file_path):
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
score_mask = proposal_scores > rpn_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
avod_score_mask = prediction_scores >= avod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
prediction_scores = prediction_scores[avod_score_mask]
prediction_class_indices = \
prediction_class_indices[avod_score_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
if dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
boxes2d, _, _ = obj_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx)
calib_p2 = stereo_calib.p2
##############################
# Reformat and prepare to draw
##############################
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_encoder.box_3d_to_anchor(
proposal_boxes_3d)
proposal_boxes, _ = anchor_projector.project_to_image_space(
proposals_as_anchors, calib_p2, image_size)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_2d_axes, prop_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False)
draw_proposals(filtered_gt_objs, calib_p2, num_of_proposals,
proposal_objs, proposal_boxes, prop_2d_axes,
prop_3d_axes, draw_orientations_on_prop)
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
calib_p2,
truncate=True,
image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
prop_2d_axes, prop_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
filename = overlaid_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_2d_axes, pred_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
else:
pred_fig, pred_3d_axes = \
vis_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs, calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_3d_axes, draw_score, draw_iou,
gt_classes, draw_orientations_on_pred)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
print('\nDone')
def evaluate(model_config, eval_config, dataset_config):
# Parse eval config
eval_mode = eval_config.eval_mode
if eval_mode not in ['val', 'test']:
raise ValueError('Evaluation mode can only be set to `val` or `test`')
evaluate_repeatedly = eval_config.evaluate_repeatedly
# Parse dataset config
data_split = dataset_config.data_split
if data_split == 'train':
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = True
elif data_split.startswith('val'):
dataset_config.data_split_dir = 'training'
# Don't load labels for val split when running in test mode
if eval_mode == 'val':
dataset_config.has_labels = True
elif eval_mode == 'test':
dataset_config.has_labels = False
elif data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
else:
raise ValueError('Invalid data split', data_split)
# Convert to object to overwrite repeated fields
dataset_config = config_builder.proto_to_obj(dataset_config)
# Remove augmentation during evaluation
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Convert to object to overwrite repeated fields
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
if evaluate_repeatedly:
model_evaluator.repeated_checkpoint_run()
else:
model_evaluator.run_latest_checkpoints()
def main():
"""Show histograms of ground truth labels
"""
dataset = DatasetBuilder.build_kitti_dataset(
# DatasetBuilder.KITTI_TRAIN
# DatasetBuilder.KITTI_VAL
DatasetBuilder.KITTI_TRAINVAL)
difficulty = 2
centroid_bins = 51
dimension_bins = 21
orientation_bins = 65
classes = ['Car']
# classes = ['Pedestrian']
# classes = ['Cyclist']
# classes = ['Pedestrian', 'Cyclist']
# Dataset values
num_samples = dataset.num_samples
all_centroids_x = []
all_centroids_y = []
all_centroids_z = []
all_lengths = []
all_widths = []
all_heights = []
all_orientations = []
# Counter for total number of valid samples
num_valid_samples = 0
for sample_idx in range(num_samples):
sys.stdout.write('\r{} / {}'.format(sample_idx + 1, num_samples))
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
obj_labels = dataset.kitti_utils.filter_labels(obj_labels,
classes=classes,
difficulty=difficulty)
centroids = np.asarray([obj.t for obj in obj_labels])
lengths = np.asarray([obj.l for obj in obj_labels])
widths = np.asarray([obj.w for obj in obj_labels])
heights = np.asarray([obj.h for obj in obj_labels])
orientations = np.asarray([obj.ry for obj in obj_labels])
if any(orientations) and np.amax(np.abs(orientations) > np.pi):
raise ValueError('Invalid orientation')
if len(centroids) > 0:
all_centroids_x.extend(centroids[:, 0])
all_centroids_y.extend(centroids[:, 1])
all_centroids_z.extend(centroids[:, 2])
all_lengths.extend(lengths)
all_widths.extend(widths)
all_heights.extend(heights)
all_orientations.extend(orientations)
num_valid_samples += 1
print('Finished reading labels, num_valid_samples', num_valid_samples)
# Get means
mean_centroid_x = np.mean(all_centroids_x)
mean_centroid_y = np.mean(all_centroids_y)
mean_centroid_z = np.mean(all_centroids_z)
mean_dims = np.mean([all_lengths, all_widths, all_heights])
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)})
print('mean_centroid_x {0:0.3f}'.format(mean_centroid_x))
print('mean_centroid_y {0:0.3f}'.format(mean_centroid_y))
print('mean_centroid_z {0:0.3f}'.format(mean_centroid_z))
print('mean_dims {0:0.3f}'.format(mean_dims))
# Make plots
f, ax_arr = plt.subplots(3, 3)
# xyz
ax_arr[0, 0].hist(all_centroids_x, centroid_bins, facecolor='green')
ax_arr[0, 1].hist(all_centroids_y, centroid_bins, facecolor='green')
ax_arr[0, 2].hist(all_centroids_z, centroid_bins, facecolor='green')
# lwh
ax_arr[1, 0].hist(all_lengths, dimension_bins, facecolor='green')
ax_arr[1, 1].hist(all_widths, dimension_bins, facecolor='green')
ax_arr[1, 2].hist(all_heights, dimension_bins, facecolor='green')
# orientations
ax_arr[2, 0].hist(all_orientations, orientation_bins, facecolor='green')
plt.show(block=True)
def main():
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
##############################
# Options
##############################
checkpoint_name = 'fpn_people_dual_NHSP_train'#'fpn_people_dual_SHPL_train'#'pyramid_people_with_NHSP_example_train'#'pyramid_people_with_NHSP_example'
data_split = 'val'#'test'#'val'
global_steps = None
# global_steps = [28000, 19000, 33000, 34000]
score_threshold = 0.1
save_2d = False # Save 2D predictions
save_3d = True # Save 2D and 3D predictions together
save_alphas = True # Save alphas (observation angles)
# Checkpoints below this are skipped
min_step = 20000
##############################
# End of Options
##############################
# Parse experiment config
pipeline_config_file = \
avod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
# Overwrite defaults
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.aug_list = []
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Get available prediction folders
predictions_root_dir = avod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
final_predictions_root_dir = predictions_root_dir + \
'/final_predictions_and_scores/' + dataset.data_split
print('Converting detections from', final_predictions_root_dir)
if not global_steps:
global_steps = os.listdir(final_predictions_root_dir)
global_steps.sort(key=int)
print('Checkpoints found ', global_steps)
for step_idx in range(len(global_steps)):
global_step = global_steps[step_idx]
# Skip first checkpoint
if int(global_step) < min_step:
continue
final_predictions_dir = final_predictions_root_dir + \
'/' + str(global_step)
# 2D and 3D prediction directories
kitti_predictions_2d_dir = predictions_root_dir + \
'/kitti_predictions_2d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
kitti_predictions_3d_dir = predictions_root_dir + \
'/kitti_predictions_3d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
if save_2d and not os.path.exists(kitti_predictions_2d_dir):
os.makedirs(kitti_predictions_2d_dir)
if save_3d and not os.path.exists(kitti_predictions_3d_dir):
os.makedirs(kitti_predictions_3d_dir)
# Do conversion
num_samples = dataset.num_samples
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting detections from:', final_predictions_dir)
if save_2d:
print('2D Detections saved to:', kitti_predictions_2d_dir)
if save_3d:
print('3D Detections saved to:', kitti_predictions_3d_dir)
for sample_idx in range(num_samples):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
prediction_file = sample_name + '.txt'
kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \
'/' + prediction_file
kitti_predictions_3d_file_path = kitti_predictions_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
# # Swap l, w for predictions where w > l
# swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
# fixed_predictions = np.copy(all_predictions)
# fixed_predictions[swapped_indices, 3] = all_predictions[
# swapped_indices, 4]
# fixed_predictions[swapped_indices, 4] = all_predictions[
# swapped_indices, 3]
score_filter = all_predictions[:, 7] >= score_threshold
all_predictions = all_predictions[score_filter]
# If no predictions, skip to next file
if len(all_predictions) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(dataset.get_rgb_image_path(sample_name))
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, stereo_calib_p2,
truncate=True, image_size=image.size)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
else:
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Get object types
all_pred_classes = all_predictions[:, 8].astype(np.int32)
obj_types = [dataset.classes[class_idx]
for class_idx in all_pred_classes]
# Truncation and Occlusion are always empty (see below)
# Alpha
if not save_alphas:
kitti_predictions[:, 3] = -10 * \
np.ones((len(kitti_predictions)), dtype=np.int32)
else:
alphas = all_predictions[:, 6] - \
np.arctan2(all_predictions[:, 0], all_predictions[:, 2])
kitti_predictions[:, 3] = alphas
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones((len(kitti_predictions), 2),
dtype=np.int32)
# Empty 3D (x, y, z)
kitti_empty_2 = -1 * np.ones((len(kitti_predictions), 3),
dtype=np.int32)
# Empty 3D (h, w, l)
kitti_empty_3 = -1000 * np.ones((len(kitti_predictions), 3),
dtype=np.int32)
# Empty 3D (ry)
kitti_empty_4 = -10 * np.ones((len(kitti_predictions), 1),
dtype=np.int32)
# Stack 2D predictions text
kitti_text_2d = np.column_stack([obj_types,
kitti_empty_1,
kitti_predictions[:, 3:8],
kitti_empty_2,
kitti_empty_3,
kitti_empty_4,
kitti_predictions[:, 15]])
# Stack 3D predictions text
kitti_text_3d = np.column_stack([obj_types,
kitti_empty_1,
kitti_predictions[:, 3:16]])
# Save to text files
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, kitti_text_2d,
newline='\r\n', fmt='%s')
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d,
newline='\r\n', fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
def estimate_ground_planes(base_dir,
dataset_config,
plane_method=0,
specific_idx=-1):
velo_dir = base_dir + 'velodyne'
plane_dir = base_dir + 'planes'
if plane_method == 1:
plane_dir = plane_dir + '_ransac'
ground_points_dir = base_dir + 'ground_points'
grid_points_dir = base_dir + 'ground_points_grid'
calib_dir = base_dir + 'calib'
files = os.listdir(velo_dir)
num_files = len(files)
file_idx = 0
#For checking ground planes
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
kitti_utils = dataset.kitti_utils
if not os.path.exists(plane_dir):
os.makedirs(plane_dir)
#Estimate each idx
for file in files:
filepath = velo_dir + '/' + file
idx = int(os.path.splitext(file)[0])
if idx < cfg.MIN_IDX or idx > cfg.MAX_IDX:
continue
if specific_idx != -1:
idx = specific_idx
planes_file = plane_dir + '/%06d.txt' % idx
logging.debug("Index: %d", idx)
lidar_point_cloud = obj_utils.get_lidar_point_cloud(
idx, calib_dir, velo_dir)
# Reshape points into N x [x, y, z]
point_cloud = np.array(lidar_point_cloud).transpose().reshape(
(-1, 3)).T
ground_points_failed = False
if plane_method == use_ground_points:
s = loadGroundPointsFromFile(idx, ground_points_dir,
grid_points_dir)
if s.shape[0] < 4:
logging.debug("Not enough points at idx: %d", idx)
ground_points_failed = True
else:
m = estimate(s)
a, b, c, d = m
plane = loadKittiPlane(m)
#ground_points_failed = checkBadSlices(point_cloud, plane, kitti_utils)
ransac_failed = False
if plane_method == ransac or ground_points_failed:
logging.debug("PC shape: {}".format(point_cloud.shape))
points = point_cloud.T
all_points_near = points[(points[:, 0] > -3.0)
& (points[:, 0] < 3.0) &
(points[:, 1] > -3.0) &
(points[:, 1] < 0.0) &
(points[:, 2] < 20.0) &
(points[:, 2] > 2.0)]
n = all_points_near.shape[0]
logging.debug("Number of points near: %d", n)
if n < 3:
ransac_failed = True
else:
max_iterations = 100
goal_inliers = n * 0.5
m, b = run_ransac(all_points_near,
lambda x, y: is_inlier(x, y, 0.2), 3,
goal_inliers, max_iterations)
a, b, c, d = m
if plane_method == moose or ransac_failed:
plane_coeffs = estimate_plane_coeffs(point_cloud.T)
with open(planes_file, 'w+') as f:
f.write('# Plane\nWidth 4\nHeight 1\n')
if (plane_method == ransac and not ransac_failed) or \
(plane_method == use_ground_points and not ground_points_failed):
coeff_string = '%.6e %.6e %.6e %.6e' % (a, b, c, d)
else:
coeff_string = '%.6e %.6e %.6e %.6e' % (
plane_coeffs[0], plane_coeffs[1], plane_coeffs[2],
plane_coeffs[3])
f.write(coeff_string)
sys.stdout.write("\rGenerating plane {} / {}".format(
file_idx + 1, num_files))
sys.stdout.flush()
file_idx = file_idx + 1
if testing and idx == 2 or specific_idx != -1:
quit()
def setUpClass(cls):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
cls.label_dir = cls.dataset.label_dir
def main():
"""
Displays the bird's eye view maps for a KITTI sample.
"""
##############################
# Options
##############################
bev_generator = 'slices'
slices_config = \
"""
slices {
height_lo: -0.2
height_hi: 2.3
num_slices: 5
}
"""
# Use None for a random image
img_idx = None
# img_idx = 142
# img_idx = 191
show_ground_truth = True # Whether to overlay ground_truth boxes
point_cloud_source = 'lidar'
##############################
# End of Options
##############################
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
dataset_config = DatasetBuilder.merge_defaults(dataset_config)
# Overwrite bev_generator
if bev_generator == 'slices':
text_format.Merge(slices_config,
dataset_config.kitti_utils_config.bev_generator)
else:
raise ValueError('Invalid bev_generator')
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
if img_idx is None:
img_idx = int(random.random() * dataset.num_samples)
sample_name = "{:06}".format(img_idx)
print('=== Showing BEV maps for image: {}.png ==='.format(sample_name))
# Load image
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = image.shape[0:2]
kitti_utils = dataset.kitti_utils
print(image_shape)
point_cloud = kitti_utils.get_point_cloud(point_cloud_source, img_idx,
image_shape)
ground_plane = kitti_utils.get_ground_plane(sample_name)
bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane)
height_maps = np.array(bev_images.get("height_maps"))
density_map = np.array(bev_images.get("density_map"))
box_points, box_points_norm = [None, None]
if show_ground_truth:
# Get projected boxes
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
print(obj_labels)
filtered_objs = obj_labels
label_boxes = []
for label in filtered_objs:
box = box_3d_encoder.object_label_to_box_3d(label)
label_boxes.append(box)
label_boxes = np.array(label_boxes)
print(label_boxes)
box_points, box_points_norm = box_3d_projector.project_to_bev(
label_boxes, [[-40, 40], [0, 70]])
rgb_img_size = (np.array((1242, 375)) * 0.75).astype(np.int16)
img_x_start = 60
img_y_start = 330
img_x = img_x_start
img_y = img_y_start
img_w = 400
img_h = 350
img_titlebar_h = 20
# Show images
vis_utils.cv2_show_image("Image",
image,
size_wh=rgb_img_size,
location_xy=(img_x, 0))
# Height maps
for map_idx in range(len(height_maps)):
height_map = height_maps[map_idx]
height_map = draw_boxes(height_map, box_points_norm)
vis_utils.cv2_show_image("Height Map {}".format(map_idx),
height_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
img_x += img_w
# Wrap around
if (img_x + img_w) > 1920:
img_x = img_x_start
img_y += img_h + img_titlebar_h
# Density map
density_map = draw_boxes(density_map, box_points_norm)
vis_utils.cv2_show_image("Density Map",
density_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
cv2.waitKey()