def main():
test_pipeline_config_path = mlod.root_dir() + \
'/data/configs/official/cars/cars_000_vanilla.config'
model_config, train_config, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# train_val_test = 'val'
# dataset_config.data_split = 'val'
train_val_test = 'test'
dataset_config.data_split = 'trainval'
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = False
# dataset_config.cache_config.cache_images = True
# dataset_config.cache_config.cache_depth_maps = True
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
kitti_utils = dataset.kitti_utils
bev_source = 'lidar'
# sample_name = '000000'
# img_idx = np.random.randint(0, 1000)
# sample_name = '{:06d}'.format(img_idx)
num_samples = 200
all_load_times = []
all_bev_times = []
for sample_idx in range(num_samples):
sys.stdout.write('\rSample {} / {}'.format(sample_idx,
num_samples - 1))
img_idx = sample_idx
sample_name = '{:06d}'.format(sample_idx)
loading_start_time = time.time()
# Load image
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = image.shape[0:2]
calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx)
point_cloud = kitti_utils.get_point_cloud(bev_source, int(sample_name),
image_shape)
ground_plane = kitti_utils.get_ground_plane(sample_name)
all_load_times.append(time.time() - loading_start_time)
bev_start_time = time.time()
bev_maps = kitti_utils.create_bev_maps(point_cloud, ground_plane)
bev_end_time = time.time()
all_bev_times.append(bev_end_time - bev_start_time)
print('')
print('Load mean:', np.mean(all_load_times))
print('Load median:', np.median(all_load_times))
print('BEV mean:', np.mean(all_bev_times))
print('BEV median:', np.median(all_bev_times))
def setUp(self):
tf.test.TestCase.setUp(self)
img_feature_extractor = img_vgg.ImgVgg16()
# dummy imageplaceholder
img_input_placeholder = tf.placeholder(tf.float32, [480, 1590, 3])
img_input_batches = tf.expand_dims(img_input_placeholder, axis=0)
img_preprocessed = img_feature_extractor.preprocess_input(
img_input_batches, [224, 224])
_, img_end_points = img_feature_extractor.build(img_preprocessed)
# download the vgg weights
url = "http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz"
self.vgg_checkpoint_dir = mlod.root_dir() + '/checkpoints/vgg_16'
if not tf.gfile.Exists(self.vgg_checkpoint_dir):
tf.gfile.MakeDirs(self.vgg_checkpoint_dir)
# Download vgg16 if it hasn't been downloaded yet
if not os.path.isfile(self.vgg_checkpoint_dir +
"/vgg_16_2016_08_28.tar.gz"):
dataset_utils.download_and_uncompress_tarball(
url, self.vgg_checkpoint_dir)
else:
print("Already downloaded")
def run_kitti_native_script_with_05_iou(checkpoint_name,
score_threshold,
global_step,
branch_idx=None):
"""Runs the kitti native code script."""
eval_script_dir = mlod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
make_script = eval_script_dir + \
'/kitti_native_eval/run_eval_05_iou.sh'
script_folder = eval_script_dir + \
'/kitti_native_eval/'
results_dir = mlod.top_dir() + '/scripts/offline_eval/results_05_iou/'
# Round this because protobuf encodes default values as full decimal
score_threshold = round(score_threshold, 3)
if branch_idx is None:
branch_idx_str = 'main'
else:
branch_idx_str = 'br_' + str(branch_idx)
subprocess.call([
make_script, script_folder,
str(score_threshold),
str(global_step),
str(checkpoint_name),
str(results_dir), branch_idx_str
])
def set_up_model():
test_pipeline_config_path = mlod.root_dir() + \
'/configs/mlod_exp_example.config'
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
dataset_config = config_builder.proto_to_obj(dataset_config)
train_val_test = 'test'
dataset_config.data_split = 'test'
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset_config.aug_list = []
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model
def test_get_clusters(self):
# classes = ['Car', 'Pedestrian', 'Cyclist']
num_clusters = [2, 1, 1]
label_cluster_utils = LabelClusterUtils(self.dataset)
clusters, std_devs = label_cluster_utils.get_clusters()
# Check that correct number of clusters are returned
clusters_per_class = [len(cls_clusters) for cls_clusters in clusters]
std_devs_per_class = [len(cls_std_devs) for cls_std_devs in std_devs]
self.assertEqual(clusters_per_class, num_clusters)
self.assertEqual(std_devs_per_class, num_clusters)
# Check that text files were saved
txt_folder_exists = os.path.isdir(
mlod.root_dir() + "/data/label_clusters/unittest-kitti")
self.assertTrue(txt_folder_exists)
# Calling get_clusters again should read from files
read_clusters, read_std_devs = label_cluster_utils.get_clusters()
# Check that read values are the same as generated ones
np.testing.assert_allclose(np.vstack(clusters),
np.vstack(read_clusters))
np.testing.assert_allclose(np.vstack(std_devs),
np.vstack(read_std_devs))
def copy_kitti_native_code(checkpoint_name):
"""Copies and compiles kitti native code.
It also creates neccessary directories for storing the results
of the kitti native evaluation code.
"""
mlod_root_dir = mlod.root_dir()
kitti_native_code_copy = mlod_root_dir + '/data/outputs/' + \
checkpoint_name + '/predictions/kitti_native_eval/'
# Only copy if the code has not been already copied over
if not os.path.exists(kitti_native_code_copy):
os.makedirs(kitti_native_code_copy)
original_kitti_native_code = mlod.top_dir() + \
'/scripts/offline_eval/kitti_native_eval/'
predictions_dir = mlod_root_dir + '/data/outputs/' + \
checkpoint_name + '/predictions/'
# create dir for it first
dir_util.copy_tree(original_kitti_native_code, kitti_native_code_copy)
# run the script to compile the c++ code
script_folder = predictions_dir + \
'/kitti_native_eval/'
make_script = script_folder + 'run_make.sh'
subprocess.call([make_script, script_folder])
# Set up the results folders if they don't exist
results_dir = mlod.top_dir() + '/scripts/offline_eval/results'
results_05_dir = mlod.top_dir() + '/scripts/offline_eval/results_05_iou'
if not os.path.exists(results_dir):
os.makedirs(results_dir)
if not os.path.exists(results_05_dir):
os.makedirs(results_05_dir)
def __init__(self, dataset):
self._dataset = dataset
self.cluster_split = dataset.cluster_split
self.data_dir = mlod.root_dir() + "/data/label_clusters"
self.clusters = []
self.std_devs = []
def main(_):
parser = argparse.ArgumentParser()
# Example usage
# --checkpoint_name='mlod_exp_example'
# --data_split='test'
# --ckpt_indices=50 100 112
# Optional arg:
# --device=0
parser.add_argument('--checkpoint_name',
type=str,
dest='checkpoint_name',
required=True,
help='Checkpoint name must be specified as a str\
and must match the experiment config file name.')
parser.add_argument('--data_split',
type=str,
dest='data_split',
required=True,
help='Data split must be specified e.g. val or test')
parser.add_argument('--ckpt_indices',
type=int,
nargs='+',
dest='ckpt_indices',
required=True,
help='Checkpoint indices must be a set of \
integers with space in between -> 0 10 20 etc')
parser.add_argument('--device',
type=str,
dest='device',
default='0',
help='CUDA device id')
args = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
experiment_config = args.checkpoint_name + '.config'
# Read the config from the experiment folder
experiment_config_path = mlod.root_dir() + '/data/outputs/' +\
args.checkpoint_name + '/' + experiment_config
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
inference(model_config, eval_config, dataset_config, args.data_split,
args.ckpt_indices)
def main():
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL)
# get proposals
proposal_output_dir = mlod.root_dir() + \
"/data/predictions/rpn_model/proposals_and_scores/" + \
dataset.data_split
global_steps = os.listdir(proposal_output_dir)
print('Checkpoints found ', global_steps)
all_recalls = []
for step in global_steps:
for sample_name in dataset.sample_list:
img_idx = int(sample_name)
# ------------------------------------
# Load proposals and scores from files
# ------------------------------------
proposals_scores_dir = proposal_output_dir + \
"{}/{}/{}.txt".format(dataset.data_split,
step,
sample_name)
if not os.path.exists(proposals_scores_dir):
print('File {} not found, skipping'.format(sample_name))
continue
proposals_scores = np.loadtxt(proposals_scores_dir)
proposals = proposals_scores[:, 0:-1]
proposal_iou_format = \
box_3d_encoder.box_3d_to_3d_iou_format(proposals)
# scores are in the last column
scores = proposals_scores[:, -1]
# -----------------------
# Get ground truth labels
# -----------------------
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
_, gt_3d_bbs, _ = obj_utils.build_bbs_from_objects(
gt_objects, ['Car', 'car'])
score_thresholds = np.array([0.3])
iou_threshold = 0.0025
# calculate RPN recall and precision
precision, recall = evaluation.evaluate_3d([gt_3d_bbs],
[proposal_iou_format],
[scores],
score_thresholds,
iou_threshold)
print('Recall ', recall[0])
print('Precision ', precision[0])
all_recalls.append(recall)
def inference(sess, image):
# Download VGG16
url = "http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz"
# checkpoints_dir = '/tmp/mlod/checkpoints'
checkpoints_dir = mlod.root_dir() + '/checkpoints/vgg_16'
if not tf.gfile.Exists(checkpoints_dir):
tf.gfile.MakeDirs(checkpoints_dir)
# Download vgg16 if it hasn't been downloaded yet
if not os.path.isfile(checkpoints_dir + "/vgg_16_2016_08_28.tar.gz"):
dataset_utils.download_and_uncompress_tarball(url, checkpoints_dir)
else:
print("Already downloaded")
# Set the image size to [224, 224]
image_size = vgg.vgg_16.default_image_size
# Pre-process input
with tf.name_scope('input_reshape'):
processed_image = vgg_preprocessing.preprocess_image(
image, image_size, image_size, is_training=False)
image_summary = tf.expand_dims(image, 0)
processed_images = tf.expand_dims(processed_image, 0)
tf.summary.image('images', image_summary, max_outputs=5)
tf.summary.image('processed_images', processed_images, max_outputs=5)
# Create the model, use the default arg scope to configure the
# batch norm parameters.
with slim.arg_scope(vgg.vgg_arg_scope()):
# 1000 classes instead of 1001.
logits, end_points = vgg.vgg_16(
processed_images, num_classes=1000, is_training=False)
probabilities = tf.nn.softmax(logits)
# Add some images for feature maps from conv layers
vis_feature_maps(end_points, 'vgg_16/conv1/conv1_1')
vis_feature_maps(end_points, 'vgg_16/conv2/conv2_1')
vis_feature_maps(end_points, 'vgg_16/conv3/conv3_1')
vis_feature_maps(end_points, 'vgg_16/conv4/conv4_1')
vis_feature_maps(end_points, 'vgg_16/conv5/conv5_1')
vis_feature_maps(end_points, 'vgg_16/conv5/conv5_3')
# Initialize vgg16 weights
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, 'vgg_16.ckpt'),
slim.get_model_variables('vgg_16'))
init_fn(sess)
return logits, probabilities
def setUpClass(cls):
pipeline_config = pipeline_pb2.NetworkPipelineConfig()
dataset_config = pipeline_config.dataset_config
config_path = mlod.root_dir() + '/configs/unittest_model.config'
cls.model_config = config_build.get_model_config_from_file(config_path)
input_config = cls.model_config.input_config
input_config.bev_depth = 5 # unittest-kitti has [2, 1, 1] clusters
dataset_config.MergeFrom(DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
def main():
# Profile large csv
csv_file_path = mlod.root_dir() + \
'/data/mini_batches/iou_2d/kitti/train/depth/People[ 0.3 0.3]/007463.csv'
###################
# Numpy loadtxt
###################
np_data = np.loadtxt(csv_file_path, delimiter=',')
###################
# Numpy genfromtxt
###################
np_data = np.genfromtxt(csv_file_path, delimiter=',')
###################
# Pandas read_csv
###################
start_time = time.time()
pandas_data = pandas.read_csv(csv_file_path).values
print('pandas.read_csv took :', time.time() - start_time)
start_time = time.time()
# Profile small csv
calib_dir = mlod.root_dir() + \
'/tests/datasets/Kitti/object/training/calib/000001.txt'
start_time = time.time()
data_file = open(calib_dir, 'r')
csv_reader = csv.reader(data_file, delimiter=' ')
data = []
for row in csv_reader:
data.append(row)
print('csv.reader took :', time.time() - start_time)
pandas_data = pandas.read_csv(csv_file_path).values
def main(_):
parser = argparse.ArgumentParser()
default_rpn_config_path = mlod.root_dir() + \
'/configs/stagewise_rpn_example.config'
parser.add_argument('--pipeline_rpn_config',
type=str,
dest='rpn_config_path',
default=default_rpn_config_path,
help='Path to the rpn config')
default_mlod_config_path = mlod.root_dir() + \
'/configs/stagewise_mlod_example.config'
parser.add_argument('--mlod_config_path',
type=str,
dest='mlod_config_path',
default=default_mlod_config_path,
help='Path to the mlod config')
args = parser.parse_args()
rpn_model_config, rpn_train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.rpn_config_path,
is_training=True,
stagewise_training=True)
mlod_model_config, mlod_train_config, _, _ = \
config_builder.get_configs_from_pipeline_file(
args.mlod_config_path,
is_training=True,
stagewise_training=True)
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
train(rpn_model_config, mlod_model_config, rpn_train_config,
mlod_train_config, dataset_config)
def fill_feed_dict(input_pl):
image_name = 'cat.jpg'
# image_name = 'school_bus.jpg'
# image_name = 'table.jpg'
cv2_bgr_image = cv2.imread(mlod.root_dir() + '/tests/images/' + image_name)
# BGR -> RGB
rgb_image = cv2_bgr_image[..., ::-1]
feed_dict = {
# input_pl: bev_image,
input_pl: rgb_image,
}
return feed_dict
def setUp(self):
tf.test.TestCase.setUp(self)
test_pipeline_config_path = mlod.root_dir() + \
'/configs/unittest_stagewise.config'
self.model_config, self.train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# unittest-kitti has [2, 1, 1] clusters
input_config = self.model_config.input_config
input_config.bev_depth = 5
# Generate dataset
self.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST,
use_defaults=False,
new_cfg=dataset_config)
def main(_):
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
parser = argparse.ArgumentParser()
default_train_config_path = mlod.root_dir() + \
'/configs/mlod_exp_example.config'
parser.add_argument('--pipeline_config',
type=str,
dest='pipeline_config_path',
default=default_train_config_path,
help='Path to the train config')
args = parser.parse_args()
model_config, train_config, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.pipeline_config_path,
is_training=True)
train_and_eval(model_config, train_config, eval_config, dataset_config)
def main(_):
parser = argparse.ArgumentParser()
# Defaults
default_pipeline_config_path = mlod.root_dir() + \
'/configs/mlod_exp_example.config'
default_data_split = 'train'
default_device = '0'
parser.add_argument('--pipeline_config',
type=str,
dest='pipeline_config_path',
default=default_pipeline_config_path,
help='Path to the pipeline config')
parser.add_argument('--data_split',
type=str,
dest='data_split',
default=default_data_split,
help='Data split for training')
parser.add_argument('--device',
type=str,
dest='device',
default=default_device,
help='CUDA device id')
args = parser.parse_args()
# Parse pipeline config
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.pipeline_config_path, is_training=True)
# Overwrite data split
dataset_config.data_split = args.data_split
# Set CUDA device id
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
train(model_config, train_config, dataset_config)
def run_kitti_native_script(checkpoint_name, score_threshold, global_step,
sub_score_threshold):
"""Runs the kitti native code script."""
eval_script_dir = mlod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
make_script = eval_script_dir + \
'/kitti_native_eval/run_eval.sh'
script_folder = eval_script_dir + \
'/kitti_native_eval/'
results_dir = mlod.top_dir() + '/scripts/offline_eval/results/'
# Round this because protobuf encodes default values as full decimal
score_threshold = round(score_threshold, 3)
print('evaluating....')
subprocess.call([
make_script, script_folder,
str(score_threshold),
str(global_step),
str(checkpoint_name),
str(results_dir), sub_score_threshold
])
def main():
"""This demo shows RPN proposals and MLOD predictions in the
3D point cloud.
Keys:
F1: Toggle proposals
F2: Toggle predictions
F3: Toggle 3D voxel grid
F4: Toggle point cloud
F5: Toggle easy ground truth objects (Green)
F6: Toggle medium ground truth objects (Orange)
F7: Toggle hard ground truth objects (Red)
F8: Toggle all ground truth objects (default off)
F9: Toggle ground slice filter (default off)
F10: Toggle offset slice filter (default off)
"""
##############################
# Options
##############################
rpn_score_threshold = 0.1
mlod_score_threshold = 0.1
proposals_line_width = 1.0
predictions_line_width = 3.0
show_orientations = True
point_cloud_source = 'lidar'
# Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>)
config_dir = None
checkpoint_name = 'mlod_fpn_people_n_m'
global_step = 135000 # Latest checkpoint
#data_split = 'val_half'
#data_split = 'val'
data_split = 'test'
# Show 3D iou text
draw_ious_3d = False
sample_name = '000031'
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# val split
# sample_name = '000181'
# sample_name = '000751'
# sample_name = '000843'
# sample_name = '000944'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# val split
# sample_name = '000015'
# sample_name = '000048'
# sample_name = '000058'
# sample_name = '000076' # Medium, few ped, 1 cyc
# sample_name = '000108'
# sample_name = '000118'
# sample_name = '000145'
# sample_name = '000153'
# sample_name = '000186'
# sample_name = '000195'
# sample_name = '000199'
# sample_name = '000397'
# sample_name = '004425'
# sample_name = '004474' # Hard, many ped, 1 cyc
# sample_name = '004657' # Hard, Few cycl, few ped
# sample_name = '006071'
# sample_name = '006828' # Hard, Few cycl, few ped
# sample_name = '006908' # Hard, Few cycl, few ped
# sample_name = '007412'
# sample_name = '007318' # Hard, Few cycl, few ped
##############################
# End of Options
##############################
if data_split == 'test':
draw_ious_3d = False
if config_dir is None:
config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name
# Parse experiment config
pipeline_config_file = \
config_dir + '/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
dataset_config.data_split = data_split
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_names[sample_idx]
##############################
# Setup Paths
##############################
img_idx = int(sample_name)
# Text files directory
proposals_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_predictions_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
# Output images directory
img_out_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/predictions/images_3d/{}/{}/{}'.format(dataset.data_split,
global_step,
rpn_score_threshold)
if not os.path.exists(img_out_dir):
os.makedirs(img_out_dir)
##############################
# Proposals
##############################
# Load proposals from files
proposals_and_scores = np.loadtxt(
proposals_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
proposals = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
rpn_score_mask = proposal_scores > rpn_score_threshold
proposals = proposals[rpn_score_mask]
proposal_scores = proposal_scores[rpn_score_mask]
print('Proposals:', len(proposal_scores), proposal_scores)
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposals]
##############################
# Predictions
##############################
# Load proposals from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name)).reshape(-1, 9)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_types = np.asarray(predictions_and_scores[:, 8], dtype=np.int32)
mlod_score_mask = prediction_scores >= mlod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[mlod_score_mask]
prediction_scores = prediction_scores[mlod_score_mask]
print('Predictions: ', len(prediction_scores), prediction_scores)
final_predictions = np.copy(prediction_boxes_3d)
# # Swap l, w for predictions where w > l
# swapped_indices = predictions[:, 4] > predictions[:, 3]
# final_predictions[swapped_indices, 3] = predictions[swapped_indices, 4]
# final_predictions[swapped_indices, 4] = predictions[swapped_indices, 3]
prediction_objs = []
dataset.classes = ['Pedestrian', 'Cyclist', 'Car']
for pred_idx in range(len(final_predictions)):
prediction_box_3d = final_predictions[pred_idx]
prediction_type = dataset.classes[prediction_types[pred_idx]]
prediction_obj = box_3d_encoder.box_3d_to_object_label(
prediction_box_3d, obj_type=prediction_type)
prediction_objs.append(prediction_obj)
##############################
# Ground Truth
##############################
dataset.has_labels = False
if dataset.has_labels:
# Get ground truth labels
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset, img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# 3D IoU
##############################
if draw_ious_3d:
# Convert to box_3d
all_gt_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in all_gt_objs
]
pred_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(pred_obj)
for pred_obj in prediction_objs
]
max_ious_3d = demo_utils.get_max_ious_3d(all_gt_boxes_3d,
pred_boxes_3d)
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
point_cloud = dataset.kitti_utils.get_point_cloud(point_cloud_source,
img_idx,
image_shape=image.shape)
point_cloud = np.asarray(point_cloud)
# Filter point cloud to extents
area_extents = np.asarray([[-40, 40], [-5, 3], [0, 70]])
bev_extents = area_extents[[0, 2]]
points = point_cloud.T
point_filter = obj_utils.get_point_filter(point_cloud, area_extents)
points = points[point_filter]
point_colours = vis_utils.project_img_to_point_cloud(
points, image, dataset.calib_dir, img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False)
# Ground plane
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
vtk_axes = vtk.vtkAxesActor()
vtk_axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for proposal boxes
vtk_proposal_boxes = VtkBoxes()
vtk_proposal_boxes.set_line_width(proposals_line_width)
vtk_proposal_boxes.set_objects(proposal_objs, COLOUR_SCHEME_PREDICTIONS)
# Create VtkBoxes for prediction boxes
vtk_prediction_boxes = VtkPyramidBoxes()
vtk_prediction_boxes.set_line_width(predictions_line_width)
vtk_prediction_boxes.set_objects(prediction_objs,
COLOUR_SCHEME_PREDICTIONS,
show_orientations)
# Create VtkBoxes for ground truth
vtk_hard_gt_boxes = VtkBoxes()
vtk_medium_gt_boxes = VtkBoxes()
vtk_easy_gt_boxes = VtkBoxes()
vtk_all_gt_boxes = VtkBoxes()
vtk_hard_gt_boxes.set_objects(hard_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_medium_gt_boxes.set_objects(medium_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_easy_gt_boxes.set_objects(easy_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_all_gt_boxes.set_objects(all_gt_objs, VtkBoxes.COLOUR_SCHEME_KITTI,
show_orientations)
# Create VtkTextLabels for 3D ious
vtk_text_labels = VtkTextLabels()
if draw_ious_3d and len(all_gt_boxes_3d) > 0:
gt_positions_3d = np.asarray(all_gt_boxes_3d)[:, 0:3]
vtk_text_labels.set_text_labels(
gt_positions_3d,
['{:0.3f}'.format(iou_3d) for iou_3d in max_ious_3d])
# Create VtkGroundPlane
vtk_ground_plane = VtkGroundPlane()
vtk_slice_bot_plane = VtkGroundPlane()
vtk_slice_top_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, bev_extents)
vtk_slice_bot_plane.set_plane(ground_plane + [0, 0, 0, -0.2], bev_extents)
vtk_slice_top_plane.set_plane(ground_plane + [0, 0, 0, -2.0], bev_extents)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_prediction_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_text_labels.vtk_actor)
# Add ground plane and slice planes
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_bot_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_top_plane.vtk_actor)
#vtk_renderer.AddActor(vtk_axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(3)
vtk_proposal_boxes.vtk_actor.SetVisibility(0)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.SetVisibility(0)
vtk_slice_bot_plane.vtk_actor.SetVisibility(0)
vtk_slice_top_plane.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_bot_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_top_plane.vtk_actor.GetProperty().SetOpacity(0.9)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(3.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
mlod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
# Add custom interactor to toggle actor visibilities
custom_interactor = vis_utils.CameraInfoInteractorStyle([
vtk_proposal_boxes.vtk_actor,
vtk_prediction_boxes.vtk_actor,
vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor,
vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor,
vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor,
vtk_ground_plane.vtk_actor,
vtk_slice_bot_plane.vtk_actor,
vtk_slice_top_plane.vtk_actor,
vtk_text_labels.vtk_actor,
])
vtk_render_window_interactor.SetInteractorStyle(custom_interactor)
# Render in VTK
vtk_render_window.Render()
# Take a screenshot
window_to_image_filter = vtk.vtkWindowToImageFilter()
window_to_image_filter.SetInput(vtk_render_window)
window_to_image_filter.Update()
png_writer = vtk.vtkPNGWriter()
file_name = img_out_dir + "/{}.png".format(sample_name)
png_writer.SetFileName(file_name)
png_writer.SetInputData(window_to_image_filter.GetOutput())
png_writer.Write()
print('Screenshot saved to ', file_name)
vtk_render_window_interactor.Start() # Blocking
def main():
"""This demo visualizes box 8C format predicted by MLOD, before
getting converted to Box 3D.
Keys:
F1: Toggle predictions
F2: Toggle easy ground truth objects (Green)
F3: Toggle medium ground truth objects (Orange)
F4: Toggle hard ground truth objects (Red)
F5: Toggle all ground truth objects (default off)
F6: Toggle 3D voxel grid
F7: Toggle point cloud
"""
##############################
# Options
##############################
mlod_score_threshold = 0.1
show_orientations = True
checkpoint_name = 'mlod_exp_8c'
global_step = None
sample_name = None
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL_HALF)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
##############################
# Setup Paths
##############################
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001'
sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_list[sample_idx]
img_idx = int(sample_name)
# Text files directory
predictions_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_boxes_8c_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(predictions_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
##############################
# predictions
##############################
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
predictions_boxes_8c = predictions_and_scores[:, 0:24]
prediction_scores = predictions_and_scores[:, 24]
score_mask = prediction_scores >= mlod_score_threshold
predictions_boxes_8c = predictions_boxes_8c[score_mask]
all_vtk_box_corners = []
predictions_boxes_8c = np.reshape(predictions_boxes_8c, [-1, 3, 8])
for i in range(len(predictions_boxes_8c)):
box_8c = predictions_boxes_8c[i, :, :]
vtk_box_corners = VtkBox8c()
vtk_box_corners.set_objects(box_8c)
all_vtk_box_corners.append(vtk_box_corners)
##############################
# Ground Truth
##############################
if dataset.has_labels:
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset,
img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
img_idx = int(sample_name)
points, point_colours = demo_utils.get_filtered_pc_and_colours(
dataset, image, img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for ground truth
vtk_easy_gt_boxes, vtk_medium_gt_boxes, \
vtk_hard_gt_boxes, vtk_all_gt_boxes = \
demo_utils.create_gt_vtk_boxes(easy_gt_objs,
medium_gt_objs,
hard_gt_objs,
all_gt_objs,
show_orientations)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_box_actors = vtk.vtkAssembly()
# Create VtkBoxes for prediction boxes
for i in range(len(all_vtk_box_corners)):
# Adding labels, slows down rendering
# vtk_renderer.AddActor(all_vtk_box_corners[i].
# vtk_text_labels.vtk_actor)
vtk_box_actors.AddPart(all_vtk_box_corners[i].vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_box_actors)
vtk_renderer.AddActor(axes)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
mlod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_box_actors, vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor, vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor
]))
vtk_render_window_interactor.Start()
def main():
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
##############################
# Options
##############################
checkpoint_name = 'mlod_exp_example'
# data_split = 'val'
data_split = 'val_half'
global_steps = None
# global_steps = [28000, 19000, 33000, 34000]
score_threshold = 0.1
save_2d = False # Save 2D predictions
save_3d = True # Save 2D and 3D predictions together
# Checkpoints below this are skipped
min_step = 20000
# Object Type
obj_type = 'obj'
##############################
# End of Options
##############################
# Parse experiment config
pipeline_config_file = \
mlod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
# Overwrite defaults
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.aug_list = []
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Get available prediction folders
predictions_root_dir = mlod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
proposals_root_dir = predictions_root_dir + \
'/proposals_and_scores/' + dataset.data_split
print('Converting proposals from', proposals_root_dir)
if not global_steps:
global_steps = os.listdir(proposals_root_dir)
global_steps.sort(key=int)
print('Checkpoints found ', global_steps)
for step_idx in range(len(global_steps)):
global_step = global_steps[step_idx]
# Skip first checkpoint
if int(global_step) < min_step:
continue
final_predictions_dir = proposals_root_dir + \
'/' + str(global_step)
# 2D and 3D prediction directories
kitti_predictions_2d_dir = predictions_root_dir + \
'/kitti_proposals_2d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
kitti_proposals_3d_dir = predictions_root_dir + \
'/kitti_proposals_3d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
if save_2d and not os.path.exists(kitti_predictions_2d_dir):
os.makedirs(kitti_predictions_2d_dir)
if save_3d and not os.path.exists(kitti_proposals_3d_dir):
os.makedirs(kitti_proposals_3d_dir)
# Do conversion
num_samples = dataset.num_samples
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting proposals from:', final_predictions_dir)
if save_2d:
print('2D Detections saved to:', kitti_predictions_2d_dir)
if save_3d:
print('Proposals saved to:', kitti_proposals_3d_dir)
for sample_idx in range(num_samples):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
prediction_file = sample_name + '.txt'
kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \
'/' + prediction_file
kitti_predictions_3d_file_path = kitti_proposals_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
# Swap l, w for predictions where w > l
swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
fixed_predictions = np.copy(all_predictions)
fixed_predictions[swapped_indices,
3] = all_predictions[swapped_indices, 4]
fixed_predictions[swapped_indices,
4] = all_predictions[swapped_indices, 3]
fixed_predictions[swapped_indices, 6] = np.pi / 2
score_filter = all_predictions[:, 7] >= score_threshold
all_predictions = fixed_predictions[score_filter]
# If no predictions, skip to next file
if len(all_predictions) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(dataset.get_rgb_image_path(sample_name))
stereo_calib_p2 = calib_utils.read_calibration(
dataset.calib_dir, img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
stereo_calib_p2,
truncate=True,
image_size=image.size,
discard_before_truncation=False)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
else:
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Truncation and Occlusion are always empty (see below)
# Alpha (Not computed)
kitti_predictions[:, 3] = -10 * np.ones(
(len(kitti_predictions)), dtype=np.int32)
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones(
(len(kitti_predictions), 2), dtype=np.int32)
# Empty 3D (x, y, z)
kitti_empty_2 = -1 * np.ones(
(len(kitti_predictions), 3), dtype=np.int32)
# Empty 3D (h, w, l)
kitti_empty_3 = -1000 * np.ones(
(len(kitti_predictions), 3), dtype=np.int32)
# Empty 3D (ry)
kitti_empty_4 = -10 * np.ones(
(len(kitti_predictions), 1), dtype=np.int32)
# Create Type Array
obj_types = [obj_type for i in range(len(kitti_predictions))]
# Stack 2D predictions text
kitti_text_2d = np.column_stack([
obj_types, kitti_empty_1, kitti_predictions[:, 3:8],
kitti_empty_2, kitti_empty_3, kitti_empty_4,
kitti_predictions[:, 15]
])
# Stack 3D predictions text
kitti_text_3d = np.column_stack(
[obj_types, kitti_empty_1, kitti_predictions[:, 3:16]])
# Save to text files
if save_2d:
np.savetxt(kitti_predictions_2d_file_path,
kitti_text_2d,
newline='\r\n',
fmt='%s')
if save_3d:
np.savetxt(kitti_predictions_3d_file_path,
kitti_text_3d,
newline='\r\n',
fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
class DatasetBuilder(object):
"""
Static class to return preconfigured dataset objects
"""
KITTI_UNITTEST = KittiDatasetConfig(
name="unittest-kitti",
dataset_dir=mlod.root_dir() + "/tests/datasets/Kitti/object",
data_split="train",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car", "Pedestrian", "Cyclist"],
num_clusters=[2, 1, 1],
)
KITTI_TRAIN = KittiDatasetConfig(name="kitti",
data_split="train",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2])
KITTI_VAL = KittiDatasetConfig(
name="kitti",
data_split="val",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
KITTI_TEST = KittiDatasetConfig(
name="kitti",
data_split="test",
data_split_dir="testing",
has_labels=False,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
KITTI_TRAINVAL = KittiDatasetConfig(
name="kitti",
data_split="trainval",
data_split_dir="training",
has_labels=True,
cluster_split="trainval",
classes=["Car"],
num_clusters=[2],
)
KITTI_TRAIN_MINI = KittiDatasetConfig(
name="kitti",
data_split="train_mini",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
KITTI_VAL_MINI = KittiDatasetConfig(
name="kitti",
data_split="val_mini",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
KITTI_TEST_MINI = KittiDatasetConfig(
name="kitti",
data_split="test_mini",
data_split_dir="testing",
has_labels=False,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
KITTI_VAL_HALF = KittiDatasetConfig(
name="kitti",
data_split="val_half",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
CONFIG_DEFAULTS_PROTO = \
"""
bev_source: 'stereo'
kitti_utils_config {
area_extents: [-40, 40, -5, 3, 0, 70]
voxel_size: 0.1
anchor_strides: [0.5, 0.5, 0.2, 0.2, 0.3, 0.3]
bev_generator {
height_priors {
ground_filter_offset: 0.2
offset_filter_distance: 2.0
}
# slices {
# height_lo: -0.5
# height_hi: 2.5
# num_slices: 5
# }
}
mini_batch_config {
density_threshold: 1
rpn_config {
iou_2d_thresholds {
neg_iou_lo: 0.001
neg_iou_hi: 0.05
pos_iou_lo: 0.3
pos_iou_hi: 1.0
}
# iou_3d_thresholds {
# neg_iou_lo: 0.001
# neg_iou_hi: 0.005
# pos_iou_lo: 0.1
# pos_iou_hi: 1.0
# }
mini_batch_size: 512
}
mlod_config {
iou_2d_thresholds {
neg_iou_lo: 0.0
neg_iou_hi: 0.5
pos_iou_lo: 0.7
pos_iou_hi: 1.0
}
mini_batch_size: 2000
}
}
}
"""
@staticmethod
def load_dataset_from_config(dataset_config_path):
dataset_config = kitti_dataset_pb2.KittiDatasetConfig()
with open(dataset_config_path, 'r') as f:
text_format.Merge(f.read(), dataset_config)
return DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
@staticmethod
def copy_config(cfg):
return deepcopy(cfg)
@staticmethod
def merge_defaults(cfg):
cfg_copy = DatasetBuilder.copy_config(cfg)
text_format.Merge(DatasetBuilder.CONFIG_DEFAULTS_PROTO, cfg_copy)
return cfg_copy
@staticmethod
def build_kitti_dataset(base_cfg,
use_defaults=True,
new_cfg=None) -> KittiDataset:
"""Builds a KittiDataset object using the provided configurations
Args:
base_cfg: a base dataset configuration
use_defaults: whether to use the default config values
new_cfg: (optional) a custom dataset configuration, no default
values will be used, all config values must be provided
Returns:
KittiDataset object
"""
cfg_copy = DatasetBuilder.copy_config(base_cfg)
if use_defaults:
# Use default values
text_format.Merge(DatasetBuilder.CONFIG_DEFAULTS_PROTO, cfg_copy)
if new_cfg:
# Use new config values if provided
cfg_copy.MergeFrom(new_cfg)
return KittiDataset(cfg_copy)
def main(dataset=None):
"""Generates anchors info which is used for mini batch sampling.
Processing on 'Cars' can be split into multiple processes, see the Options
section for configuration.
Args:
dataset: KittiDataset (optional)
If dataset is provided, only generate info for that dataset.
If no dataset provided, generates info for all 3 classes.
"""
if dataset is not None:
do_preprocessing(dataset, None)
return
car_dataset_config_path = mlod.root_dir() + \
'/configs/mb_preprocessing/rpn_cars.config'
ped_dataset_config_path = mlod.root_dir() + \
'/configs/mb_preprocessing/rpn_pedestrians.config'
cyc_dataset_config_path = mlod.root_dir() + \
'/configs/mb_preprocessing/rpn_cyclists.config'
ppl_dataset_config_path = mlod.root_dir() + \
'/configs/mb_preprocessing/rpn_people.config'
##############################
# Options
##############################
# Serial vs parallel processing
in_parallel = True
process_car = True # Cars
process_ped = False # Pedestrians
process_cyc = False # Cyclists
process_ppl = True # People (Pedestrians + Cyclists)
# Number of child processes to fork, samples will
# be divided evenly amongst the processes (in_parallel must be True)
num_car_children = 8
num_ped_children = 8
num_cyc_children = 8
num_ppl_children = 8
##############################
# Dataset setup
##############################
if process_car:
car_dataset = DatasetBuilder.load_dataset_from_config(
car_dataset_config_path)
if process_ped:
ped_dataset = DatasetBuilder.load_dataset_from_config(
ped_dataset_config_path)
if process_cyc:
cyc_dataset = DatasetBuilder.load_dataset_from_config(
cyc_dataset_config_path)
if process_ppl:
ppl_dataset = DatasetBuilder.load_dataset_from_config(
ppl_dataset_config_path)
##############################
# Serial Processing
##############################
if not in_parallel:
if process_car:
do_preprocessing(car_dataset, None)
if process_ped:
do_preprocessing(ped_dataset, None)
if process_cyc:
do_preprocessing(cyc_dataset, None)
if process_ppl:
do_preprocessing(ppl_dataset, None)
print('All Done (Serial)')
##############################
# Parallel Processing
##############################
else:
# List of all child pids to wait on
all_child_pids = []
# Cars
if process_car:
car_indices_split = split_indices(car_dataset, num_car_children)
split_work(all_child_pids, car_dataset, car_indices_split,
num_car_children)
# Pedestrians
if process_ped:
ped_indices_split = split_indices(ped_dataset, num_ped_children)
split_work(all_child_pids, ped_dataset, ped_indices_split,
num_ped_children)
# Cyclists
if process_cyc:
cyc_indices_split = split_indices(cyc_dataset, num_cyc_children)
split_work(all_child_pids, cyc_dataset, cyc_indices_split,
num_cyc_children)
# People (Pedestrians + Cyclists)
if process_ppl:
ppl_indices_split = split_indices(ppl_dataset, num_ppl_children)
split_work(all_child_pids, ppl_dataset, ppl_indices_split,
num_ppl_children)
# Wait to child processes to finish
print('num children:', len(all_child_pids))
for i, child_pid in enumerate(all_child_pids):
os.waitpid(child_pid, 0)
print('All Done (Parallel)')
def main():
"""
This demo shows example mini batch info for full MlodModel training.
This includes ground truth, ortho rotated ground truth,
negative proposal anchors, positive proposal anchors, and a sampled
mini batch.
The 2D iou can be modified to show the effect of changing the iou
threshold for mini batch sampling.
In order to let this demo run without training an RPN, the proposals
shown are being read from a text file.
Keys:
F1: Toggle ground truth
F2: Toggle ortho rotated ground truth
F3: Toggle negative proposal anchors
F4: Toggle positive proposal anchors
F5: Toggle mini batch anchors
"""
##############################
# Options
##############################
# Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>)
config_dir = None
# checkpoint_name = None
checkpoint_name = 'mlod_exp_example'
data_split = 'val_half'
# global_step = None
global_step = 100000
# # # Cars # # #
# sample_name = "000050"
sample_name = "000104"
# sample_name = "000764"
# # # People # # #
# val_half
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
mini_batch_size = 512
neg_proposal_2d_iou_hi = 0.6
pos_proposal_2d_iou_lo = 0.65
bkg_proposals_line_width = 0.5
neg_proposals_line_width = 0.5
mid_proposals_line_width = 0.5
pos_proposals_line_width = 1.0
##############################
# End of Options
##############################
img_idx = int(sample_name)
print("Showing mini batch for sample {}".format(sample_name))
# Read proposals from file
if checkpoint_name is None:
# Use VAL Dataset
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL)
# Load demo proposals
proposals_and_scores_dir = mlod.top_dir() + \
'/demos/data/predictions/' + checkpoint_name + \
'/proposals_and_scores/' + dataset.data_split
else:
if config_dir is None:
config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name
# Parse experiment config
pipeline_config_file = \
config_dir + '/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
dataset_config.data_split = data_split
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Overwrite
mini_batch_utils = dataset.kitti_utils.mini_batch_utils
mini_batch_utils.mlod_neg_iou_range[1] = neg_proposal_2d_iou_hi
mini_batch_utils.mlod_pos_iou_range[0] = pos_proposal_2d_iou_lo
# Load proposals from outputs folder
proposals_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + \
'/predictions/proposals_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
proposals_and_scores = np.loadtxt(
proposals_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_anchors = box_3d_encoder.box_3d_to_anchor(proposal_boxes_3d)
# Get filtered ground truth
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_objs = dataset.kitti_utils.filter_labels(obj_labels)
# Convert ground truth to anchors
gt_boxes_3d = np.asarray([
box_3d_encoder.object_label_to_box_3d(obj_label)
for obj_label in filtered_objs
])
gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d,
ortho_rotate=True)
# Ortho rotate ground truth
gt_ortho_boxes_3d = box_3d_encoder.anchors_to_box_3d(gt_anchors)
gt_ortho_objs = [
box_3d_encoder.box_3d_to_object_label(box_3d, obj_type='OrthoGt')
for box_3d in gt_ortho_boxes_3d
]
# Project gt and anchors into BEV
gt_bev_anchors, _ = \
anchor_projector.project_to_bev(gt_anchors,
dataset.kitti_utils.bev_extents)
bev_anchors, _ = \
anchor_projector.project_to_bev(proposal_anchors,
dataset.kitti_utils.bev_extents)
# Reorder boxes into (y1, x1, y2, x2) order
gt_bev_anchors_tf_order = anchor_projector.reorder_projected_boxes(
gt_bev_anchors)
bev_anchors_tf_order = anchor_projector.reorder_projected_boxes(
bev_anchors)
# Convert to box_list format for iou calculation
gt_anchor_box_list = box_list.BoxList(
tf.cast(gt_bev_anchors_tf_order, tf.float32))
anchor_box_list = box_list.BoxList(
tf.cast(bev_anchors_tf_order, tf.float32))
# Get IoU for every anchor
tf_all_ious = box_list_ops.iou(gt_anchor_box_list, anchor_box_list)
valid_ious = True
# Make sure the calculated IoUs contain values. Since its a [N, M]
# tensor, if there are no gt's for instance, that entry will be zero.
if tf_all_ious.shape[0] == 0 or tf_all_ious.shape[1] == 0:
print('#################################################')
print('Warning: This sample does not contain valid IoUs')
print('#################################################')
valid_ious = False
if valid_ious:
tf_max_ious = tf.reduce_max(tf_all_ious, axis=0)
tf_max_iou_indices = tf.argmax(tf_all_ious, axis=0)
# Sample an RPN mini batch from the non empty anchors
mini_batch_utils = dataset.kitti_utils.mini_batch_utils
# Overwrite mini batch size and sample a mini batch
mini_batch_utils.mlod_mini_batch_size = mini_batch_size
mb_mask_tf, _ = mini_batch_utils.sample_mlod_mini_batch(tf_max_ious)
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# Run the graph to calculate ious for every proposal and
# to get the mini batch mask
all_ious, max_ious, max_iou_indices = sess.run(
[tf_all_ious, tf_max_ious, tf_max_iou_indices])
mb_mask = sess.run(mb_mask_tf)
mb_anchors = proposal_anchors[mb_mask]
mb_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d(mb_anchors)
mb_anchor_ious = max_ious[mb_mask]
else:
# We have no valid IoU's, so assume all IoUs are zeros
# and the mini-batch contains all the anchors since we cannot
# mask without IoUs.
max_ious = np.zeros(proposal_boxes_3d.shape[0])
mb_anchor_ious = max_ious
mb_anchors = proposal_anchors
mb_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d(mb_anchors)
# Create list of positive/negative proposals based on iou
pos_proposal_objs = []
mid_proposal_objs = []
neg_proposal_objs = []
bkg_proposal_objs = []
for i in range(len(proposal_boxes_3d)):
box_3d = proposal_boxes_3d[i]
if max_ious[i] == 0.0:
# Background proposals
bkg_proposal_objs.append(
box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='BackgroundProposal'))
elif max_ious[i] < neg_proposal_2d_iou_hi:
# Negative proposals
neg_proposal_objs.append(
box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='NegativeProposal'))
elif max_ious[i] < pos_proposal_2d_iou_lo:
# Middle proposals (in between negative and positive)
mid_proposal_objs.append(
box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='MiddleProposal'))
elif max_ious[i] <= 1.0:
# Positive proposals
pos_proposal_objs.append(
box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='PositiveProposal'))
else:
raise ValueError('Invalid IoU > 1.0')
print('{} bkg, {} neg, {} mid, {} pos proposals:'.format(
len(bkg_proposal_objs), len(neg_proposal_objs), len(mid_proposal_objs),
len(pos_proposal_objs)))
# Convert the mini_batch anchors to object list
mb_obj_list = []
for i in range(len(mb_anchor_ious)):
if valid_ious and (mb_anchor_ious[i] >
mini_batch_utils.mlod_pos_iou_range[0]):
obj_type = "Positive"
else:
obj_type = "Negative"
obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i],
obj_type)
mb_obj_list.append(obj)
# Point cloud
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
points, point_colours = demo_utils.get_filtered_pc_and_colours(
dataset, image, img_idx)
# Visualize from here
vis_utils.visualization(dataset.rgb_image_dir, img_idx)
plt.show(block=False)
# VtkPointCloud
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# VtkBoxes for ground truth
vtk_gt_boxes = VtkBoxes()
vtk_gt_boxes.set_objects(filtered_objs, COLOUR_SCHEME)
# VtkBoxes for ortho ground truth
vtk_gt_ortho_boxes = VtkBoxes()
vtk_gt_ortho_boxes.set_objects(gt_ortho_objs, COLOUR_SCHEME)
# VtkBoxes for background proposals
vtk_bkg_proposal_boxes = VtkBoxes()
vtk_bkg_proposal_boxes.set_objects(bkg_proposal_objs, COLOUR_SCHEME)
vtk_bkg_proposal_boxes.set_line_width(bkg_proposals_line_width)
# VtkBoxes for negative proposals
vtk_neg_proposal_boxes = VtkBoxes()
vtk_neg_proposal_boxes.set_objects(neg_proposal_objs, COLOUR_SCHEME)
vtk_neg_proposal_boxes.set_line_width(neg_proposals_line_width)
# VtkBoxes for middle proposals
vtk_mid_proposal_boxes = VtkBoxes()
vtk_mid_proposal_boxes.set_objects(mid_proposal_objs, COLOUR_SCHEME)
vtk_mid_proposal_boxes.set_line_width(mid_proposals_line_width)
# VtkBoxes for positive proposals
vtk_pos_proposal_boxes = VtkBoxes()
vtk_pos_proposal_boxes.set_objects(pos_proposal_objs, COLOUR_SCHEME)
vtk_pos_proposal_boxes.set_line_width(pos_proposals_line_width)
# Create VtkBoxes for mini batch anchors
vtk_mb_boxes = VtkBoxes()
vtk_mb_boxes.set_objects(mb_obj_list, COLOUR_SCHEME)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Add actors
vtk_renderer.AddActor(axes)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_gt_ortho_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_bkg_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_neg_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_mid_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_pos_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_mb_boxes.vtk_actor)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("MLOD Mini Batch")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_gt_boxes.vtk_actor,
vtk_gt_ortho_boxes.vtk_actor,
vtk_bkg_proposal_boxes.vtk_actor,
vtk_neg_proposal_boxes.vtk_actor,
vtk_mid_proposal_boxes.vtk_actor,
vtk_pos_proposal_boxes.vtk_actor,
vtk_mb_boxes.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
# Create Dataset
dataset_config_path = mlod.root_dir() + \
'/configs/mb_preprocessing/rpn_cars.config'
dataset = DatasetBuilder.load_dataset_from_config(dataset_config_path)
# Random sample
sample_name = '000169'
anchor_strides = dataset.kitti_utils.anchor_strides
img_idx = int(sample_name)
print("Showing mini batch for sample {}".format(sample_name))
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# KittiUtils class
dataset_utils = dataset.kitti_utils
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
# Grab ground truth
ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx)
ground_truth_list = dataset_utils.filter_labels(ground_truth_list)
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
##############################
# Flip sample info
##############################
start_time = time.time()
flipped_image = kitti_aug.flip_image(image)
flipped_point_cloud = kitti_aug.flip_point_cloud(point_cloud)
flipped_gt_list = [kitti_aug.flip_label_in_3d_only(obj)
for obj in ground_truth_list]
flipped_ground_plane = kitti_aug.flip_ground_plane(ground_plane)
flipped_calib_p2 = kitti_aug.flip_stereo_calib_p2(
stereo_calib_p2, image_shape)
flipped_points = flipped_point_cloud.T
print('flip sample', time.time() - start_time)
##############################
# Generate anchors
##############################
clusters, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Read mini batch info
anchors_info = dataset_utils.get_anchors_info(
dataset.classes_name,
anchor_strides,
sample_name)
all_anchor_boxes_3d = []
all_ious = []
for class_idx in range(len(dataset.classes)):
anchor_boxes_3d = anchor_generator.generate(
area_3d=dataset.kitti_utils.area_extents,
anchor_3d_sizes=clusters[class_idx],
anchor_stride=anchor_strides[class_idx],
ground_plane=ground_plane)
if anchors_info:
indices, ious, offsets, classes = anchors_info
# Get non empty anchors from the indices
non_empty_anchor_boxes_3d = anchor_boxes_3d[indices]
all_anchor_boxes_3d.extend(non_empty_anchor_boxes_3d)
all_ious.extend(ious)
if not len(all_anchor_boxes_3d) > 0:
# Exit early if anchors_info is empty
print("No anchors, Please try a different sample")
return
# Convert to ndarrays
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
all_ious = np.asarray(all_ious)
##############################
# Flip anchors
##############################
start_time = time.time()
flipped_anchor_boxes_3d = kitti_aug.flip_boxes_3d(all_anchor_boxes_3d,
flip_ry=False)
print('flip anchors', time.time() - start_time)
# Overwrite with flipped things
all_anchor_boxes_3d = flipped_anchor_boxes_3d
points = flipped_points
ground_truth_list = flipped_gt_list
ground_plane = flipped_ground_plane
import tensorflow as tf
import numpy as np
from tensorflow.python.framework import ops
from tensorflow.python.ops import gen_nn_ops
import mlod
import mlod.builders.config_builder_util as config_builder
from mlod.builders.dataset_builder import DatasetBuilder
from wavedata.tools.obj_detection import obj_utils
from mlod.core import constants
from occlusion_mask_layer import OccMaskLayer
parser = argparse.ArgumentParser()
default_pipeline_config_path = mlod.root_dir() + \
'/configs/mlod_fpn_people.config'
parser.add_argument('--pipeline_config',
type=str,
dest='pipeline_config_path',
default=default_pipeline_config_path,
help='Path to the pipeline config')
parser.add_argument('--device',
type=str,
dest='device',
default='0',
help='CUDA device id')
checkpoint_to_restore = ''
def __init__(self, dataset):
self._dataset = dataset
self._mini_batch_sampler = \
balanced_positive_negative_sampler.BalancedPositiveNegativeSampler()
self._mini_batch_sampler2 = \
balanced_positive_negative_sampler2.BalancedPositiveNegativeSampler()
##############################
# Parse KittiUtils config
##############################
self.kitti_utils_config = dataset.config.kitti_utils_config
self._area_extents = self.kitti_utils_config.area_extents
self._anchor_strides = np.reshape(
self.kitti_utils_config.anchor_strides, (-1, 2))
##############################
# Parse MiniBatchUtils config
##############################
self.config = self.kitti_utils_config.mini_batch_config
self._density_threshold = self.config.density_threshold
# RPN mini batches
rpn_config = self.config.rpn_config
rpn_iou_type = rpn_config.WhichOneof('iou_type')
if rpn_iou_type == 'iou_2d_thresholds':
self.rpn_iou_type = '2d'
self.rpn_iou_thresholds = rpn_config.iou_2d_thresholds
elif rpn_iou_type == 'iou_3d_thresholds':
self.rpn_iou_type = '3d'
self.rpn_iou_thresholds = rpn_config.iou_3d_thresholds
self.rpn_neg_iou_range = [self.rpn_iou_thresholds.neg_iou_lo,
self.rpn_iou_thresholds.neg_iou_hi]
self.rpn_pos_iou_range = [self.rpn_iou_thresholds.pos_iou_lo,
self.rpn_iou_thresholds.pos_iou_hi]
rpn_img_config = self.config.rpn_img_config
self.rpn_img_iou_thresholds = rpn_img_config.iou_2d_thresholds
self.rpn_img_neg_iou_range = [self.rpn_img_iou_thresholds.neg_iou_lo,
self.rpn_img_iou_thresholds.neg_iou_hi]
self.rpn_img_pos_iou_range = [self.rpn_img_iou_thresholds.pos_iou_lo,
self.rpn_img_iou_thresholds.pos_iou_hi]
self.rpn_mini_batch_size = rpn_config.mini_batch_size
# MLOD mini batches
mlod_config = self.config.mlod_config
self.mlod_iou_type = '2d'
self.mlod_iou_thresholds = mlod_config.iou_2d_thresholds
self.mlod_neg_iou_range = [self.mlod_iou_thresholds.neg_iou_lo,
self.mlod_iou_thresholds.neg_iou_hi]
self.mlod_pos_iou_range = [self.mlod_iou_thresholds.pos_iou_lo,
self.mlod_iou_thresholds.pos_iou_hi]
self.mlod_mini_batch_size = mlod_config.mini_batch_size
mlod_img_config = self.config.mlod_img_config
self.mlod_img_iou_thresholds = mlod_img_config.iou_2d_thresholds
self.mlod_img_neg_iou_range = [self.mlod_img_iou_thresholds.neg_iou_lo,
self.mlod_img_iou_thresholds.neg_iou_hi]
self.mlod_img_pos_iou_range = [self.mlod_img_iou_thresholds.pos_iou_lo,
self.mlod_img_iou_thresholds.pos_iou_hi]
self.mlod_img_mini_batch_size = mlod_img_config.mini_batch_size
# Setup paths
self.mini_batch_dir = mlod.root_dir() + '/data/mini_batches/' + \
'iou_{}/'.format(self.rpn_iou_type) + \
dataset.name + '/' + dataset.cluster_split + '/' + \
dataset.bev_source
# Array column indices for saving to files
self.col_length = 15
self.col_anchor_indices = 0
self.col_ious = 1
self.col_offsets_lo = 2
self.col_offsets_hi = 8
self.col_class_idx = 8
self.col_img_ious = 9
self.col_img_offsets_lo = 10
self.col_img_offsets_hi = 14
self.col_img_class_idx = 14
def save_predictions_in_kitti_format(sample_names, checkpoint_name,
final_predictions_root_dir,
branch_predictions_root_dirs,
kitti_predictions_3d_dir, score_threshold,
sub_score_threshold, global_step):
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
# Round this because protobuf encodes default values as full decimal
score_threshold = round(score_threshold, 3)
# Get available prediction folders
predictions_root_dir = mlod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
final_predictions_dir = final_predictions_root_dir + \
'/' + str(global_step)
branch_prediction_dir_list = [
branch_dir + '/' + str(global_step)
for branch_dir in branch_predictions_root_dirs
]
if not os.path.exists(kitti_predictions_3d_dir):
os.makedirs(kitti_predictions_3d_dir)
# Do conversion
num_samples = len(sample_names)
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting detections from:', final_predictions_dir)
print('3D Detections being saved to:', kitti_predictions_3d_dir)
for sample_idx, sample_name in enumerate(sample_names):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(sample_idx + 1,
num_samples))
sys.stdout.flush()
prediction_file = sample_name + '.txt'
kitti_predictions_3d_file_path = kitti_predictions_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
sub_predictions_list = []
for branch_prediction_dir in branch_prediction_dir_list:
sub_predictions_file_path = branch_prediction_dir + \
'/' + prediction_file
sub_predictions = np.loadtxt(sub_predictions_file_path)
sub_predictions_list.append(sub_predictions)
# # Swap l, w for predictions where w > l
# swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
# fixed_predictions = np.copy(all_predictions)
# fixed_predictions[swapped_indices, 3] = all_predictions[
# swapped_indices, 4]
# fixed_predictions[swapped_indices, 4] = all_predictions[
# swapped_indices, 3]
score_filter = np.array(all_predictions[:, 7] >= score_threshold)
#print('main',score_filter)
score_filter1 = np.array(
sub_predictions_list[0][:, 7] >= sub_score_threshold[0])
#print('br0',score_filter1)
score_filter2 = np.array(
sub_predictions_list[1][:, 7] >= sub_score_threshold[1])
#print('br1',score_filter2)
all_predictions = all_predictions[score_filter & score_filter1
& score_filter2]
# If no predictions, skip to next file
if len(all_predictions) == 0:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(get_rgb_image_path(sample_name))
calib_dir = '/home/j7deng/Kitti/object/training/calib'
stereo_calib_p2 = calib_utils.read_calibration(calib_dir, img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, stereo_calib_p2, truncate=True, image_size=image.size)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
continue
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Get object types
all_pred_classes = all_predictions[:, 8].astype(np.int32)
dataset_classes = ['Pedestrian', 'Cyclist']
obj_types = [
dataset_classes[class_idx] for class_idx in all_pred_classes
]
# Truncation and Occlusion are always empty (see below)
# Alpha (Not computed)
kitti_predictions[:, 3] = -10 * np.ones(
(len(kitti_predictions)), dtype=np.int32)
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones(
(len(kitti_predictions), 2), dtype=np.int32)
# Stack 3D predictions text
kitti_text_3d = np.column_stack(
[obj_types, kitti_empty_1, kitti_predictions[:, 3:16]])
# Save to text files
np.savetxt(kitti_predictions_3d_file_path,
kitti_text_3d,
newline='\r\n',
fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
def main():
"""Flip RPN Mini Batch
Visualization of the mini batch anchors for RpnModel training.
Keys:
F1: Toggle mini batch anchors
F2: Flipped
"""
anchor_colour_scheme = {
"Car": (255, 0, 0), # Red
"Pedestrian": (255, 150, 50), # Orange
"Cyclist": (150, 50, 100), # Purple
"DontCare": (255, 255, 255), # White
"Anchor": (150, 150, 150), # Gray
"Regressed Anchor": (255, 255, 0), # Yellow
"Positive": (0, 255, 255), # Teal
"Negative": (255, 0, 255) # Purple
}
dataset_config_path = mlod.root_dir() + \
'/configs/mb_rpn_demo_cars.config'
# dataset_config_path = mlod.root_dir() + \
# '/configs/mb_rpn_demo_people.config'
##############################
# Options
##############################
# # # Random sample # # #
sample_name = None
# # # Cars # # #
# sample_name = "000001"
# sample_name = "000050"
# sample_name = "000104"
# sample_name = "000112"
# sample_name = "000169"
# sample_name = "000191"
sample_name = "003801"
# # # Pedestrians # # #
# sample_name = "000000"
# sample_name = "000011"
# sample_name = "000015"
# sample_name = "000028"
# sample_name = "000035"
# sample_name = "000134"
# sample_name = "000167"
# sample_name = '000379'
# sample_name = '000381'
# sample_name = '000397'
# sample_name = '000398'
# sample_name = '000401'
# sample_name = '000407'
# sample_name = '000486'
# sample_name = '000509'
# # Cyclists # # #
# sample_name = '000122'
# sample_name = '000448'
# # # Multiple classes # # #
# sample_name = "000764"
##############################
# End of Options
##############################
# Create Dataset
dataset = DatasetBuilder.load_dataset_from_config(dataset_config_path)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_list[sample_idx]
anchor_strides = dataset.kitti_utils.anchor_strides
img_idx = int(sample_name)
print("Showing mini batch for sample {}".format(sample_name))
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# KittiUtils class
dataset_utils = dataset.kitti_utils
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
points = point_cloud.T
# Grab ground truth
ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx)
ground_truth_list = dataset_utils.filter_labels(ground_truth_list)
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
##############################
# Flip sample info
##############################
start_time = time.time()
flipped_image = kitti_aug.flip_image(image)
flipped_point_cloud = kitti_aug.flip_point_cloud(point_cloud)
flipped_gt_list = [
kitti_aug.flip_label_in_3d_only(obj) for obj in ground_truth_list
]
flipped_ground_plane = kitti_aug.flip_ground_plane(ground_plane)
flipped_calib_p2 = kitti_aug.flip_stereo_calib_p2(stereo_calib_p2,
image_shape)
print('flip sample', time.time() - start_time)
flipped_points = flipped_point_cloud.T
point_colours = vis_utils.project_img_to_point_cloud(
points, image, dataset.calib_dir, img_idx)
##############################
# Generate anchors
##############################
clusters, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Read mini batch info
anchors_info = dataset_utils.get_anchors_info(sample_name)
all_anchor_boxes_3d = []
all_ious = []
all_offsets = []
for class_idx in range(len(dataset.classes)):
anchor_boxes_3d = anchor_generator.generate(
area_3d=dataset.kitti_utils.area_extents,
anchor_3d_sizes=clusters[class_idx],
anchor_stride=anchor_strides[class_idx],
ground_plane=ground_plane)
if len(anchors_info[class_idx]) > 0:
indices, ious, offsets, classes = anchors_info[class_idx]
# Get non empty anchors from the indices
non_empty_anchor_boxes_3d = anchor_boxes_3d[indices]
all_anchor_boxes_3d.extend(non_empty_anchor_boxes_3d)
all_ious.extend(ious)
all_offsets.extend(offsets)
if not len(all_anchor_boxes_3d) > 0:
# Exit early if anchors_info is empty
print("No anchors, Please try a different sample")
return
# Convert to ndarrays
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
all_ious = np.asarray(all_ious)
all_offsets = np.asarray(all_offsets)
##############################
# Flip anchors
##############################
start_time = time.time()
# Flip anchors and offsets
flipped_anchor_boxes_3d = kitti_aug.flip_boxes_3d(all_anchor_boxes_3d,
flip_ry=False)
all_offsets[:, 0] = -all_offsets[:, 0]
print('flip anchors and offsets', time.time() - start_time)
# Overwrite with flipped things
all_anchor_boxes_3d = flipped_anchor_boxes_3d
points = flipped_points
ground_truth_list = flipped_gt_list
ground_plane = flipped_ground_plane
##############################
# Mini batch sampling
##############################
# Sample an RPN mini batch from the non empty anchors
mini_batch_utils = dataset.kitti_utils.mini_batch_utils
mb_mask_tf, _ = mini_batch_utils.sample_rpn_mini_batch(all_ious)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
mb_mask = sess.run(mb_mask_tf)
mb_anchor_boxes_3d = all_anchor_boxes_3d[mb_mask]
mb_anchor_ious = all_ious[mb_mask]
mb_anchor_offsets = all_offsets[mb_mask]
# ObjectLabel list that hold all boxes to visualize
obj_list = []
# Convert the mini_batch anchors to object list
for i in range(len(mb_anchor_boxes_3d)):
if mb_anchor_ious[i] > mini_batch_utils.rpn_pos_iou_range[0]:
obj_type = "Positive"
else:
obj_type = "Negative"
obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i],
obj_type)
obj_list.append(obj)
# Convert all non-empty anchors to object list
non_empty_anchor_objs = \
[box_3d_encoder.box_3d_to_object_label(
anchor_box_3d, obj_type='Anchor')
for anchor_box_3d in all_anchor_boxes_3d]
##############################
# Regress Positive Anchors
##############################
# Convert anchor_boxes_3d to anchors and apply offsets
mb_pos_mask = mb_anchor_ious > mini_batch_utils.rpn_pos_iou_range[0]
mb_pos_anchor_boxes_3d = mb_anchor_boxes_3d[mb_pos_mask]
mb_pos_anchor_offsets = mb_anchor_offsets[mb_pos_mask]
mb_pos_anchors = box_3d_encoder.box_3d_to_anchor(mb_pos_anchor_boxes_3d)
regressed_pos_anchors = anchor_encoder.offset_to_anchor(
mb_pos_anchors, mb_pos_anchor_offsets)
# Convert regressed anchors to ObjectLabels for visualization
regressed_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d(
regressed_pos_anchors, fix_lw=True)
regressed_anchor_objs = \
[box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='Regressed Anchor')
for box_3d in regressed_anchor_boxes_3d]
##############################
# Visualization
##############################
cv2.imshow('{} flipped'.format(sample_name), flipped_image)
cv2.waitKey()
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for mini batch anchors
vtk_pos_anchor_boxes = VtkBoxes()
vtk_pos_anchor_boxes.set_objects(obj_list, anchor_colour_scheme)
# VtkBoxes for non empty anchors
vtk_non_empty_anchors = VtkBoxes()
vtk_non_empty_anchors.set_objects(non_empty_anchor_objs,
anchor_colour_scheme)
vtk_non_empty_anchors.set_line_width(0.1)
# VtkBoxes for regressed anchors
vtk_regressed_anchors = VtkBoxes()
vtk_regressed_anchors.set_objects(regressed_anchor_objs,
anchor_colour_scheme)
vtk_regressed_anchors.set_line_width(5.0)
# Create VtkBoxes for ground truth
vtk_gt_boxes = VtkBoxes()
vtk_gt_boxes.set_objects(ground_truth_list,
anchor_colour_scheme,
show_orientations=True)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
vtk_ground_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, dataset.kitti_utils.bev_extents)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_non_empty_anchors.vtk_actor)
vtk_renderer.AddActor(vtk_pos_anchor_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_regressed_anchors.vtk_actor)
vtk_renderer.AddActor(vtk_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("RPN Mini Batch")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_non_empty_anchors.vtk_actor,
vtk_pos_anchor_boxes.vtk_actor,
vtk_regressed_anchors.vtk_actor,
vtk_ground_plane.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
"""This demo shows RPN proposals and MLOD predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
##############################
# Options
##############################
dataset_config.data_split = 'val'
fig_size = (10, 6.1)
rpn_score_threshold = 0.1
mlod_score_threshold = 0.1
# Flag for projecting the 3D boxes to image space
# in tensor format (for testing purposes)
test_img_tensor_projection = False
gt_classes = ['Pedestrian', 'Cyclist']
# gt_classes = ['Pedestrian', 'Cyclist']
# Overwrite this to select a specific checkpoint
global_step = 44000
checkpoint_name = 'mlod_fpn_people'
# Drawing Toggles
draw_proposals_separate = False
draw_overlaid = False
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = False
draw_orientations_on_pred = False
# Draw 2D bounding boxes
draw_projected_2d_boxes = False
# Save images for samples with no detections
save_empty_images = True
draw_score = True
draw_iou = False
iou_3d = False
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
# Setup Paths
predictions_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir = predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, rpn_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, mlod_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step,
mlod_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
for sample_idx in range(dataset.num_samples):
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s'. format(
sample_idx + 1,
dataset.num_samples,
avg_time,
est_time_left))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
if not os.path.exists(proposals_file_path):
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
score_mask = proposal_scores > rpn_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step,
sample_name))
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
mlod_score_mask = prediction_scores >= 0.1
mlod_show_mask = mlod_score_mask
prediction_boxes_3d = prediction_boxes_3d[mlod_show_mask]
prediction_scores = prediction_scores[mlod_show_mask]
prediction_class_indices = \
prediction_class_indices[mlod_show_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
dataset.has_labels = False
if dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
boxes2d, _, _ = obj_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(dataset.calib_dir,
img_idx)
calib_p2 = stereo_calib.p2
##############################
# Reformat and prepare to draw
##############################
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_encoder.box_3d_to_anchor(
proposal_boxes_3d)
if test_img_tensor_projection:
proposal_boxes = demo_utils.tf_project_to_image_space(
proposals_as_anchors, calib_p2, image_size, img_idx)
else:
proposal_boxes, _ = anchor_projector.project_to_image_space(
proposals_as_anchors, calib_p2, image_size)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_2d_axes, prop_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False)
draw_proposals(filtered_gt_objs,
calib_p2,
num_of_proposals,
proposal_objs,
proposal_boxes,
prop_2d_axes,
prop_3d_axes,
draw_orientations_on_prop)
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, calib_p2,
truncate=True, image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
prop_2d_axes,
prop_3d_axes,
draw_score,
draw_iou,
gt_classes,
draw_orientations_on_pred,
iou_3d)
filename = overlaid_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_2d_axes,
pred_3d_axes,
draw_score,
draw_iou,
gt_classes,
draw_orientations_on_pred,
iou_3d)
else:
pred_fig, pred_3d_axes = \
vis_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_3d_axes,
draw_score,
draw_iou,
gt_classes,
draw_orientations_on_pred)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
print('\nDone')
