def preprocess(self, indices):
"""Preprocesses anchor info and saves info to files
Args:
indices (int array): sample indices to process.
If None, processes all samples
"""
# Get anchor stride for class
anchor_strides = self._anchor_strides
dataset = self._dataset
dataset_utils = self._dataset.kitti_utils
classes_name = dataset.classes_name
# Make folder if it doesn't exist yet
output_dir = self.mini_batch_utils.get_file_path(classes_name,
anchor_strides,
sample_name=None)
os.makedirs(output_dir, exist_ok=True)
# Get clusters for class
all_clusters_sizes, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Load indices of data_split
all_samples = dataset.sample_list
if indices is None:
indices = np.arange(len(all_samples))
num_samples = len(indices)
# For each image in the dataset, save info on the anchors
for sample_idx in indices:
# Get image name for given cluster
sample_name = all_samples[sample_idx].name
img_idx = int(sample_name)
# Check for existing files and skip to the next
if self._check_for_existing(classes_name, anchor_strides,
sample_name):
print("{} / {}: Sample already preprocessed".format(
sample_idx + 1, num_samples, sample_name))
continue
# Get ground truth and filter based on difficulty
ground_truth_list = obj_utils.read_labels(dataset.label_dir,
img_idx)
# Filter objects to dataset classes
filtered_gt_list = dataset_utils.filter_labels(ground_truth_list)
filtered_gt_list = np.asarray(filtered_gt_list)
# Filtering by class has no valid ground truth, skip this image
if len(filtered_gt_list) == 0:
print("{} / {} No {}s for sample {} "
"(Ground Truth Filter)".format(
sample_idx + 1, num_samples,
classes_name, sample_name))
# Output an empty file and move on to the next image.
self._save_to_file(classes_name, anchor_strides, sample_name)
continue
# Get ground plane
ground_plane = obj_utils.get_road_plane(img_idx,
dataset.planes_dir)
image = Image.open(dataset.get_rgb_image_path(sample_name))
image_shape = [image.size[1], image.size[0]]
# Generate sliced 2D voxel grid for filtering
vx_grid_2d = dataset_utils.create_sliced_voxel_grid_2d(
sample_name,
source=dataset.bev_source,
image_shape=image_shape)
# List for merging all anchors
all_anchor_boxes_3d = []
# Create anchors for each class
for class_idx in range(len(dataset.classes)):
# Generate anchors for all classes
grid_anchor_boxes_3d = anchor_generator.generate(
area_3d=self._area_extents,
anchor_3d_sizes=all_clusters_sizes[class_idx],
anchor_stride=self._anchor_strides[class_idx],
ground_plane=ground_plane)
all_anchor_boxes_3d.extend(grid_anchor_boxes_3d)
# Filter empty anchors
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
anchors = box_3d_encoder.box_3d_to_anchor(all_anchor_boxes_3d)
empty_anchor_filter = anchor_filter.get_empty_anchor_filter_2d(
anchors, vx_grid_2d, self._density_threshold)
# Calculate anchor info
anchors_info = self._calculate_anchors_info(
all_anchor_boxes_3d, empty_anchor_filter, filtered_gt_list)
anchor_ious = anchors_info[:, self.mini_batch_utils.col_ious]
valid_iou_indices = np.where(anchor_ious > 0.0)[0]
print("{} / {}:"
"{:>6} anchors, "
"{:>6} iou > 0.0, "
"for {:>3} {}(s) for sample {}".format(
sample_idx + 1, num_samples,
len(anchors_info),
len(valid_iou_indices),
len(filtered_gt_list), classes_name, sample_name
))
# Save anchors info
self._save_to_file(classes_name, anchor_strides,
sample_name, anchors_info)
def _fill_anchor_pl_inputs(self, anchors_info, ground_plane, image_shape,
stereo_calib_p2, sample_name, sample_augs):
"""
Fills anchor placeholder inputs with corresponding data
Args:
anchors_info: anchor info from mini_batch_utils
ground_plane: ground plane coefficients
image_shape: image shape (h, w), used for projecting anchors
sample_name: name of the sample, e.g. "000001"
sample_augs: list of sample augmentations
"""
# Lists for merging anchors info
all_anchor_boxes_3d = []
anchors_ious = []
anchor_offsets = []
anchor_classes = []
# Create anchors for each class
if len(self.dataset.classes) > 1:
for class_idx in range(len(self.dataset.classes)):
# Generate anchors for all classes
grid_anchor_boxes_3d = self._anchor_generator.generate(
area_3d=self._area_extents,
anchor_3d_sizes=self._cluster_sizes[class_idx],
anchor_stride=self._anchor_strides[class_idx],
ground_plane=ground_plane)
all_anchor_boxes_3d.append(grid_anchor_boxes_3d)
all_anchor_boxes_3d = np.concatenate(all_anchor_boxes_3d)
else:
# Don't loop for a single class
class_idx = 0
grid_anchor_boxes_3d = self._anchor_generator.generate(
area_3d=self._area_extents,
anchor_3d_sizes=self._cluster_sizes[class_idx],
anchor_stride=self._anchor_strides[class_idx],
ground_plane=ground_plane)
all_anchor_boxes_3d = grid_anchor_boxes_3d
# Filter empty anchors
# Skip if anchors_info is []
sample_has_labels = True
if self._train_val_test in ['train', 'val']:
# Read in anchor info during training / validation
if anchors_info:
anchor_indices, anchors_ious, anchor_offsets, \
anchor_classes = anchors_info
anchor_boxes_3d_to_use = all_anchor_boxes_3d[anchor_indices]
else:
train_cond = (self._train_val_test == "train"
and self._train_on_all_samples)
eval_cond = (self._train_val_test == "val"
and self._eval_all_samples)
if train_cond or eval_cond:
sample_has_labels = False
else:
sample_has_labels = False
if not sample_has_labels:
# During testing, or validation with no anchor info, manually
# filter empty anchors
# TODO: share voxel_grid_2d with BEV generation if possible
voxel_grid_2d = \
self.dataset.kitti_utils.create_sliced_voxel_grid_2d(
sample_name, self.dataset.bev_source,
image_shape=image_shape)
# Convert to anchors and filter
anchors_to_use = box_3d_encoder.box_3d_to_anchor(
all_anchor_boxes_3d)
empty_filter = anchor_filter.get_empty_anchor_filter_2d(
anchors_to_use, voxel_grid_2d, density_threshold=1)
anchor_boxes_3d_to_use = all_anchor_boxes_3d[empty_filter]
# Convert lists to ndarrays
anchor_boxes_3d_to_use = np.asarray(anchor_boxes_3d_to_use)
anchors_ious = np.asarray(anchors_ious)
anchor_offsets = np.asarray(anchor_offsets)
anchor_classes = np.asarray(anchor_classes)
# Flip anchors and centroid x offsets for augmented samples
if kitti_aug.AUG_FLIPPING in sample_augs:
anchor_boxes_3d_to_use = kitti_aug.flip_boxes_3d(
anchor_boxes_3d_to_use, flip_ry=False)
if anchors_info:
anchor_offsets[:, 0] = -anchor_offsets[:, 0]
# Convert to anchors
anchors_to_use = box_3d_encoder.box_3d_to_anchor(
anchor_boxes_3d_to_use)
num_anchors = len(anchors_to_use)
# Project anchors into bev
bev_anchors, bev_anchors_norm = anchor_projector.project_to_bev(
anchors_to_use, self._bev_extents)
# Project box_3d anchors into image space
img_anchors, img_anchors_norm = \
anchor_projector.project_to_image_space(
anchors_to_use, stereo_calib_p2, image_shape)
# Reorder into [y1, x1, y2, x2] for tf.crop_and_resize op
self._bev_anchors_norm = bev_anchors_norm[:, [1, 0, 3, 2]]
self._img_anchors_norm = img_anchors_norm[:, [1, 0, 3, 2]]
# Fill in placeholder inputs
self._placeholder_inputs[self.PL_ANCHORS] = anchors_to_use
# If we are in train/validation mode, and the anchor infos
# are not empty, store them. Checking for just anchors_ious
# to be non-empty should be enough.
if self._train_val_test in ['train', 'val'] and \
len(anchors_ious) > 0:
self._placeholder_inputs[self.PL_ANCHOR_IOUS] = anchors_ious
self._placeholder_inputs[self.PL_ANCHOR_OFFSETS] = anchor_offsets
self._placeholder_inputs[self.PL_ANCHOR_CLASSES] = anchor_classes
# During test, or val when there is no anchor info
elif self._train_val_test in ['test'] or \
len(anchors_ious) == 0:
# During testing, or validation with no gt, fill these in with 0s
self._placeholder_inputs[self.PL_ANCHOR_IOUS] = \
np.zeros(num_anchors)
self._placeholder_inputs[self.PL_ANCHOR_OFFSETS] = \
np.zeros([num_anchors, 6])
self._placeholder_inputs[self.PL_ANCHOR_CLASSES] = \
np.zeros(num_anchors)
else:
raise ValueError(
'Got run mode {}, and non-empty anchor info'.format(
self._train_val_test))
self._placeholder_inputs[self.PL_BEV_ANCHORS] = bev_anchors
self._placeholder_inputs[self.PL_BEV_ANCHORS_NORM] = \
self._bev_anchors_norm
self._placeholder_inputs[self.PL_IMG_ANCHORS] = img_anchors
self._placeholder_inputs[self.PL_IMG_ANCHORS_NORM] = \
self._img_anchors_norm
def preprocess(self, indices):
"""Preprocesses anchor info and saves info to files
Args:
indices (int array): sample indices to process.
If None, processes all samples
"""
# Get anchor stride for class
anchor_params = self._anchor_params
dataset = self._dataset
dataset_utils = self._dataset.kitti_utils
classes_name = dataset.classes_name
anchor_strides = anchor_params['anchor_strides']
# Make folder if it doesn't exist yet
output_dir = self.mini_batch_utils.get_file_path(classes_name,
anchor_strides,
sample_name=None)
os.makedirs(output_dir, exist_ok=True)
# Get clusters for class
#all_clusters_sizes, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_bev_generator.GridAnchorBevGenerator()
#anchor_type = self._dataset.kitti_utils.anchor_type
# Load indices of data_split
all_samples = dataset.sample_list
if indices is None:
indices = np.arange(len(all_samples))
#indices = indices[:10]
num_samples = len(indices)
# For each image in the dataset, save info on the anchors
for sample_idx in indices:
# Get image name for given cluster
sample_name = all_samples[sample_idx].name
img_idx = int(sample_name)
# Check for existing files and skip to the next
if self._check_for_existing(classes_name, anchor_strides,
sample_name):
print("{} / {}: Sample already preprocessed".format(
sample_idx + 1, num_samples, sample_name))
#continue
# Get ground truth and filter based on difficulty
ground_truth_list = obj_utils.read_labels(dataset.label_dir,
img_idx)
# Filter objects to dataset classes
filtered_gt_list = dataset_utils.filter_labels(ground_truth_list)
filtered_gt_list = np.asarray(filtered_gt_list)
# Filtering by class has no valid ground truth, skip this image
if len(filtered_gt_list) == 0:
print("{} / {} No {}s for sample {} "
"(Ground Truth Filter)".format(sample_idx + 1,
num_samples, classes_name,
sample_name))
# Output an empty file and move on to the next image.
#comment out for DEBUG
self._save_to_file(classes_name, anchor_strides, sample_name)
continue
# Get ground plane
ground_plane = obj_utils.get_road_plane(img_idx,
dataset.planes_dir)
image = Image.open(dataset.get_rgb_image_path(sample_name))
image_shape = [image.size[1], image.size[0]]
# List for merging all anchors
all_level_anchor_boxes_bev = anchor_generator.generate(\
image_shapes=anchor_params['image_shapes'],
anchor_base_sizes=anchor_params['anchor_base_sizes'],
anchor_strides=anchor_params['anchor_strides'],
anchor_ratios=anchor_params['anchor_ratios'],
anchor_scales=anchor_params['anchor_scales'],
anchor_init_ry_type=anchor_params['anchor_init_ry_type'])
#concate all levels anchors
#commentt out for DEBUG
all_anchor_boxes_bev = np.concatenate(all_level_anchor_boxes_bev)
#all_anchor_boxes_bev = all_level_anchor_boxes_bev[-1]
# Filter empty anchors (whose pts num < density_threshold)
# prepare for anchors_3d which dont need ry.
anchors_bev = all_anchor_boxes_bev.copy()
if anchor_params['anchor_init_ry_type'] == -90:
anchors_bev[:, [2, 3]] = anchors_bev[:, [3, 2]]
anchors_3d = box_bev_encoder.box_bev_to_anchor_3d(anchors_bev, \
bev_shape=self._bev_shape, \
bev_extents=self._dataset.kitti_utils.area_extents[[0, 2]])
#print(anchors_3d)
image = Image.open(dataset.get_rgb_image_path(sample_name))
image_shape = [image.size[1], image.size[0]]
# Generate sliced 2D voxel grid for filtering
vx_grid_2d = dataset_utils.create_sliced_voxel_grid_2d(
sample_name,
source=dataset.bev_source,
image_shape=image_shape)
empty_anchor_filter = anchor_filter.get_empty_anchor_filter_2d(
anchors_3d, vx_grid_2d, self._density_threshold)
print(
f'Non empty anchor: {np.sum(empty_anchor_filter)} / {len(all_anchor_boxes_bev)}, \
sample_name: {sample_name}')
#empty_anchor_filter = np.ones(all_anchor_boxes_bev.shape[0], dtype=bool)
# Calculate anchor info
anchors_info = self._calculate_anchors_info(
all_anchor_boxes_bev, empty_anchor_filter, filtered_gt_list)
n_invalid = np.sum(np.isnan(anchors_info))
if n_invalid > 0:
raise ValueError(
'Invalid value occur at anchors_info: nan, sample: ',
sample_name)
#
anchor_ious = anchors_info[:, self.mini_batch_utils.col_ious]
valid_iou_indices = np.where(anchor_ious > 0.0)[0]
print("{} / {}:"
"{:>6} anchors, "
"{:>6} iou > 0.0, "
"for {:>3} {}(s) for sample {}".format(
sample_idx + 1, num_samples, len(anchors_info),
len(valid_iou_indices), len(filtered_gt_list),
classes_name, sample_name))
# Save anchors info
#comment out for DEBUG
self._save_to_file(classes_name, anchor_strides, sample_name,
anchors_info)
