def get_gts_based_on_difficulty(dataset, img_idx):
"""Returns lists of ground-truth based on difficulty.
"""
# Get all ground truth labels
all_gt_objs = obj_utils.read_labels(dataset.label_dir, img_idx)
# Filter to dataset classes
gt_objs = dataset.kitti_utils.filter_labels(all_gt_objs)
# Filter objects to desired difficulty
easy_gt_objs = dataset.kitti_utils.filter_labels(
copy.deepcopy(gt_objs), difficulty=0)
medium_gt_objs = dataset.kitti_utils.filter_labels(
copy.deepcopy(gt_objs), difficulty=1)
hard_gt_objs = dataset.kitti_utils.filter_labels(
copy.deepcopy(gt_objs), difficulty=2)
for gt_obj in easy_gt_objs:
gt_obj.type = 'Easy GT'
for gt_obj in medium_gt_objs:
gt_obj.type = 'Medium GT'
for gt_obj in hard_gt_objs:
gt_obj.type = 'Hard GT'
return easy_gt_objs, medium_gt_objs, hard_gt_objs, all_gt_objs
def main():
"""
Calculates clusters for each class
Returns:
all_clusters: list of clusters for each class
all_std_devs: list of cluster standard deviations for each class
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN)
# Calculate the remaining clusters
# Load labels corresponding to the sample list for clustering
sample_list = dataset.load_sample_names(dataset.cluster_split)
all_dims = []
num_samples = len(sample_list)
for sample_idx in range(num_samples):
sys.stdout.write("\rClustering labels {} / {}".format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = sample_list[sample_idx]
img_idx = int(sample_name)
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_lwh = LabelClusterUtils._filter_labels_by_class(
obj_labels, dataset.classes)
if filtered_lwh[0]:
all_dims.extend(filtered_lwh[0])
all_dims = np.array(all_dims)
print("\nFinished reading labels, clustering data...\n")
# Print 3 decimal places
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)})
# Calculate average cluster
k_means = KMeans(n_clusters=1,
random_state=0).fit(all_dims)
cluster_centre = k_means.cluster_centers_[0]
# Calculate std. dev
std_dev = np.std(all_dims, axis=0)
# Calculate 2 and 3 standard deviations below the mean
two_sigma_length_lo = cluster_centre[0] - 2 * std_dev[0]
three_sigma_length_lo = cluster_centre[0] - 3 * std_dev[0]
# Remove all labels with length above two std dev
# from the mean and re-cluster
small_mask_2 = all_dims[:, 0] < two_sigma_length_lo
small_dims_2 = all_dims[small_mask_2]
small_mask_3 = all_dims[:, 0] < three_sigma_length_lo
small_dims_3 = all_dims[small_mask_3]
small_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(small_dims_2)
small_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(small_dims_3)
small_std_dev_2 = np.std(small_dims_2, axis=0)
small_std_dev_3 = np.std(small_dims_3, axis=0)
print('small_k_means_2:', small_k_means_2.cluster_centers_)
print('small_k_means_3:', small_k_means_3.cluster_centers_)
print('small_std_dev_2:', small_std_dev_2)
print('small_std_dev_3:', small_std_dev_3)
# Calculate 2 and 3 standard deviations above the mean
two_sigma_length_hi = cluster_centre[0] + 2 * std_dev[0]
three_sigma_length_hi = cluster_centre[0] + 3 * std_dev[0]
# Remove all labels with length above two std dev
# from the mean and re-cluster
large_mask_2 = all_dims[:, 0] > two_sigma_length_hi
large_dims_2 = all_dims[large_mask_2]
large_mask_3 = all_dims[:, 0] > three_sigma_length_hi
large_dims_3 = all_dims[large_mask_3]
large_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(large_dims_2)
large_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(large_dims_3)
large_std_dev_2 = np.std(large_dims_2, axis=0)
large_std_dev_3 = np.std(large_dims_3, axis=0)
print('large_k_means_2:', large_k_means_2.cluster_centers_)
print('large_k_means_3:', large_k_means_3.cluster_centers_)
print('large_std_dev_2:', large_std_dev_2)
print('large_std_dev_3:', large_std_dev_3)
def main():
"""This demo runs through all samples in the trainval set, and checks
that the 3D box projection of all 'Car', 'Van', 'Pedestrian', and 'Cyclist'
objects are in the correct flipped 2D location after applying
modifications to the stereo p2 matrix.
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL,
use_defaults=True)
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)})
all_samples = dataset.sample_names
all_pixel_errors = []
all_max_pixel_errors = []
total_flip_time = 0.0
for sample_idx in range(dataset.num_samples):
sys.stdout.write('\r{} / {}'.format(sample_idx,
dataset.num_samples - 1))
sample_name = all_samples[sample_idx]
img_idx = int(sample_name)
# Run the main loop to run throughout the images
frame_calibration_info = calib_utils.read_calibration(
dataset.calib_dir, img_idx)
# Load labels
gt_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
gt_labels = dataset.kitti_utils.filter_labels(
gt_labels, ['Car', 'Van', 'Pedestrian', 'Cyclist'])
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_size = [image.shape[1], image.shape[0]]
# Flip p2 matrix
calib_p2 = frame_calibration_info.p2
flipped_p2 = np.copy(calib_p2)
flipped_p2[0, 2] = image.shape[1] - flipped_p2[0, 2]
flipped_p2[0, 3] = -flipped_p2[0, 3]
for obj_idx in range(len(gt_labels)):
obj = gt_labels[obj_idx]
# Get original 2D bounding boxes
orig_box_3d = box_3d_encoder.object_label_to_box_3d(obj)
orig_bbox_2d = box_3d_projector.project_to_image_space(
orig_box_3d, calib_p2, truncate=True, image_size=image_size)
# Skip boxes outside image
if orig_bbox_2d is None:
continue
orig_bbox_2d_flipped = flip_box_2d(orig_bbox_2d, image_size)
# Do flipping
start_time = time.time()
flipped_obj = kitti_aug.flip_label_in_3d_only(obj)
flip_time = time.time() - start_time
total_flip_time += flip_time
box_3d_flipped = box_3d_encoder.object_label_to_box_3d(flipped_obj)
new_bbox_2d_flipped = box_3d_projector.project_to_image_space(
box_3d_flipped,
flipped_p2,
truncate=True,
image_size=image_size)
pixel_errors = new_bbox_2d_flipped - orig_bbox_2d_flipped
max_pixel_error = np.amax(np.abs(pixel_errors))
all_pixel_errors.append(pixel_errors)
all_max_pixel_errors.append(max_pixel_error)
if max_pixel_error > 5:
print(' Error > 5px', sample_idx, max_pixel_error)
print(np.round(orig_bbox_2d_flipped, 3),
np.round(new_bbox_2d_flipped, 3))
print('Avg flip time:', total_flip_time / dataset.num_samples)
# Convert to ndarrays
all_pixel_errors = np.asarray(all_pixel_errors)
all_max_pixel_errors = np.asarray(all_max_pixel_errors)
# Print max values
print(np.amax(all_max_pixel_errors))
# Plot pixel errors
fig, axes = plt.subplots(nrows=3, ncols=1)
ax0, ax1, ax2 = axes.flatten()
ax0.hist(all_pixel_errors[:, 0], 50, histtype='bar', facecolor='green')
ax1.hist(all_pixel_errors[:, 2], 50, histtype='bar', facecolor='green')
ax2.hist(all_max_pixel_errors, 50, histtype='bar', facecolor='green')
plt.show()
def preprocess(self, indices):
"""Preprocesses anchor info and saves info to files
Args:
indices (int array): sample indices to process.
If None, processes all samples
"""
# Get anchor stride for class
anchor_strides = self._anchor_strides
dataset = self._dataset
dataset_utils = self._dataset.kitti_utils
classes_name = dataset.classes_name
# Make folder if it doesn't exist yet
output_dir = self.mini_batch_utils.get_file_path(classes_name,
anchor_strides,
sample_name=None)
os.makedirs(output_dir, exist_ok=True)
# Get clusters for class
all_clusters_sizes, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Load indices of data_split
all_samples = dataset.sample_list
if indices is None:
indices = np.arange(len(all_samples))
num_samples = len(indices)
# For each image in the dataset, save info on the anchors
for sample_idx in indices:
# Get image name for given cluster
sample_name = all_samples[sample_idx].name
img_idx = int(sample_name)
# Check for existing files and skip to the next
if self._check_for_existing(classes_name, anchor_strides,
sample_name):
print("{} / {}: Sample already preprocessed".format(
sample_idx + 1, num_samples, sample_name))
continue
# Get ground truth and filter based on difficulty
ground_truth_list = obj_utils.read_labels(dataset.label_dir,
img_idx)
# Filter objects to dataset classes
filtered_gt_list = dataset_utils.filter_labels(ground_truth_list)
filtered_gt_list = np.asarray(filtered_gt_list)
# Filtering by class has no valid ground truth, skip this image
if len(filtered_gt_list) == 0:
print("{} / {} No {}s for sample {} "
"(Ground Truth Filter)".format(
sample_idx + 1, num_samples,
classes_name, sample_name))
# Output an empty file and move on to the next image.
self._save_to_file(classes_name, anchor_strides, sample_name)
continue
# Get ground plane
ground_plane = obj_utils.get_road_plane(img_idx,
dataset.planes_dir)
image = Image.open(dataset.get_rgb_image_path(sample_name))
image_shape = [image.size[1], image.size[0]]
# Generate sliced 2D voxel grid for filtering
vx_grid_2d = dataset_utils.create_sliced_voxel_grid_2d(
sample_name,
source=dataset.bev_source,
image_shape=image_shape)
# List for merging all anchors
all_anchor_boxes_3d = []
# Create anchors for each class
for class_idx in range(len(dataset.classes)):
# Generate anchors for all classes
grid_anchor_boxes_3d = anchor_generator.generate(
area_3d=self._area_extents,
anchor_3d_sizes=all_clusters_sizes[class_idx],
anchor_stride=self._anchor_strides[class_idx],
ground_plane=ground_plane)
all_anchor_boxes_3d.extend(grid_anchor_boxes_3d)
# Filter empty anchors
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
anchors = box_3d_encoder.box_3d_to_anchor(all_anchor_boxes_3d)
empty_anchor_filter = anchor_filter.get_empty_anchor_filter_2d(
anchors, vx_grid_2d, self._density_threshold)
# Calculate anchor info
anchors_info = self._calculate_anchors_info(
all_anchor_boxes_3d, empty_anchor_filter, filtered_gt_list)
anchor_ious = anchors_info[:, self.mini_batch_utils.col_ious]
valid_iou_indices = np.where(anchor_ious > 0.0)[0]
print("{} / {}:"
"{:>6} anchors, "
"{:>6} iou > 0.0, "
"for {:>3} {}(s) for sample {}".format(
sample_idx + 1, num_samples,
len(anchors_info),
len(valid_iou_indices),
len(filtered_gt_list), classes_name, sample_name
))
# Save anchors info
self._save_to_file(classes_name, anchor_strides,
sample_name, anchors_info)
def main():
"""This demo shows p1 proposals and ammf predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
##############################
# Options
##############################
dataset_config.data_split = 'val' #bqx!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
fig_size = (10, 6.1)
p1_score_threshold = 0.8
ammf_score_threshold = 0.1
#gt_classes = ['Car']
gt_classes = ['Pedestrian', 'Cyclist']
# Overwrite this to select a specific checkpoint
global_step = None
#checkpoint_name = 'ammf_cars_example'
#checkpoint_name = 'pyramid_cars_with_aug_example'
checkpoint_name = 'people'
# Drawing Toggles
draw_proposals_separate = True
draw_overlaid = True
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = True
draw_orientations_on_pred = True
# Draw 2D bounding boxes
draw_projected_2d_boxes = True
# Save images for samples with no detections
save_empty_images = True
draw_score = True
draw_iou = True
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
# Setup Paths
predictions_dir = ammf.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir= predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
#steps = os.listdir()
#steps.sort(key=int)
#print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
#global_step = steps[-1]
global_step = '120000' #!!!!!!!!!!!!!!!!!!!!!!
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, p1_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, ammf_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step, ammf_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
for sample_idx in range(dataset.num_samples):
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s'.format(
sample_idx + 1, dataset.num_samples, avg_time,
est_time_left))
sys.stdout.flush()
#sample_idx=188
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
#img_idx = 188 #bqx!!!!!!!!!!!!!!!!!!!!111
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
if not os.path.exists(proposals_file_path):
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
score_mask = proposal_scores > p1_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
ammf_score_mask = prediction_scores >= ammf_score_threshold
prediction_boxes_3d = prediction_boxes_3d[ammf_score_mask]
prediction_scores = prediction_scores[ammf_score_mask]
prediction_class_indices = \
prediction_class_indices[ammf_score_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
if dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
boxes2d, _, _ = obj_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx)
calib_p2 = stereo_calib.p2
##############################
# Reformat and prepare to draw
##############################
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_encoder.box_3d_to_anchor(
proposal_boxes_3d)
proposal_boxes, _ = anchor_projector.project_to_image_space(
proposals_as_anchors, calib_p2, image_size)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_2d_axes, prop_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False)
draw_proposals(filtered_gt_objs, calib_p2, num_of_proposals,
proposal_objs, proposal_boxes, prop_2d_axes,
prop_3d_axes, draw_orientations_on_prop)
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
calib_p2,
truncate=True,
image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
prop_2d_axes, prop_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
filename = overlaid_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_2d_axes, pred_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
else:
pred_fig, pred_3d_axes = \
vis_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs, calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_3d_axes, draw_score, draw_iou,
gt_classes, draw_orientations_on_pred)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
print('\nDone')
def get_clusters(self):
"""
Calculates clusters for each class
Returns:
all_clusters: list of clusters for each class
all_std_devs: list of cluster standard deviations for each class
"""
classes = self._dataset.classes
num_clusters = self._dataset.num_clusters
all_clusters = [[] for _ in range(len(classes))]
all_std_devs = [[] for _ in range(len(classes))]
classes_not_loaded = []
# Try to read from file first
for class_idx in range(len(classes)):
clusters, std_devs = self._read_clusters_from_file(
self._dataset, classes[class_idx], num_clusters[class_idx])
if clusters is not None:
all_clusters[class_idx].extend(np.asarray(clusters))
all_std_devs[class_idx].extend(np.asarray(std_devs))
else:
classes_not_loaded.append(class_idx)
# Return the data flattened into N x 3 arrays
if len(classes_not_loaded) == 0:
return all_clusters, all_std_devs
# Calculate the remaining clusters
# Load labels corresponding to the sample list for clustering
sample_list = self._dataset.load_sample_names(self.cluster_split)
all_labels = [[] for _ in range(len(classes))]
num_samples = len(sample_list)
for sample_idx in range(num_samples):
sys.stdout.write("\rClustering labels {} / {}".format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = sample_list[sample_idx]
img_idx = int(sample_name)
obj_labels = obj_utils.read_labels(self._dataset.label_dir,
img_idx)
filtered_labels = LabelClusterUtils._filter_labels_by_class(
obj_labels, self._dataset.classes)
for class_idx in range(len(classes)):
all_labels[class_idx].extend(filtered_labels[class_idx])
print("\nFinished reading labels, clustering data...\n")
# Cluster
for class_idx in classes_not_loaded:
labels_for_class = np.array(all_labels[class_idx])
n_clusters_for_class = num_clusters[class_idx]
if len(labels_for_class) < n_clusters_for_class:
raise ValueError(
"Number of samples is less than number of clusters "
"{} < {}".format(len(labels_for_class),
n_clusters_for_class))
k_means = KMeans(n_clusters=n_clusters_for_class,
random_state=0).fit(labels_for_class)
clusters_for_class = []
std_devs_for_class = []
for cluster_idx in range(len(k_means.cluster_centers_)):
cluster_centre = k_means.cluster_centers_[cluster_idx]
labels_in_cluster = labels_for_class[
k_means.labels_ == cluster_idx]
# Calculate std. dev
std_dev = np.std(labels_in_cluster, axis=0)
formatted_cluster = [float('%.3f' % value)
for value in cluster_centre]
formatted_std_dev = [float('%.3f' % value)
for value in std_dev]
clusters_for_class.append(formatted_cluster)
std_devs_for_class.append(formatted_std_dev)
# Write to files
file_path = self._get_cluster_file_path(self._dataset,
classes[class_idx],
num_clusters[class_idx])
self._write_clusters_to_file(file_path, clusters_for_class,
std_devs_for_class)
# Add to full list
all_clusters[class_idx].extend(np.asarray(clusters_for_class))
all_std_devs[class_idx].extend(np.asarray(std_devs_for_class))
# Return the data flattened into N x 3 arrays
return all_clusters, all_std_devs
def main():
"""
Displays the bird's eye view maps for a KITTI sample.
"""
##############################
# Options
##############################
bev_generator = 'slices'
slices_config = \
"""
slices {
height_lo: -0.2
height_hi: 2.3
num_slices: 5
}
"""
# Use None for a random image
#img_idx = None
img_idx = 848
# img_idx = 191
show_ground_truth = True # Whether to overlay ground_truth boxes
point_cloud_source = 'lidar'
##############################
# End of Options
##############################
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
dataset_config = DatasetBuilder.merge_defaults(dataset_config)
# Overwrite bev_generator
if bev_generator == 'slices':
text_format.Merge(slices_config,
dataset_config.kitti_utils_config.bev_generator)
else:
raise ValueError('Invalid bev_generator')
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
if img_idx is None:
img_idx = int(random.random() * dataset.num_samples)
sample_name = "{:06}".format(img_idx)
print('=== Showing BEV maps for image: {}.png ==='.format(sample_name))
# Load image
#image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image = plt.imread(dataset.get_rgb_image_path(sample_name))
image_shape = image.shape[0:2]
kitti_utils = dataset.kitti_utils
point_cloud = kitti_utils.get_point_cloud(point_cloud_source, img_idx,
image_shape)
ground_plane = kitti_utils.get_ground_plane(sample_name)
bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane)
height_maps = np.array(bev_images.get("height_maps"))
density_map = np.array(bev_images.get("density_map"))
box_points, box_points_norm = [None, None]
if show_ground_truth:
# Get projected boxes
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_objs = obj_labels
label_boxes = []
for label in filtered_objs:
box = box_3d_encoder.object_label_to_box_3d(label)
label_boxes.append(box)
label_boxes = np.array(label_boxes)
box_points, box_points_norm = box_3d_projector.project_to_bev(
label_boxes, [[-40, 40], [0, 70]])
rgb_img_size = (np.array((1242, 375)) * 0.75).astype(np.int16)
img_x_start = 60
img_y_start = 330
img_x = img_x_start
img_y = img_y_start
img_w = 400
img_h = 350
img_titlebar_h = 20
# Show images
vis_utils.cv2_show_image("Image",
image,
size_wh=rgb_img_size,
location_xy=(img_x, 0))
# Height maps
for map_idx in range(len(height_maps)):
height_map = height_maps[map_idx]
height_map = draw_boxes(height_map, box_points_norm)
vis_utils.cv2_show_image("Height Map {}".format(map_idx),
height_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
img_x += img_w
# Wrap around
if (img_x + img_w) > 1920:
img_x = img_x_start
img_y += img_h + img_titlebar_h
# Density map
density_map = draw_boxes(density_map, box_points_norm)
vis_utils.cv2_show_image("Density Map",
density_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
cv2.waitKey()
def load_samples(self, indices):
""" Loads input-output data for a set of samples. Should only be
called when a particular sample dict is required. Otherwise,
samples should be provided by the next_batch function
Args:
indices: A list of sample indices from the dataset.sample_list
to be loaded
Return:
samples: a list of data sample dicts
"""
sample_dicts = []
for sample_idx in indices:
sample = self.sample_list[sample_idx]
sample_name = sample.name
#8.1 labels
# Only read labels if they exist
if self.has_labels:
# Read mini batch first to see if it is empty
anchors_info = self.get_anchors_info(sample_name)
if (not anchors_info) and self.train_val_test == 'train' \
and (not self.train_on_all_samples):
empty_sample_dict = {
constants.KEY_SAMPLE_NAME: sample_name,
constants.KEY_ANCHORS_INFO: anchors_info
}
return [empty_sample_dict]
obj_labels = obj_utils.read_labels(self.label_dir,
int(sample_name))
# Only use objects that match dataset classes
obj_labels = self.kitti_utils.filter_labels(obj_labels)
else:
obj_labels = None
anchors_info = []
label_anchors = np.zeros((1, 6))
label_boxes_3d = np.zeros((1, 7))
label_classes = np.zeros(1)
img_idx = int(sample_name)
#2. Load image (BGR -> RGB)
cv_bgr_image = cv2.imread(self.get_rgb_image_path(sample_name))
#cv_bgr_image = plt.imread(self.get_rgb_image_path(
#sample_name))
print(cv_bgr_image.shape)
print("cv_bgr_image.shape")
rgb_image = cv_bgr_image[..., ::-1]
image_shape = rgb_image.shape[0:2]
image_input = rgb_image
#.Load seg (BGR -> RGB)
#cv_bgr_seg = cv2.imread(self.get_rgb_seg_path(sample_name))
seg_input = 0
label_seg_input = 0
seg_filelist = os.listdir(
'/media/bangquanxie/4FCF996C7FA0ED8D/Kitti/object/training/image_seg_2'
)
if int(sample_name) < len(seg_filelist):
cv_bgr_seg = plt.imread(self.get_rgb_seg_path(sample_name))
print(cv_bgr_seg.shape)
print("cv_bgr_seg.shape")
rgb_seg = cv_bgr_seg[..., ::-1]
seg_shape = rgb_seg.shape[0:2]
seg_input = rgb_seg
cv_bgr_label_seg = plt.imread(
self.get_rgb_label_seg_path(sample_name))
print(cv_bgr_seg.shape)
print("cv_bgr_seg.shape")
rgb_label_seg = cv_bgr_label_seg[..., ::-1]
label_seg_shape = rgb_label_seg.shape[0:2]
label_seg_input = rgb_label_seg
'''
#.Load label_seg (BGR -> RGB) self.label_seg_dir
cv_bgr_label_seg = cv2.imread(self.get_rgb_label_seg_path(sample_name))
#cv_bgr_image = plt.imread(self.get_rgb_image_path(
#sample_name))
rgb_label_seg = cv_bgr_label_seg[..., :: -1]
label_seg_shape = rgb_label_seg.shape[0:2]
label_seg_input = rgb_label_seg
#bqx:road
dir_seg = 0
dir_seg = self.image_dir_seg
dir_segs = obj_utils.get_rgb_image_path_seg(self.image_dir_seg)
for i in range(len(dir_segs)):
dir_seg=dir_segs[i]
rgb_image = plt.imread(dir_seg)
#rgb_image = cv_bgr_image[..., :: -1]
image_shape = rgb_image.shape[0:2]
image_input_seg = rgb_image
'''
# Get ground plane
ground_plane = obj_utils.get_road_plane(int(sample_name),
self.planes_dir)
# Get calibration
stereo_calib_p2 = calib_utils.read_calibration(
self.calib_dir, int(sample_name)).p2
point_cloud = self.kitti_utils.get_point_cloud(
self.bev_source, img_idx, image_shape)
# Augmentation (Flipping)
if kitti_aug.AUG_FLIPPING in sample.augs:
image_input = kitti_aug.flip_image(image_input)
point_cloud = kitti_aug.flip_point_cloud(point_cloud)
obj_labels = [
kitti_aug.flip_label_in_3d_only(obj) for obj in obj_labels
]
ground_plane = kitti_aug.flip_ground_plane(ground_plane)
stereo_calib_p2 = kitti_aug.flip_stereo_calib_p2(
stereo_calib_p2, image_shape)
# Augmentation (Image Jitter)
if kitti_aug.AUG_PCA_JITTER in sample.augs:
image_input[:, :,
0:3] = kitti_aug.apply_pca_jitter(image_input[:, :,
0:3])
#bqx:
if obj_labels is not None:
label_boxes_3d = np.asarray([
box_3d_encoder.object_label_to_box_3d(obj_label)
for obj_label in obj_labels
])
label_classes = [
self.kitti_utils.class_str_to_index(obj_label.type)
for obj_label in obj_labels
]
label_classes = np.asarray(label_classes, dtype=np.int32)
#seg_label = np.asarray(seg_label, dtype=np.int32)
# Return empty anchors_info if no ground truth after filtering
if len(label_boxes_3d) == 0:
anchors_info = []
if self.train_on_all_samples:
# If training without any positive labels, we cannot
# set these to zeros, because later on the offset calc
# uses log on these anchors. So setting any arbitrary
# number here that does not break the offset calculation
# should work, since the negative samples won't be
# regressed in any case.
dummy_anchors = [[-1000, -1000, -1000, 1, 1, 1]]
label_anchors = np.asarray(dummy_anchors)
dummy_boxes = [[-1000, -1000, -1000, 1, 1, 1, 0]]
label_boxes_3d = np.asarray(dummy_boxes)
else:
label_anchors = np.zeros((1, 6))
label_boxes_3d = np.zeros((1, 7))
label_classes = np.zeros(1)
else:
label_anchors = box_3d_encoder.box_3d_to_anchor(
label_boxes_3d, ortho_rotate=True)
# Create BEV maps
bev_images = self.kitti_utils.create_bev_maps(
point_cloud, ground_plane)
height_maps = bev_images.get('height_maps')
density_map = bev_images.get('density_map')
bev_input = np.dstack((*height_maps, density_map))
sample_dict = {
constants.KEY_LABEL_BOXES_3D: label_boxes_3d,
constants.KEY_LABEL_ANCHORS: label_anchors,
constants.KEY_LABEL_CLASSES: label_classes,
#bqx:road
constants.KEY_IMAGE_INPUT: image_input,
constants.KEY_BEV_INPUT: bev_input,
#bqx:road
constants.KEY_SEG_INPUT: seg_input,
constants.KEY_LABEL_SEG: label_seg_input,
constants.KEY_ANCHORS_INFO: anchors_info,
constants.KEY_POINT_CLOUD: point_cloud,
constants.KEY_GROUND_PLANE: ground_plane,
constants.KEY_STEREO_CALIB_P2: stereo_calib_p2,
constants.KEY_SAMPLE_NAME: sample_name,
constants.KEY_SAMPLE_AUGS: sample.augs
}
sample_dicts.append(sample_dict)
return sample_dicts
def main():
"""Show histograms of ground truth labels
"""
dataset = DatasetBuilder.build_kitti_dataset(
# DatasetBuilder.KITTI_TRAIN
# DatasetBuilder.KITTI_VAL
DatasetBuilder.KITTI_TRAINVAL)
difficulty = 2
centroid_bins = 51
dimension_bins = 21
orientation_bins = 65
classes = ['Car']
# classes = ['Pedestrian']
# classes = ['Cyclist']
# classes = ['Pedestrian', 'Cyclist']
# Dataset values
num_samples = dataset.num_samples
all_centroids_x = []
all_centroids_y = []
all_centroids_z = []
all_lengths = []
all_widths = []
all_heights = []
all_orientations = []
# Counter for total number of valid samples
num_valid_samples = 0
for sample_idx in range(num_samples):
sys.stdout.write('\r{} / {}'.format(sample_idx + 1, num_samples))
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
obj_labels = dataset.kitti_utils.filter_labels(obj_labels,
classes=classes,
difficulty=difficulty)
centroids = np.asarray([obj.t for obj in obj_labels])
lengths = np.asarray([obj.l for obj in obj_labels])
widths = np.asarray([obj.w for obj in obj_labels])
heights = np.asarray([obj.h for obj in obj_labels])
orientations = np.asarray([obj.ry for obj in obj_labels])
if any(orientations) and np.amax(np.abs(orientations) > np.pi):
raise ValueError('Invalid orientation')
if len(centroids) > 0:
all_centroids_x.extend(centroids[:, 0])
all_centroids_y.extend(centroids[:, 1])
all_centroids_z.extend(centroids[:, 2])
all_lengths.extend(lengths)
all_widths.extend(widths)
all_heights.extend(heights)
all_orientations.extend(orientations)
num_valid_samples += 1
print('Finished reading labels, num_valid_samples', num_valid_samples)
# Get means
mean_centroid_x = np.mean(all_centroids_x)
mean_centroid_y = np.mean(all_centroids_y)
mean_centroid_z = np.mean(all_centroids_z)
mean_dims = np.mean([all_lengths, all_widths, all_heights])
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)})
print('mean_centroid_x {0:0.3f}'.format(mean_centroid_x))
print('mean_centroid_y {0:0.3f}'.format(mean_centroid_y))
print('mean_centroid_z {0:0.3f}'.format(mean_centroid_z))
print('mean_dims {0:0.3f}'.format(mean_dims))
# Make plots
f, ax_arr = plt.subplots(3, 3)
# xyz
ax_arr[0, 0].hist(all_centroids_x, centroid_bins, facecolor='green')
ax_arr[0, 1].hist(all_centroids_y, centroid_bins, facecolor='green')
ax_arr[0, 2].hist(all_centroids_z, centroid_bins, facecolor='green')
# lwh
ax_arr[1, 0].hist(all_lengths, dimension_bins, facecolor='green')
ax_arr[1, 1].hist(all_widths, dimension_bins, facecolor='green')
ax_arr[1, 2].hist(all_heights, dimension_bins, facecolor='green')
# orientations
ax_arr[2, 0].hist(all_orientations, orientation_bins, facecolor='green')
plt.show(block=True)