def load_samples(self, indices, OrientNet=False):
""" Loads input-output data for a set of samples. Should only be
called when a particular sample dict is required. Otherwise,
samples should be provided by the next_batch function
Args:
indices: A list of sample indices from the dataset.sample_list
to be loaded
Return:
samples: a list of data sample dicts
"""
sample_dicts = []
# print('+++++++++++++ load_samples ++++++++++++++++')
# print('load_samples() input : indices = ', indices)
for sample_idx in indices:
sample = self.sample_list[sample_idx]
sample_name = sample.name
# Only read labels if they exist
if self.has_labels:
# Read mini batch first to see if it is empty
anchors_info = self.get_anchors_info(sample_name)
if (not anchors_info) and self.train_val_test == 'train' \
and (not self.train_on_all_samples):
empty_sample_dict = {
constants.KEY_SAMPLE_NAME: sample_name,
constants.KEY_ANCHORS_INFO: anchors_info
}
return [empty_sample_dict]
obj_labels = obj_panoptic_utils.read_labels(
self.label_dir, int(sample_name))
# Only use objects that match dataset classes
obj_labels = self.panoptic_utils.filter_labels(obj_labels)
else:
obj_labels = None
anchors_info = []
label_anchors = np.zeros((1, 6))
label_boxes_3d = np.zeros((1, 7))
label_boxes_2d = np.zeros((1, 4))
label_classes = np.zeros(1)
img_idx = int(sample_name)
# Load image (BGR -> RGB)
# print('Load image (BGR -> RGB)')
cv_bgr_image = cv2.imread(self.get_rgb_image_path(sample_name))
rgb_image = cv_bgr_image[..., ::-1]
image_shape = rgb_image.shape[0:2]
image_input = rgb_image
# Load MRCNN mask and features
# print('Load MRCNN mask and features')
mrcnn_result = self.panoptic_utils.get_mrcnn_result(img_idx)
# If no pedestrian can be seen on the images, break
if not mrcnn_result:
print('+++++++++++++ No mrcnn_result for ', img_idx,
'. load_samples, early end ++++++++++++++++')
return []
# mrcnn_result.item().get('keypoints'): shape = Nx17x3. [N, 17, [x, y, 1]]
image_mrcnn_keypoints_input = mrcnn_result.item().get(
'keypoints')[:, :, :2]
# image_mrcnn_keypoints_input: shape = Nx17x2. [N, 17, [x, y]]
image_mrcnn_feature_input = mrcnn_result.item().get('features')
image_mrcnn_bbox_input = mrcnn_result.item().get('rois')
# rois: [batch, N, (y1, x1, y2, x2)] detection bounding boxes
image_mask_input = mrcnn_result.item().get('masks')
image_full_mask_input = mrcnn_result.item().get('full_masks')
# Get ground plane
# print('Get ground plane')
ground_plane = obj_panoptic_utils.get_road_plane(
int(sample_name), self.planes_dir)
# Get calibration
# print('Get calibration')
stereo_calib_p2 = calib_panoptic_utils.read_calibration(
self.calib_dir, int(sample_name)).HD_11
point_cloud = self.panoptic_utils.get_point_cloud(
self.bev_source, img_idx, image_shape)
# Augmentation (Flipping)
if panoptic_aug.AUG_FLIPPING in sample.augs:
print('Flipping images')
image_input = panoptic_aug.flip_image(image_input)
point_cloud = panoptic_aug.flip_point_cloud(point_cloud)
obj_labels = [
panoptic_aug.flip_label_in_3d_only(obj)
for obj in obj_labels
]
ground_plane = panoptic_aug.flip_ground_plane(ground_plane)
stereo_calib_p2 = panoptic_aug.flip_stereo_calib_p2(
stereo_calib_p2, image_shape)
# Augmentation (Image Jitter)
if panoptic_aug.AUG_PCA_JITTER in sample.augs:
print('Image Jitter')
image_input[:, :, 0:3] = panoptic_aug.apply_pca_jitter(
image_input[:, :, 0:3])
if obj_labels is not None:
# print('obj_labels is not None!!')
label_boxes_3d = np.asarray( # [x, y, z, l, w, h, ry]
[
box_3d_panoptic_encoder.object_label_to_box_3d(
obj_label) for obj_label in obj_labels
])
label_boxes_2d = np.asarray( # [x1, y1, x2, y2]
[
box_3d_panoptic_encoder.object_label_to_box_2d(
obj_label) for obj_label in obj_labels
])
label_classes = [
self.panoptic_utils.class_str_to_index(obj_label.type)
for obj_label in obj_labels
]
label_classes = np.asarray(label_classes, dtype=np.int32)
# Return empty anchors_info if no ground truth after filtering
if len(label_boxes_3d) == 0:
print('len(label_boxes_3d) = ', len(label_boxes_3d))
if self.train_on_all_samples:
# If training without any positive labels, we cannot
# set these to zeros, because later on the offset calc
# uses log on these anchors. So setting any arbitrary
# number here that does not break the offset calculation
# should work, since the negative samples won't be
# regressed in any case.
dummy_anchors = [[-1000, -1000, -1000, 1, 1, 1]]
label_anchors = np.asarray(dummy_anchors)
dummy_boxes_3d = [[-1000, -1000, -1000, 1, 1, 1, 0]]
label_boxes_3d = np.asarray(dummy_boxes_3d)
dummy_boxes_2d = [[-1000, -1000, -1000, -1000]]
label_boxes_2d = np.asarray(dummy_boxes_2d)
else:
label_anchors = np.zeros((1, 6))
label_boxes_3d = np.zeros((1, 7))
label_boxes_2d = np.zeros((1, 4))
label_classes = np.zeros(1)
else:
# print('label_boxes_3d = ', label_boxes_3d)
label_anchors = box_3d_panoptic_encoder.box_3d_to_anchor(
label_boxes_3d, ortho_rotate=True)
# Read OrientNet data to overwrite the label_boxes_3d
if OrientNet:
orientnet_result = self.panoptic_utils.get_orientnet_result(
img_idx)
label_boxes_3d = orientnet_result.item().get('boxes_3d')
# Create BEV maps
# print('Create BEV maps')
bev_images = self.panoptic_utils.create_bev_maps(
point_cloud, ground_plane)
height_maps = bev_images.get('height_maps')
occupancy_maps = bev_images.get('occupancy_maps')
density_map = bev_images.get('density_map')
# bev_input = np.dstack(height_maps)
# density_map = bev_images.get('density_map')
# bev_input = np.dstack((*height_maps, density_map))
# print('^&^&^&^&^&^& Saving BEV images ^&^&^&^&^&^&^&^&')
# file_name = '/home/boom/play/try_python/500100018739_bev_height.npy'
# print('_save_mrcnn_to_file :: file_name = ', file_name)
# np.save(file_name, height_maps)
# file_name = '/home/boom/play/try_python/500100018739_bev_density.npy'
# print('_save_mrcnn_to_file :: file_name = ', file_name)
# np.save(file_name, density_map)
bev_input = np.dstack((*height_maps, density_map, *occupancy_maps))
# file_name = '/home/boom/play/try_python/500100018739_bev_stack.npy'
# print('_save_mrcnn_to_file :: file_name = ', file_name)
# np.save(file_name, bev_input)
sample_dict = {
constants.KEY_LABEL_BOXES_3D: label_boxes_3d,
constants.KEY_LABEL_BOXES_2D: label_boxes_2d,
constants.KEY_LABEL_ANCHORS: label_anchors,
constants.KEY_LABEL_CLASSES: label_classes,
constants.KEY_IMAGE_INPUT: image_input,
constants.KEY_BEV_INPUT: bev_input,
constants.KEY_IMAGE_MASK_INPUT: image_mask_input,
constants.KEY_IMAGE_FULL_MASK_INPUT: image_full_mask_input,
constants.KEY_IMAGE_MRCNN_FEATURE_INPUT:
image_mrcnn_feature_input,
constants.KEY_IMAGE_MRCNN_BBOX_INPUT: image_mrcnn_bbox_input,
constants.KEY_IMAGE_MRCNN_KEYPOINTS_INPUT:
image_mrcnn_keypoints_input,
constants.KEY_ANCHORS_INFO: anchors_info,
constants.KEY_POINT_CLOUD: point_cloud,
constants.KEY_GROUND_PLANE: ground_plane,
constants.KEY_STEREO_CALIB_P2: stereo_calib_p2,
constants.KEY_SAMPLE_NAME: sample_name,
constants.KEY_SAMPLE_AUGS: sample.augs
}
sample_dicts.append(sample_dict)
# print('+++++++++++++ load_samples end ++++++++++++++++')
return sample_dicts
def main():
"""This demo shows OrientNet predictions on 2D image space.
Given certain thresholds for predictions, it selects and draws the
groundtruth bounding boxes on the image sample. It goes through the entire
prediction samples for the given dataset split.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.PANOPTIC_VAL)
##############################
# Options
##############################
dataset_config = DatasetBuilder.merge_defaults(dataset_config)
dataset_config.data_split = 'val'
# fig_size = (10, 6.1)
fig_size = (12, 23) # The size of final picture as a whole.
rpn_score_threshold = 0.00
orientnet_score_threshold = 0.30
gt_classes = ['Pedestrian']
# Overwrite this to select a specific checkpoint
print('!!!Please make sure your settings are all correct!!!!!')
global_step = 298261 # None
checkpoint_name = 'orientation_pedestrian_panoptic'
# Drawing Toggles
draw_proposals_separate = False
draw_overlaid = False # To draw both proposal and predcition bounding boxes
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = False # Set it to false would be OK, since all the orietations of proposals are poiting to the 0 angle.
draw_orientations_on_pred = True
# Draw 2D bounding boxes
draw_projected_2d_boxes = True
# Draw BEV bounding boxes
draw_bev_map = False
# Draw pointclouds
draw_point_cloud = True
point_cloud_source = 'lidar'
slices_config = \
"""
slices {
height_lo: -5 # -0.2
height_hi: 2 # 2.3
num_slices: 1 # 5
}
"""
print('slices_config = ', slices_config)
text_format.Merge(slices_config,
dataset_config.panoptic_utils_config.bev_generator)
# Save images for samples with no detections
save_empty_images = False
draw_proposals_bev = True
draw_proposals_2d_box = False
draw_proposals_3d_box = True
draw_proposals_score = True
draw_proposals_iou = True
draw_prediction_score = True
draw_prediction_iou = True
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_panoptic_dataset(dataset_config,
use_defaults=False)
# Setup Paths
predictions_dir = pplp.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir = predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, rpn_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, orientnet_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step, orientnet_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
# for sample_idx in [100]:
# print('Hack the number!!!!!')
for sample_idx in range(dataset.num_samples):
print('\nStart sample #', sample_idx + 1)
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s \n'.format(
sample_idx + 1, dataset.num_samples, avg_time,
est_time_left))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
print('proposals_file_path = ', proposals_file_path)
if not os.path.exists(proposals_file_path):
print(proposals_file_path, 'does not exist!')
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
# change 1D array in to 2D array even if it has only one row.
if len(proposals_and_scores.shape) == 1:
proposals_and_scores.shape = (1, -1)
# proposals_and_scores, 1~7th colunms are the boxes_3d,
# the 8th colunm is the score.
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
print('rpn_score_threshold = ', rpn_score_threshold)
score_mask = proposal_scores >= rpn_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
print('There are ', len(proposal_scores), 'proposals left. ')
proposal_objs = \
[box_3d_panoptic_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
print('predictions_file_path NOT EXIST: ',
predictions_file_path)
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
# change 1D array in to 2D array even if it has only one row.
if len(predictions_and_scores.shape) == 1:
predictions_and_scores.shape = (1, -1)
# print('predictions_and_scores = ', predictions_and_scores)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
# print('prediction_scores = ', prediction_scores)
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
avod_score_mask = prediction_scores >= orientnet_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
print('orientnet_score_threshold = ',
orientnet_score_threshold)
print('There are ', len(prediction_boxes_3d),
' predictions left.')
prediction_scores = prediction_scores[avod_score_mask]
prediction_class_indices = \
prediction_class_indices[avod_score_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
if dataset.has_labels:
print('dataset.label_dir = ', dataset.label_dir)
print('img_idx = ', img_idx)
gt_objects = obj_panoptic_utils.read_labels(
dataset.label_dir, img_idx)
# for obj in gt_objects:
# print('obj.x1 = ', obj.x1)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.panoptic_utils.filter_labels(
gt_objects, classes=gt_classes)
# if sample_idx == 100:
# for obj in filtered_gt_objs:
# if obj.t[0]>1:
# # print('obj.x1 = ', obj.x1)
# # print('obj.y1 = ', obj.y1)
# # print('obj.x2 = ', obj.x2)
# # print('obj.y2 = ', obj.y2)
# print('obj.t = ', obj.t)
# print('obj.w = ', obj.w)
# print('obj.h = ', obj.h)
# print('obj.l = ', obj.l)
# # print('filtered_gt_objs.x1 = ', filtered_gt_objs.x1)
# # print('filtered_gt_objs.x2 = ', filtered_gt_objs.x2)
# # print('filtered_gt_objs.y1 = ', filtered_gt_objs.y1)
# # print('filtered_gt_objs.y2 = ', filtered_gt_objs.y2)
boxes2d, _, _ = obj_panoptic_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_panoptic_utils.read_calibration(
dataset.calib_dir, img_idx)
calib_p2 = stereo_calib.HD_11
distortion = stereo_calib.Kd_11
##############################
# Reformat and prepare to draw
##############################
# To get the BEV occupancy map, we need to find the ground plane first.
panoptic_utils = dataset.panoptic_utils
ground_plane = panoptic_utils.get_ground_plane(sample_name)
image_shape = [image.size[1], image.size[0]]
point_cloud = panoptic_utils.get_point_cloud('lidar', img_idx,
image_shape)
bev_maps = panoptic_utils.create_bev_maps(point_cloud, ground_plane)
bev_img = np.array(
bev_maps['occupancy_maps'], dtype=np.int
) # Remember, the original occupancy grid format is int.
bev_img = np.resize(
bev_img, (bev_img.shape[1], bev_img.shape[2])) # [height, width]
if not draw_bev_map:
bev_img = np.zeros((bev_img.shape[1], bev_img.shape[2]),
dtype=np.float)
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_panoptic_encoder.box_3d_to_anchor(
proposal_boxes_3d)
proposal_boxes, _ = anchor_panoptic_projector.project_to_image_space(
proposals_as_anchors,
calib_p2,
image_size,
distortion=distortion)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_bev_axes, prop_2d_axes, prop_3d_axes = \
vis_panoptic_utils.visualization(dataset.rgb_image_dir,
img_idx,
bev_img,
display=False,
fig_size=fig_size)
draw_proposals(filtered_gt_objs,
calib_p2,
num_of_proposals,
proposal_objs,
proposal_scores,
proposal_boxes,
prop_2d_axes,
prop_3d_axes,
prop_bev_axes,
panoptic_utils.area_extents,
bev_img.shape,
draw_proposals_bev,
draw_proposals_2d_box,
draw_proposals_3d_box,
draw_proposals_score,
draw_proposals_iou,
draw_orientations_on_prop,
distortion=distortion)
if draw_point_cloud:
# First,get pointclouds. Now pointclouds are in camera coordinates.
panoptic_utils = dataset.panoptic_utils
image_shape = [image_size[1], image_size[0]]
point_cloud = panoptic_utils.get_point_cloud(
point_cloud_source, img_idx, image_shape)
# print('point_cloud =', point_cloud)
# Now point_cloud is a 4XN array, in Lidar frame, but only
# includes those points that can be seen on the image
# Filter the useful pointclouds from all points
# In order to do that, we need to find the ground plane first.
ground_plane = panoptic_utils.get_ground_plane(sample_name)
filtered_points = panoptic_utils.filter_bev_points(
point_cloud, ground_plane)
# if len(filtered_points) > 0:
# print('point_cloud =', point_cloud)
# print('filtered_points =', filtered_points)
# Now, filtered_points is transposed, so filtered_points should
# be Nx4
# Project the filtered pointclouds on 2D image. Now filtered
# pointclouds are already in camera coordinates.
point_2d = obj_panoptic_utils.project_points_on_2D_image(
img_idx, dataset.calib_dir, image_size, filtered_points)
draw_points(prop_2d_axes, point_2d, 'red', pt_size=4)
# TODO: Project the filtered pointclouds on BEV image. Now filtered
# pointclouds are already in camera coordinates.
# point_bev = obj_panoptic_utils.project_points_on_BEV_image(img_idx,
# dataset.calib_dir,
# image_size,
# filtered_points)
# draw_points(prop_bev_axes, point_bev, 'red', pt_size=4)
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.jpg'
print('Draw proposals_separate: ', filename)
# Now add the legends
# prop_bev_axes.legend(loc='best', shadow=True, fontsize=20)
# prop_2d_axes.legend(loc='best', shadow=True, fontsize=20)
# prop_3d_axes.legend(loc='upper right', shadow=True, fontsize=20)
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
# print('prediction_boxes_3d = ', prediction_boxes_3d)
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_panoptic_projector.project_to_image_space(
box_3d,
calib_p2,
truncate=True,
image_size=image_size,
discard_before_truncation=False,
distortion=distortion)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_panoptic_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_bev_axes, pred_2d_axes, pred_3d_axes = \
vis_panoptic_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.jpg'
plt.savefig(filename)
print('Draw empty_images: ', filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
prop_2d_axes,
prop_3d_axes,
prop_bev_axes,
panoptic_utils.area_extents,
bev_img.shape,
draw_prediction_score,
draw_prediction_iou,
gt_classes,
draw_orientations_on_pred,
distortion=distortion)
filename = overlaid_out_dir + '/' + sample_name + '.jpg'
# Now add the legends
# prop_bev_axes.legend(loc='best', shadow=True, fontsize=20)
# prop_2d_axes.legend(loc='best', shadow=True, fontsize=20)
# prop_3d_axes.legend(loc='upper right', shadow=True, fontsize=20)
plt.savefig(filename)
print('Draw overlaid: ', filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_bev_axes, pred_2d_axes, pred_3d_axes = \
vis_panoptic_utils.visualization(dataset.rgb_image_dir,
img_idx,
bev_img,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_2d_axes,
pred_3d_axes,
pred_bev_axes,
panoptic_utils.area_extents,
bev_img.shape,
draw_prediction_score,
draw_prediction_iou,
gt_classes,
draw_orientations_on_pred,
distortion=distortion)
# Now add the legends
# pred_bev_axes.legend(loc='best', shadow=True, fontsize=20)
# pred_2d_axes.legend(loc='best', shadow=True, fontsize=20)
# pred_3d_axes.legend(loc='best', shadow=True, fontsize=20)
else:
pred_fig, pred_3d_axes = \
vis_panoptic_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_3d_axes,
draw_prediction_score,
draw_prediction_iou,
gt_classes,
draw_orientations_on_pred,
distortion=distortion)
# Now add the legends
# pred_3d_axes.legend(loc='upper right', shadow=True, fontsize=20)
filename = pred_out_dir + '/' + sample_name + '.jpg'
plt.savefig(filename)
print('Draw predictions_separate: ', filename)
plt.close(pred_fig)
print('\nDone')
def preprocess(self, indices):
"""Preprocesses anchor info and saves info to files
Args:
indices (int array): sample indices to process.
If None, processes all samples
"""
# Get anchor stride for class
anchor_strides = self._anchor_strides
dataset = self._dataset
dataset_utils = self._dataset.panoptic_utils
classes_name = dataset.classes_name
# Make folder if it doesn't exist yet
output_dir = self.mini_batch_panoptic_utils.get_file_path(
classes_name, anchor_strides, sample_name=None)
os.makedirs(output_dir, exist_ok=True)
# Get clusters for class
all_clusters_sizes, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Load indices of data_split
all_samples = dataset.sample_list
if indices is None:
indices = np.arange(len(all_samples))
num_samples = len(indices)
print('indices = ', indices)
# For each image in the dataset, save info on the anchors
for sample_idx in indices:
# Get image name for given cluster
sample_name = all_samples[sample_idx].name
img_idx = int(sample_name)
# Check for existing files and skip to the next
if self._check_for_existing(classes_name, anchor_strides,
sample_name):
print("{} / {}: Sample already preprocessed".format(
sample_idx + 1, num_samples, sample_name))
continue
# Get ground truth and filter based on difficulty
ground_truth_list = obj_panoptic_utils.read_labels(
dataset.label_dir, img_idx)
# Filter objects to dataset classes
# print('mini_batch_panoptic_preprocessor.py : ')
# print('ground_truth_list = ', ground_truth_list)
# If no valid ground truth, skip this image
if not ground_truth_list:
print("{} / {} No {}s for sample {} "
"(Ground Truth Filter)".format(sample_idx + 1,
num_samples, classes_name,
sample_name))
# Output an empty file and move on to the next image.
self._save_to_file(classes_name, anchor_strides, sample_name)
continue
filtered_gt_list = dataset_utils.filter_labels(ground_truth_list)
filtered_gt_list = np.asarray(filtered_gt_list)
# Filtering by class has no valid ground truth, skip this image
if len(filtered_gt_list) == 0:
print("{} / {} No {}s for sample {} "
"(Ground Truth Filter)".format(sample_idx + 1,
num_samples, classes_name,
sample_name))
# Output an empty file and move on to the next image.
self._save_to_file(classes_name, anchor_strides, sample_name)
continue
# Get ground plane
ground_plane = obj_panoptic_utils.get_road_plane(
img_idx, dataset.planes_dir)
image = Image.open(dataset.get_rgb_image_path(sample_name))
image_shape = [image.size[1], image.size[0]]
# Generate sliced 2D voxel grid for filtering
# print('******** Generate sliced 2D voxel grid for filtering *********')
print('sample_name = ', sample_name)
# print('dataset.bev_source = ', dataset.bev_source)
# print('image_shape = ', image_shape)
# If run with density filter:
# vx_grid_2d = dataset_utils.create_sliced_voxel_grid_2d(
# sample_name,
# source=dataset.bev_source,
# image_shape=image_shape)
# If run with occupancy filter:
point_cloud = dataset_utils.get_point_cloud(
'lidar', img_idx, image_shape)
bev_maps = dataset_utils.create_bev_maps(point_cloud, ground_plane)
# List for merging all anchors
all_anchor_boxes_3d = []
# Create anchors for each class
# print('mini_batch_panoptic_preprocessor.py : Create anchors for each class')
# print('mini_batch_panoptic_preprocessor.py : dataset.classes = ', dataset.classes)
for class_idx in range(len(dataset.classes)):
# Generate anchors for all classes
# print('class_idx = ', class_idx)
# print('len(dataset.classes) = ', len(dataset.classes))
# print('self._area_extents = ', self._area_extents)
# print('all_clusters_sizes = ', all_clusters_sizes)
# print('self._anchor_strides = ', self._anchor_strides)
# print('ground_plane = ', ground_plane)
grid_anchor_boxes_3d = anchor_generator.generate(
area_3d=self._area_extents,
anchor_3d_sizes=all_clusters_sizes[class_idx],
anchor_stride=self._anchor_strides[class_idx],
ground_plane=ground_plane)
all_anchor_boxes_3d.extend(grid_anchor_boxes_3d)
# Filter empty anchors
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
# For panoptic dataset, we have 14*16*2(degree 0 and pi) = 448 anchor_boxes_3d here!
np.set_printoptions(threshold=np.nan)
# print('all_anchor_boxes_3d = ', all_anchor_boxes_3d)
anchors = box_3d_panoptic_encoder.box_3d_to_anchor(
all_anchor_boxes_3d)
# Use density filter as anchor filters:
# Here, we created 448 boolean maska for all 488 anchors,
# but only those whose density>1 is counted.
# Usually the final Numberis between 20~40.
# empty_anchor_filter = anchor_filter.get_empty_anchor_filter_2d(
# anchors, vx_grid_2d, self._density_threshold)
# Use occupancy filter as anchor filters:
# print('bev_maps[\'occupancy_maps\'] = ', bev_maps['occupancy_maps'])
empty_anchor_filter = anchor_filter.get_empty_anchor_filter_occupancy(
anchors, bev_maps['occupancy_maps'], self._area_extents)
# Calculate anchor info
anchors_info = self._calculate_anchors_info(
all_anchor_boxes_3d, empty_anchor_filter, filtered_gt_list)
# anchors_info : N x 9 [anchor_indices, max_gt_iou(2d or 3d), (6 x offsets), class_index]
# max_gt_out - highest 3D iou with any ground truth box
# offsets - encoded offsets [dx, dy, dz, d_dimx, d_dimy, d_dimz]
# class_index - the anchor's class as an index
# (e.g. 0 or 1, for "Background" or "Pedestrian")
# for example, anchors_info[23] = [ 3.69000000e+02 1.23000000e-01 -6.01920353e-01 5.23366792e-03
# 5.50028119e-01 -1.05079512e-01 1.73129711e-02 5.26857404e-01
# 1.00000000e+00]
# print('empty_anchor_filter = ', empty_anchor_filter)
# In our dataset, anchor_ious is the second row of anchors_info
anchor_ious = anchors_info[:,
self.mini_batch_panoptic_utils.col_ious]
# print('anchors_info[:, 0] = ', anchors_info[:, 0])
# print('anchor_ious = ', anchor_ious)
# There will always be corner cases. For example, for image 500100003527.jpg
# There are 2 pedestrians on the image, but the Panoptic dataset only
# provide groundtruth skeletons for an occluded person. In this case,
# even though we have 1 groundtruth label, the anchor_ious still equals
# to all zeros! So we better save an empty file for this image.
if np.max(anchor_ious) == 0:
print("{} / {} . Anchor_ious are all zeros for sample {} "
"(Save an empty file for this minibatch)".format(
sample_idx + 1, num_samples, sample_name))
# Output an empty file and move on to the next image.
self._save_to_file(classes_name, anchor_strides, sample_name)
continue
valid_iou_indices = np.where(anchor_ious > 0.0)[0]
print("{} / {}:"
"{:>6} anchors, "
"{:>6} iou > 0.0, "
"for {:>3} {}(s) for sample {}".format(
sample_idx + 1, num_samples, len(anchors_info),
len(valid_iou_indices), len(filtered_gt_list),
classes_name, sample_name))
# Save anchors info
# under "anchors_info", we only saved those anchors that has density>1
self._save_to_file(classes_name, anchor_strides, sample_name,
anchors_info)
def preprocess_mrcnn(self, indices):
"""Preprocesses MRCNN result to files
Args:
indices (int array): sample indices to process.
If None, processes all samples
"""
# Get anchor stride for class
dataset = self._dataset
dataset_utils = self._dataset.panoptic_utils
classes_name = dataset.classes_name
# Make folder if it doesn't exist yet
sub_str = 'mrcnn'
output_dir = self.mini_batch_panoptic_utils.make_file_path(
classes_name, sub_str, sample_name=None)
os.makedirs(output_dir, exist_ok=True)
# Get clusters for class
all_clusters_sizes, _ = dataset.get_cluster_info()
# Load indices of data_split
all_samples = dataset.sample_list
if indices is None:
indices = np.arange(len(all_samples))
num_samples = len(indices)
# Initialize MskRCNN Model:
# Root directory of the project
ROOT_DIR = os.getcwd()
# Directory to save logs and trained model
MODEL_DIR = os.path.join(ROOT_DIR, "mylogs")
# Local path to trained weights file
COCO_MODEL_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco_humanpose.h5")
# Download COCO trained weights from Releases if needed
if not os.path.exists(COCO_MODEL_PATH):
maskrcnn_utils.download_trained_weights(COCO_MODEL_PATH)
class InferenceConfig(coco.CocoConfig):
GPU_COUNT = 1
IMAGES_PER_GPU = 1
KEYPOINT_MASK_POOL_SIZE = 7
inference_config = InferenceConfig()
# Recreate the model in inference mode
model = modellib.MaskRCNN(mode="inference",
config=inference_config,
model_dir=MODEL_DIR)
# Get path to saved weights
# Either set a specific path or find last trained weights
# model_path = os.path.join(ROOT_DIR, ".h5 file name here")
# model_path = model.find_last()[1]
model_path = os.path.join(ROOT_DIR, "mask_rcnn_coco_humanpose.h5")
# Load trained weights (fill in path to trained weights here)
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
# For each image in the dataset, save info on the anchors
for sample_idx in indices:
print('########## loop starts ###################')
# Get image name for given cluster
sample_name = all_samples[sample_idx].name
img_idx = int(sample_name)
print('img_idx = ', img_idx)
# Check for existing files and skip to the next
if self._check_for_mrcnn_existing(classes_name, sub_str,
sample_name):
print("{} / {}: Sample already preprocessed".format(
sample_idx + 1, num_samples, sample_name))
continue
# Get ground truth and filter based on difficulty
ground_truth_list = obj_panoptic_utils.read_labels(
dataset.label_dir, img_idx)
# print('ground_truth_list = ', ground_truth_list)
# If no valid ground truth, skip this image
if not ground_truth_list:
print("{} / {} No {}s for sample {} "
"(Ground Truth Filter)".format(sample_idx + 1,
num_samples, classes_name,
sample_name))
# Output an empty file and move on to the next image.
self._save_mrcnn_to_file(classes_name, sub_str, sample_name)
continue
# Filter objects to dataset classes
filtered_gt_list = dataset_utils.filter_labels(ground_truth_list)
filtered_gt_list = np.asarray(filtered_gt_list)
# print('filtered_gt_list = ', filtered_gt_list)
# Filtering by class has no valid ground truth, skip this image
if len(filtered_gt_list) == 0:
print("{} / {} No {}s for sample {} "
"(Ground Truth Filter)".format(sample_idx + 1,
num_samples, classes_name,
sample_name))
# Output an empty file and move on to the next image.
self._save_mrcnn_to_file(classes_name, sub_str, sample_name)
continue
# Get RGB image
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
print('Reading image: ', dataset.get_rgb_image_path(sample_name))
# BGR->RGB
image = image[:, :, ::-1]
# Run detection
mrcnn_results = model.detect_keypoint_and_feature_map([image],
verbose=0)
# print('mrcnn_results = ', mrcnn_results)
if len(mrcnn_results) == 0:
print('No people detected on the image!')
# Output an empty file and move on to the next image.
self._save_mrcnn_to_file(classes_name, sub_str, sample_name)
else:
print('There are ', len(mrcnn_results['rois']),
' people on the image.')
# Save Image MaskRCNN info
self._save_mrcnn_to_file(classes_name, sub_str, sample_name,
mrcnn_results)
self._visualize_mrcnn_to_file(image, classes_name, sub_str,
sample_name, mrcnn_results)
def get_clusters(self):
"""
Calculates clusters for each class
Returns:
all_clusters: list of clusters for each class
all_std_devs: list of cluster standard deviations for each class
"""
classes = self._dataset.classes
num_clusters = self._dataset.num_clusters
all_clusters = [[] for _ in range(len(classes))]
all_std_devs = [[] for _ in range(len(classes))]
classes_not_loaded = []
# Try to read from file first
for class_idx in range(len(classes)):
clusters, std_devs = self._read_clusters_from_file(
self._dataset, classes[class_idx], num_clusters[class_idx])
if clusters is not None:
all_clusters[class_idx].extend(np.asarray(clusters))
all_std_devs[class_idx].extend(np.asarray(std_devs))
else:
classes_not_loaded.append(class_idx)
# Return the data flattened into N x 3 arrays
if len(classes_not_loaded) == 0:
return all_clusters, all_std_devs
# Calculate the remaining clusters
# Load labels corresponding to the sample list for clustering
sample_list = self._dataset.load_sample_names(self.cluster_split)
all_labels = [[] for _ in range(len(classes))]
num_samples = len(sample_list)
for sample_idx in range(num_samples):
sys.stdout.write("\rClustering labels {} / {} \n".format(
sample_idx + 1, num_samples))
sys.stdout.flush()
# print('sample_list = ', sample_list)
# print('sample_idx = ', sample_idx)
sample_name = sample_list[sample_idx]
print('sample_name = ', sample_name)
img_idx = int(sample_name)
obj_labels = obj_panoptic_utils.read_labels(
self._dataset.label_dir, img_idx)
if not obj_labels:
print('No labels in sample ', img_idx)
continue
# print('obj_labels = ', obj_labels) # list of [<wavedata.tools.obj_detection.obj_panoptic_utils.ObjectLabel object at 0x7f13d0480438>]
# print('self._dataset.classes = ', self._dataset.classes)
filtered_labels = LabelClusterUtils._filter_labels_by_class(
obj_labels, self._dataset.classes)
# print('filtered_labels = ', filtered_labels)
for class_idx in range(len(classes)):
all_labels[class_idx].extend(filtered_labels[class_idx])
print("\nFinished reading labels, clustering data...\n")
# Cluster
for class_idx in classes_not_loaded:
labels_for_class = np.array(all_labels[class_idx])
print(
'************* label_cluster_panoptic_utils.py **************')
print('class_idx = ', class_idx)
print('num_clusters = ', num_clusters)
print('labels_for_class = ', labels_for_class)
n_clusters_for_class = num_clusters[class_idx]
print('n_clusters_for_class = ', n_clusters_for_class)
if len(labels_for_class) < n_clusters_for_class:
raise ValueError(
"Number of samples is less than number of clusters "
"{} < {}".format(len(labels_for_class),
n_clusters_for_class))
k_means = KMeans(n_clusters=n_clusters_for_class,
random_state=0).fit(labels_for_class)
clusters_for_class = []
std_devs_for_class = []
for cluster_idx in range(len(k_means.cluster_centers_)):
cluster_centre = k_means.cluster_centers_[cluster_idx]
labels_in_cluster = labels_for_class[k_means.labels_ ==
cluster_idx]
# Calculate std. dev
std_dev = np.std(labels_in_cluster, axis=0)
formatted_cluster = [
float('%.3f' % value) for value in cluster_centre
]
formatted_std_dev = [
float('%.3f' % value) for value in std_dev
]
clusters_for_class.append(formatted_cluster)
std_devs_for_class.append(formatted_std_dev)
# Write to files
file_path = self._get_cluster_file_path(self._dataset,
classes[class_idx],
num_clusters[class_idx])
self._write_clusters_to_file(file_path, clusters_for_class,
std_devs_for_class)
# Add to full list
all_clusters[class_idx].extend(np.asarray(clusters_for_class))
all_std_devs[class_idx].extend(np.asarray(std_devs_for_class))
# Return the data flattened into N x 3 arrays
return all_clusters, all_std_devs