def main():
"""Generates anchors info which is used for mini batch sampling.
This code only generate mini batches for the CMU Panoptic dataset.
"""
panoptic_dataset_config_path = pplp.root_dir() + \
'/configs/mb_preprocessing/rpn_panoptic.config'
##############################
# Options
##############################
# Serial vs parallel processing
in_parallel = False # Change it back to True after debugging
process_ped = True # Pedestrians , default: False
# Number of child processes to fork, samples will
# be divided evenly amongst the processes (in_parallel must be True)
num_ped_children = 8
##############################
# Dataset setup
##############################
if process_ped:
ped_dataset = DatasetBuilder.load_dataset_from_config(
panoptic_dataset_config_path)
##############################
# Serial Processing
##############################
if not in_parallel:
if process_ped:
do_preprocessing(ped_dataset, None)
print('All Done (Serial)')
##############################
# Parallel Processing
##############################
else:
# List of all child pids to wait on
all_child_pids = []
# Pedestrians
if process_ped:
ped_indices_split = split_indices(ped_dataset, num_ped_children)
split_work(all_child_pids, ped_dataset, ped_indices_split,
num_ped_children)
# Wait to child processes to finish
print('num children:', len(all_child_pids))
for i, child_pid in enumerate(all_child_pids):
os.waitpid(child_pid, 0)
print('All Done (Parallel)')
def train(model_config, train_config, dataset_config):
dataset = DatasetBuilder.build_panoptic_dataset(dataset_config,
use_defaults=False)
train_val_test = 'train'
model_name = model_config.model_name
with tf.Graph().as_default():
if model_name == 'orient_model':
model = OrientModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
trainer.train(model, train_config)
def main():
"""This demo shows OrientNet predictions on 2D image space.
Given certain thresholds for predictions, it selects and draws the
groundtruth bounding boxes on the image sample. It goes through the entire
prediction samples for the given dataset split.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.PANOPTIC_VAL)
##############################
# Options
##############################
dataset_config = DatasetBuilder.merge_defaults(dataset_config)
dataset_config.data_split = 'val'
# fig_size = (10, 6.1)
fig_size = (12, 23) # The size of final picture as a whole.
rpn_score_threshold = 0.00
orientnet_score_threshold = 0.30
gt_classes = ['Pedestrian']
# Overwrite this to select a specific checkpoint
print('!!!Please make sure your settings are all correct!!!!!')
global_step = 298261 # None
checkpoint_name = 'orientation_pedestrian_panoptic'
# Drawing Toggles
draw_proposals_separate = False
draw_overlaid = False # To draw both proposal and predcition bounding boxes
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = False # Set it to false would be OK, since all the orietations of proposals are poiting to the 0 angle.
draw_orientations_on_pred = True
# Draw 2D bounding boxes
draw_projected_2d_boxes = True
# Draw BEV bounding boxes
draw_bev_map = False
# Draw pointclouds
draw_point_cloud = True
point_cloud_source = 'lidar'
slices_config = \
"""
slices {
height_lo: -5 # -0.2
height_hi: 2 # 2.3
num_slices: 1 # 5
}
"""
print('slices_config = ', slices_config)
text_format.Merge(slices_config,
dataset_config.panoptic_utils_config.bev_generator)
# Save images for samples with no detections
save_empty_images = False
draw_proposals_bev = True
draw_proposals_2d_box = False
draw_proposals_3d_box = True
draw_proposals_score = True
draw_proposals_iou = True
draw_prediction_score = True
draw_prediction_iou = True
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_panoptic_dataset(dataset_config,
use_defaults=False)
# Setup Paths
predictions_dir = pplp.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir = predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, rpn_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, orientnet_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step, orientnet_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
# for sample_idx in [100]:
# print('Hack the number!!!!!')
for sample_idx in range(dataset.num_samples):
print('\nStart sample #', sample_idx + 1)
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s \n'.format(
sample_idx + 1, dataset.num_samples, avg_time,
est_time_left))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
print('proposals_file_path = ', proposals_file_path)
if not os.path.exists(proposals_file_path):
print(proposals_file_path, 'does not exist!')
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
# change 1D array in to 2D array even if it has only one row.
if len(proposals_and_scores.shape) == 1:
proposals_and_scores.shape = (1, -1)
# proposals_and_scores, 1~7th colunms are the boxes_3d,
# the 8th colunm is the score.
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
print('rpn_score_threshold = ', rpn_score_threshold)
score_mask = proposal_scores >= rpn_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
print('There are ', len(proposal_scores), 'proposals left. ')
proposal_objs = \
[box_3d_panoptic_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
print('predictions_file_path NOT EXIST: ',
predictions_file_path)
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
# change 1D array in to 2D array even if it has only one row.
if len(predictions_and_scores.shape) == 1:
predictions_and_scores.shape = (1, -1)
# print('predictions_and_scores = ', predictions_and_scores)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
# print('prediction_scores = ', prediction_scores)
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
avod_score_mask = prediction_scores >= orientnet_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
print('orientnet_score_threshold = ',
orientnet_score_threshold)
print('There are ', len(prediction_boxes_3d),
' predictions left.')
prediction_scores = prediction_scores[avod_score_mask]
prediction_class_indices = \
prediction_class_indices[avod_score_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
if dataset.has_labels:
print('dataset.label_dir = ', dataset.label_dir)
print('img_idx = ', img_idx)
gt_objects = obj_panoptic_utils.read_labels(
dataset.label_dir, img_idx)
# for obj in gt_objects:
# print('obj.x1 = ', obj.x1)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.panoptic_utils.filter_labels(
gt_objects, classes=gt_classes)
# if sample_idx == 100:
# for obj in filtered_gt_objs:
# if obj.t[0]>1:
# # print('obj.x1 = ', obj.x1)
# # print('obj.y1 = ', obj.y1)
# # print('obj.x2 = ', obj.x2)
# # print('obj.y2 = ', obj.y2)
# print('obj.t = ', obj.t)
# print('obj.w = ', obj.w)
# print('obj.h = ', obj.h)
# print('obj.l = ', obj.l)
# # print('filtered_gt_objs.x1 = ', filtered_gt_objs.x1)
# # print('filtered_gt_objs.x2 = ', filtered_gt_objs.x2)
# # print('filtered_gt_objs.y1 = ', filtered_gt_objs.y1)
# # print('filtered_gt_objs.y2 = ', filtered_gt_objs.y2)
boxes2d, _, _ = obj_panoptic_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_panoptic_utils.read_calibration(
dataset.calib_dir, img_idx)
calib_p2 = stereo_calib.HD_11
distortion = stereo_calib.Kd_11
##############################
# Reformat and prepare to draw
##############################
# To get the BEV occupancy map, we need to find the ground plane first.
panoptic_utils = dataset.panoptic_utils
ground_plane = panoptic_utils.get_ground_plane(sample_name)
image_shape = [image.size[1], image.size[0]]
point_cloud = panoptic_utils.get_point_cloud('lidar', img_idx,
image_shape)
bev_maps = panoptic_utils.create_bev_maps(point_cloud, ground_plane)
bev_img = np.array(
bev_maps['occupancy_maps'], dtype=np.int
) # Remember, the original occupancy grid format is int.
bev_img = np.resize(
bev_img, (bev_img.shape[1], bev_img.shape[2])) # [height, width]
if not draw_bev_map:
bev_img = np.zeros((bev_img.shape[1], bev_img.shape[2]),
dtype=np.float)
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_panoptic_encoder.box_3d_to_anchor(
proposal_boxes_3d)
proposal_boxes, _ = anchor_panoptic_projector.project_to_image_space(
proposals_as_anchors,
calib_p2,
image_size,
distortion=distortion)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_bev_axes, prop_2d_axes, prop_3d_axes = \
vis_panoptic_utils.visualization(dataset.rgb_image_dir,
img_idx,
bev_img,
display=False,
fig_size=fig_size)
draw_proposals(filtered_gt_objs,
calib_p2,
num_of_proposals,
proposal_objs,
proposal_scores,
proposal_boxes,
prop_2d_axes,
prop_3d_axes,
prop_bev_axes,
panoptic_utils.area_extents,
bev_img.shape,
draw_proposals_bev,
draw_proposals_2d_box,
draw_proposals_3d_box,
draw_proposals_score,
draw_proposals_iou,
draw_orientations_on_prop,
distortion=distortion)
if draw_point_cloud:
# First,get pointclouds. Now pointclouds are in camera coordinates.
panoptic_utils = dataset.panoptic_utils
image_shape = [image_size[1], image_size[0]]
point_cloud = panoptic_utils.get_point_cloud(
point_cloud_source, img_idx, image_shape)
# print('point_cloud =', point_cloud)
# Now point_cloud is a 4XN array, in Lidar frame, but only
# includes those points that can be seen on the image
# Filter the useful pointclouds from all points
# In order to do that, we need to find the ground plane first.
ground_plane = panoptic_utils.get_ground_plane(sample_name)
filtered_points = panoptic_utils.filter_bev_points(
point_cloud, ground_plane)
# if len(filtered_points) > 0:
# print('point_cloud =', point_cloud)
# print('filtered_points =', filtered_points)
# Now, filtered_points is transposed, so filtered_points should
# be Nx4
# Project the filtered pointclouds on 2D image. Now filtered
# pointclouds are already in camera coordinates.
point_2d = obj_panoptic_utils.project_points_on_2D_image(
img_idx, dataset.calib_dir, image_size, filtered_points)
draw_points(prop_2d_axes, point_2d, 'red', pt_size=4)
# TODO: Project the filtered pointclouds on BEV image. Now filtered
# pointclouds are already in camera coordinates.
# point_bev = obj_panoptic_utils.project_points_on_BEV_image(img_idx,
# dataset.calib_dir,
# image_size,
# filtered_points)
# draw_points(prop_bev_axes, point_bev, 'red', pt_size=4)
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.jpg'
print('Draw proposals_separate: ', filename)
# Now add the legends
# prop_bev_axes.legend(loc='best', shadow=True, fontsize=20)
# prop_2d_axes.legend(loc='best', shadow=True, fontsize=20)
# prop_3d_axes.legend(loc='upper right', shadow=True, fontsize=20)
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
# print('prediction_boxes_3d = ', prediction_boxes_3d)
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_panoptic_projector.project_to_image_space(
box_3d,
calib_p2,
truncate=True,
image_size=image_size,
discard_before_truncation=False,
distortion=distortion)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_panoptic_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_bev_axes, pred_2d_axes, pred_3d_axes = \
vis_panoptic_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.jpg'
plt.savefig(filename)
print('Draw empty_images: ', filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
prop_2d_axes,
prop_3d_axes,
prop_bev_axes,
panoptic_utils.area_extents,
bev_img.shape,
draw_prediction_score,
draw_prediction_iou,
gt_classes,
draw_orientations_on_pred,
distortion=distortion)
filename = overlaid_out_dir + '/' + sample_name + '.jpg'
# Now add the legends
# prop_bev_axes.legend(loc='best', shadow=True, fontsize=20)
# prop_2d_axes.legend(loc='best', shadow=True, fontsize=20)
# prop_3d_axes.legend(loc='upper right', shadow=True, fontsize=20)
plt.savefig(filename)
print('Draw overlaid: ', filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_bev_axes, pred_2d_axes, pred_3d_axes = \
vis_panoptic_utils.visualization(dataset.rgb_image_dir,
img_idx,
bev_img,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_2d_axes,
pred_3d_axes,
pred_bev_axes,
panoptic_utils.area_extents,
bev_img.shape,
draw_prediction_score,
draw_prediction_iou,
gt_classes,
draw_orientations_on_pred,
distortion=distortion)
# Now add the legends
# pred_bev_axes.legend(loc='best', shadow=True, fontsize=20)
# pred_2d_axes.legend(loc='best', shadow=True, fontsize=20)
# pred_3d_axes.legend(loc='best', shadow=True, fontsize=20)
else:
pred_fig, pred_3d_axes = \
vis_panoptic_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_3d_axes,
draw_prediction_score,
draw_prediction_iou,
gt_classes,
draw_orientations_on_pred,
distortion=distortion)
# Now add the legends
# pred_3d_axes.legend(loc='upper right', shadow=True, fontsize=20)
filename = pred_out_dir + '/' + sample_name + '.jpg'
plt.savefig(filename)
print('Draw predictions_separate: ', filename)
plt.close(pred_fig)
print('\nDone')
def evaluate(model_config, eval_config, dataset_config):
# Parse eval config
eval_mode = eval_config.eval_mode
if eval_mode not in ['val', 'test']:
raise ValueError('Evaluation mode can only be set to `val` or `test`')
evaluate_repeatedly = eval_config.evaluate_repeatedly
# Parse dataset config
data_split = dataset_config.data_split
if data_split == 'train':
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = True
elif data_split.startswith('val'):
dataset_config.data_split_dir = 'training'
# Don't load labels for val split when running in test mode
if eval_mode == 'val':
dataset_config.has_labels = True
elif eval_mode == 'test':
dataset_config.has_labels = False
elif data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
else:
raise ValueError('Invalid data split', data_split)
# Convert to object to overwrite repeated fields
dataset_config = config_builder.proto_to_obj(dataset_config)
# Remove augmentation during evaluation
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_panoptic_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Convert to object to overwrite repeated fields
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
if evaluate_repeatedly:
model_evaluator.repeated_checkpoint_run()
else:
model_evaluator.run_latest_checkpoints()