def copy_kitti_native_code(checkpoint_name):
"""Copies and compiles kitti native code.
It also creates neccessary directories for storing the results
of the kitti native evaluation code.
"""
ammf_root_dir = ammf.root_dir()
kitti_native_code_copy = ammf_root_dir + '/data/outputs/' + \
checkpoint_name + '/predictions/kitti_native_eval/'
# Only copy if the code has not been already copied over
if not os.path.exists(kitti_native_code_copy):
os.makedirs(kitti_native_code_copy)
original_kitti_native_code = ammf.top_dir() + \
'/scripts/offline_eval/kitti_native_eval/'
predictions_dir = ammf_root_dir + '/data/outputs/' + \
checkpoint_name + '/predictions/'
# create dir for it first
dir_util.copy_tree(original_kitti_native_code, kitti_native_code_copy)
# run the script to compile the c++ code
script_folder = predictions_dir + \
'/kitti_native_eval/'
make_script = script_folder + 'run_make.sh'
subprocess.call([make_script, script_folder])
# Set up the results folders if they don't exist
results_dir = ammf.top_dir() + '/scripts/offline_eval/results'
results_05_dir = ammf.top_dir() + '/scripts/offline_eval/results_05_iou'
if not os.path.exists(results_dir):
os.makedirs(results_dir)
if not os.path.exists(results_05_dir):
os.makedirs(results_05_dir)
def __init__(self, dataset):
self._dataset = dataset
self.cluster_split = dataset.cluster_split
self.data_dir = ammf.root_dir() + "/data/label_clusters"
self.clusters = []
self.std_devs = []
def main(_):
parser = argparse.ArgumentParser()
# Defaults
default_pipeline_config_path = ammf.root_dir() + \
'/configs/pyramid_cars_example.config'
default_data_split = 'train'
default_device = '1'
parser.add_argument('--pipeline_config',
type=str,
dest='pipeline_config_path',
default=default_pipeline_config_path,
help='Path to the pipeline config')
parser.add_argument('--data_split',
type=str,
dest='data_split',
default=default_data_split,
help='Data split for training')
parser.add_argument('--device',
type=str,
dest='device',
default=default_device,
help='CUDA device id')
args = parser.parse_args()
# Parse pipeline config
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
args.pipeline_config_path, is_training=True)
# Overwrite data split
dataset_config.data_split = args.data_split
# Set CUDA device id
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
train(model_config, train_config, dataset_config)
def run_kitti_native_script(checkpoint_name, score_threshold, global_step):
"""Runs the kitti native code script."""
eval_script_dir = ammf.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
make_script = eval_script_dir + \
'/kitti_native_eval/run_eval.sh'
script_folder = eval_script_dir + \
'/kitti_native_eval/'
results_dir = ammf.top_dir() + '/scripts/offline_eval/results/'
# Round this because protobuf encodes default values as full decimal
score_threshold = round(score_threshold, 3)
subprocess.call([
make_script, script_folder,
str(score_threshold),
str(global_step),
str(checkpoint_name),
str(results_dir)
])
def main():
"""This demo shows p1 proposals and ammf predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
##############################
# Options
##############################
dataset_config.data_split = 'val' #bqx!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
fig_size = (10, 6.1)
p1_score_threshold = 0.8
ammf_score_threshold = 0.1
#gt_classes = ['Car']
gt_classes = ['Pedestrian', 'Cyclist']
# Overwrite this to select a specific checkpoint
global_step = None
#checkpoint_name = 'ammf_cars_example'
#checkpoint_name = 'pyramid_cars_with_aug_example'
checkpoint_name = 'people'
# Drawing Toggles
draw_proposals_separate = True
draw_overlaid = True
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = True
draw_orientations_on_pred = True
# Draw 2D bounding boxes
draw_projected_2d_boxes = True
# Save images for samples with no detections
save_empty_images = True
draw_score = True
draw_iou = True
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
# Setup Paths
predictions_dir = ammf.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir= predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
#steps = os.listdir()
#steps.sort(key=int)
#print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
#global_step = steps[-1]
global_step = '120000' #!!!!!!!!!!!!!!!!!!!!!!
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, p1_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, ammf_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step, ammf_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
for sample_idx in range(dataset.num_samples):
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s'.format(
sample_idx + 1, dataset.num_samples, avg_time,
est_time_left))
sys.stdout.flush()
#sample_idx=188
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
#img_idx = 188 #bqx!!!!!!!!!!!!!!!!!!!!111
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
if not os.path.exists(proposals_file_path):
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
score_mask = proposal_scores > p1_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
ammf_score_mask = prediction_scores >= ammf_score_threshold
prediction_boxes_3d = prediction_boxes_3d[ammf_score_mask]
prediction_scores = prediction_scores[ammf_score_mask]
prediction_class_indices = \
prediction_class_indices[ammf_score_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
if dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
boxes2d, _, _ = obj_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx)
calib_p2 = stereo_calib.p2
##############################
# Reformat and prepare to draw
##############################
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_encoder.box_3d_to_anchor(
proposal_boxes_3d)
proposal_boxes, _ = anchor_projector.project_to_image_space(
proposals_as_anchors, calib_p2, image_size)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_2d_axes, prop_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False)
draw_proposals(filtered_gt_objs, calib_p2, num_of_proposals,
proposal_objs, proposal_boxes, prop_2d_axes,
prop_3d_axes, draw_orientations_on_prop)
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
calib_p2,
truncate=True,
image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
prop_2d_axes, prop_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
filename = overlaid_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_2d_axes, pred_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
else:
pred_fig, pred_3d_axes = \
vis_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs, calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_3d_axes, draw_score, draw_iou,
gt_classes, draw_orientations_on_pred)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
print('\nDone')
def __init__(self, dataset):
self._dataset = dataset
self._mini_batch_sampler = \
balanced_positive_negative_sampler.BalancedPositiveNegativeSampler()
##############################
# Parse KittiUtils config
##############################
self.kitti_utils_config = dataset.config.kitti_utils_config
self._area_extents = self.kitti_utils_config.area_extents
self._anchor_strides = np.reshape(
self.kitti_utils_config.anchor_strides, (-1, 2))
##############################
# Parse MiniBatchUtils config
##############################
self.config = self.kitti_utils_config.mini_batch_config
self._density_threshold = self.config.density_threshold
# p1 mini batches
p1_config = self.config.p1_config
p1_iou_type = p1_config.WhichOneof('iou_type')
if p1_iou_type == 'iou_2d_thresholds':
self.p1_iou_type = '2d'
self.p1_iou_thresholds = p1_config.iou_2d_thresholds
elif p1_iou_type == 'iou_3d_thresholds':
self.p1_iou_type = '3d'
self.p1_iou_thresholds = p1_config.iou_3d_thresholds
self.p1_neg_iou_range = [
self.p1_iou_thresholds.neg_iou_lo,
self.p1_iou_thresholds.neg_iou_hi
]
self.p1_pos_iou_range = [
self.p1_iou_thresholds.pos_iou_lo,
self.p1_iou_thresholds.pos_iou_hi
]
self.p1_mini_batch_size = p1_config.mini_batch_size
# ammf mini batches
ammf_config = self.config.ammf_config
self.ammf_iou_type = '2d'
self.ammf_iou_thresholds = ammf_config.iou_2d_thresholds
self.ammf_neg_iou_range = [
self.ammf_iou_thresholds.neg_iou_lo,
self.ammf_iou_thresholds.neg_iou_hi
]
self.ammf_pos_iou_range = [
self.ammf_iou_thresholds.pos_iou_lo,
self.ammf_iou_thresholds.pos_iou_hi
]
self.ammf_mini_batch_size = ammf_config.mini_batch_size
# Setup paths
self.mini_batch_dir = ammf.root_dir() + '/data/mini_batches/' + \
'iou_{}/'.format(self.p1_iou_type) + \
dataset.name + '/' + dataset.cluster_split + '/' + \
dataset.bev_source
# Array column indices for saving to files
self.col_length = 9
self.col_anchor_indices = 0
self.col_ious = 1
self.col_offsets_lo = 2
self.col_offsets_hi = 8
self.col_class_idx = 8
def main():
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
##############################
# Options
##############################
checkpoint_name = 'ammf_cars_example'
data_split = 'val'
global_steps = None
# global_steps = [28000, 19000, 33000, 34000]
score_threshold = 0.1
save_2d = False # Save 2D predictions
save_3d = True # Save 2D and 3D predictions together
save_alphas = True # Save alphas (observation angles)
# Checkpoints below this are skipped
min_step = 20000
##############################
# End of Options
##############################
# Parse experiment config
pipeline_config_file = \
ammf.root_dir() + '/data/outputs/' + checkpoint_name + \
'/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
# Overwrite defaults
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.aug_list = []
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Get available prediction folders
predictions_root_dir = ammf.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
final_predictions_root_dir = predictions_root_dir + \
'/final_predictions_and_scores/' + dataset.data_split
print('Converting detections from', final_predictions_root_dir)
if not global_steps:
global_steps = os.listdir(final_predictions_root_dir)
global_steps.sort(key=int)
print('Checkpoints found ', global_steps)
for step_idx in range(len(global_steps)):
global_step = global_steps[step_idx]
# Skip first checkpoint
if int(global_step) < min_step:
continue
final_predictions_dir = final_predictions_root_dir + \
'/' + str(global_step)
# 2D and 3D prediction directories
kitti_predictions_2d_dir = predictions_root_dir + \
'/kitti_predictions_2d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
kitti_predictions_3d_dir = predictions_root_dir + \
'/kitti_predictions_3d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
if save_2d and not os.path.exists(kitti_predictions_2d_dir):
os.makedirs(kitti_predictions_2d_dir)
if save_3d and not os.path.exists(kitti_predictions_3d_dir):
os.makedirs(kitti_predictions_3d_dir)
# Do conversion
num_samples = dataset.num_samples
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting detections from:', final_predictions_dir)
if save_2d:
print('2D Detections saved to:', kitti_predictions_2d_dir)
if save_3d:
print('3D Detections saved to:', kitti_predictions_3d_dir)
for sample_idx in range(num_samples):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
prediction_file = sample_name + '.txt'
kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \
'/' + prediction_file
kitti_predictions_3d_file_path = kitti_predictions_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
# # Swap l, w for predictions where w > l
# swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
# fixed_predictions = np.copy(all_predictions)
# fixed_predictions[swapped_indices, 3] = all_predictions[
# swapped_indices, 4]
# fixed_predictions[swapped_indices, 4] = all_predictions[
# swapped_indices, 3]
score_filter = all_predictions[:, 7] >= score_threshold
all_predictions = all_predictions[score_filter]
# If no predictions, skip to next file
if len(all_predictions) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(dataset.get_rgb_image_path(sample_name))
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, stereo_calib_p2,
truncate=True, image_size=image.size)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
else:
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Get object types
all_pred_classes = all_predictions[:, 8].astype(np.int32)
obj_types = [dataset.classes[class_idx]
for class_idx in all_pred_classes]
# Truncation and Occlusion are always empty (see below)
# Alpha
if not save_alphas:
kitti_predictions[:, 3] = -10 * \
np.ones((len(kitti_predictions)), dtype=np.int32)
else:
alphas = all_predictions[:, 6] - \
np.arctan2(all_predictions[:, 0], all_predictions[:, 2])
kitti_predictions[:, 3] = alphas
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones((len(kitti_predictions), 2),
dtype=np.int32)
# Empty 3D (x, y, z)
kitti_empty_2 = -1 * np.ones((len(kitti_predictions), 3),
dtype=np.int32)
# Empty 3D (h, w, l)
kitti_empty_3 = -1000 * np.ones((len(kitti_predictions), 3),
dtype=np.int32)
# Empty 3D (ry)
kitti_empty_4 = -10 * np.ones((len(kitti_predictions), 1),
dtype=np.int32)
# Stack 2D predictions text
kitti_text_2d = np.column_stack([obj_types,
kitti_empty_1,
kitti_predictions[:, 3:8],
kitti_empty_2,
kitti_empty_3,
kitti_empty_4,
kitti_predictions[:, 15]])
# Stack 3D predictions text
kitti_text_3d = np.column_stack([obj_types,
kitti_empty_1,
kitti_predictions[:, 3:16]])
# Save to text files
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, kitti_text_2d,
newline='\r\n', fmt='%s')
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d,
newline='\r\n', fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
def main(dataset=None):
"""Generates anchors info which is used for mini batch sampling.
Processing on 'Cars' can be split into multiple processes, see the Options
section for configuration.
Args:
dataset: KittiDataset (optional)
If dataset is provided, only generate info for that dataset.
If no dataset provided, generates info for all 3 classes.
"""
if dataset is not None:
do_preprocessing(dataset, None)
return
car_dataset_config_path = ammf.root_dir() + \
'/configs/mb_preprocessing/p1_cars.config'
ped_dataset_config_path = ammf.root_dir() + \
'/configs/mb_preprocessing/p1_pedestrians.config'
cyc_dataset_config_path = ammf.root_dir() + \
'/configs/mb_preprocessing/p1_cyclists.config'
ppl_dataset_config_path = ammf.root_dir() + \
'/configs/mb_preprocessing/p1_people.config'
unittest_dataset_config_path = ammf.root_dir() + \
'/configs/unittest_model.config'
##############################
# Options
##############################
# Serial vs parallel processing
in_parallel = True
process_car = True # Cars
process_ped = False # Pedestrians
process_cyc = False # Cyclists
process_ppl = True # People (Pedestrians + Cyclists)
process_unittest = False
# Number of child processes to fork, samples will
# be divided evenly amongst the processes (in_parallel must be True)
num_car_children = 8
num_ped_children = 8
num_cyc_children = 8
num_ppl_children = 8
num_unittest_children = 8
##############################
# Dataset setup
##############################
if process_car:
car_dataset = DatasetBuilder.load_dataset_from_config(
car_dataset_config_path)
if process_ped:
ped_dataset = DatasetBuilder.load_dataset_from_config(
ped_dataset_config_path)
if process_cyc:
cyc_dataset = DatasetBuilder.load_dataset_from_config(
cyc_dataset_config_path)
if process_ppl:
ppl_dataset = DatasetBuilder.load_dataset_from_config(
ppl_dataset_config_path)
if process_unittest:
unittest_dataset = DatasetBuilder.load_dataset_from_config(
unittest_dataset_config_path)
##############################
# Serial Processing
##############################
if not in_parallel:
if process_car:
do_preprocessing(car_dataset, None)
if process_ped:
do_preprocessing(ped_dataset, None)
if process_cyc:
do_preprocessing(cyc_dataset, None)
if process_ppl:
do_preprocessing(ppl_dataset, None)
if process_unittest:
do_preprocessing(unittest_dataset, None)
print('All Done (Serial)')
##############################
# Parallel Processing
##############################
else:
# List of all child pids to wait on
all_child_pids = []
# Cars
if process_car:
car_indices_split = split_indices(car_dataset, num_car_children)
split_work(all_child_pids, car_dataset, car_indices_split,
num_car_children)
# Pedestrians
if process_ped:
ped_indices_split = split_indices(ped_dataset, num_ped_children)
split_work(all_child_pids, ped_dataset, ped_indices_split,
num_ped_children)
# Cyclists
if process_cyc:
cyc_indices_split = split_indices(cyc_dataset, num_cyc_children)
split_work(all_child_pids, cyc_dataset, cyc_indices_split,
num_cyc_children)
# People (Pedestrians + Cyclists)
if process_ppl:
ppl_indices_split = split_indices(ppl_dataset, num_ppl_children)
split_work(all_child_pids, ppl_dataset, ppl_indices_split,
num_ppl_children)
if process_unittest:
unittest_indices_split = split_indices(unittest_dataset,
num_unittest_children)
split_work(all_child_pids, unittest_dataset,
unittest_indices_split, num_unittest_children)
# Wait to child processes to finish
print('num children:', len(all_child_pids))
for i, child_pid in enumerate(all_child_pids):
os.waitpid(child_pid, 0)
print('All Done (Parallel)')
def get_configs_from_pipeline_file(pipeline_config_path,
is_training):
"""Reads model configuration from a pipeline_pb2.NetworkPipelineConfig.
Args:
pipeline_config_path: A path directory to the network pipeline config
is_training: A boolean flag to indicate training stage, used for
creating the checkpoint directory which must be created at the
first training iteration.
Returns:
model_config: A model_pb2.ModelConfig
train_config: A train_pb2.TrainConfig
eval_config: A eval_pb2.EvalConfig
dataset_config: A kitti_dataset_pb2.KittiDatasetConfig
"""
pipeline_config = pipeline_pb2.NetworkPipelineConfig()
with open(pipeline_config_path, 'r') as f:
text_format.Merge(f.read(), pipeline_config)
model_config = pipeline_config.model_config
# Make sure the checkpoint name matches the config filename
config_file_name = \
os.path.split(pipeline_config_path)[1].split('.')[0]
checkpoint_name = model_config.checkpoint_name
if config_file_name != checkpoint_name:
raise ValueError('Config and checkpoint names must match.')
output_root_dir = ammf.root_dir() + '/data/outputs/' + checkpoint_name
# Construct paths
paths_config = model_config.paths_config
if not paths_config.checkpoint_dir:
checkpoint_dir = output_root_dir + '/checkpoints'
if is_training:
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
paths_config.checkpoint_dir = checkpoint_dir
if not paths_config.logdir:
paths_config.logdir = output_root_dir + '/logs/'
if not paths_config.pred_dir:
paths_config.pred_dir = output_root_dir + '/predictions'
train_config = pipeline_config.train_config
eval_config = pipeline_config.eval_config
dataset_config = pipeline_config.dataset_config
if is_training:
# Copy the config to the experiments folder
experiment_config_path = output_root_dir + '/' +\
model_config.checkpoint_name
experiment_config_path += '.config'
# Copy this even if the config exists, in case some parameters
# were modified
shutil.copy(pipeline_config_path, experiment_config_path)
return model_config, train_config, eval_config, dataset_config
def save_predictions_in_kitti_format(model, checkpoint_name, data_split,
score_threshold, global_step):
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
dataset = model.dataset
# Round this because protobuf encodes default values as full decimal
score_threshold = round(score_threshold, 3)
# Get available prediction folders
predictions_root_dir = ammf.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
final_predictions_root_dir = predictions_root_dir + \
'/final_predictions_and_scores/' + dataset.data_split
final_predictions_dir = final_predictions_root_dir + \
'/' + str(global_step)
# 3D prediction directories
kitti_predictions_3d_dir = predictions_root_dir + \
'/kitti_native_eval/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
if not os.path.exists(kitti_predictions_3d_dir):
os.makedirs(kitti_predictions_3d_dir)
# Do conversion
num_samples = dataset.num_samples
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting detections from:', final_predictions_dir)
print('3D Detections being saved to:', kitti_predictions_3d_dir)
for sample_idx in range(num_samples):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(sample_idx + 1,
num_samples))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
prediction_file = sample_name + '.txt'
kitti_predictions_3d_file_path = kitti_predictions_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
# # Swap l, w for predictions where w > l
#swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
#fixed_predictions = np.copy(all_predictions)
#fixed_predictions[swapped_indices, 3] = all_predictions[
# swapped_indices, 4]
#fixed_predictions[swapped_indices, 4] = all_predictions[
# swapped_indices, 3]
score_filter = all_predictions[:, 7] >= score_threshold
all_predictions = all_predictions[score_filter]
# If no predictions, skip to next file
if len(all_predictions) == 0:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(dataset.get_rgb_image_path(sample_name))
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, stereo_calib_p2, truncate=True, image_size=image.size)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
continue
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Get object types
all_pred_classes = all_predictions[:, 8].astype(np.int32)
obj_types = [
dataset.classes[class_idx] for class_idx in all_pred_classes
]
# Truncation and Occlusion are always empty (see below)
# Alpha (Not computed)
kitti_predictions[:, 3] = -10 * np.ones(
(len(kitti_predictions)), dtype=np.int32)
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones(
(len(kitti_predictions), 2), dtype=np.int32)
# Stack 3D predictions text
kitti_text_3d = np.column_stack(
[obj_types, kitti_empty_1, kitti_predictions[:, 3:16]])
# Save to text files
np.savetxt(kitti_predictions_3d_file_path,
kitti_text_3d,
newline='\r\n',
fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
class DatasetBuilder(object):
"""
Static class to return preconfigured dataset objects
"""
#1.unittest
KITTI_UNITTEST = KittiDatasetConfig(
name="unittest-kitti",
dataset_dir=ammf.root_dir() + "/tests/datasets/Kitti/object",
data_split="train",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car", "Pedestrian", "Cyclist"],
num_clusters=[2, 1, 1],
)
#2.train
KITTI_TRAIN = KittiDatasetConfig(name="kitti",
data_split="train",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2])
#3.val
KITTI_VAL = KittiDatasetConfig(
name="kitti",
data_split="val",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
#4.test
KITTI_TEST = KittiDatasetConfig(
name="kitti",
data_split="test",
data_split_dir="testing",
has_labels=False,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
#5.trainval
KITTI_TRAINVAL = KittiDatasetConfig(
name="kitti",
data_split="trainval",
data_split_dir="training",
has_labels=True,
cluster_split="trainval",
classes=["Car"],
num_clusters=[2],
)
#6.train_mini
KITTI_TRAIN_MINI = KittiDatasetConfig(
name="kitti",
data_split="train_mini",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
#7.val_mini
KITTI_VAL_MINI = KittiDatasetConfig(
name="kitti",
data_split="val_mini",
data_split_dir="training",
has_labels=True,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
#8.test_mini
KITTI_TEST_MINI = KittiDatasetConfig(
name="kitti",
data_split="test_mini",
data_split_dir="testing",
has_labels=False,
cluster_split="train",
classes=["Car"],
num_clusters=[2],
)
#9.CONFIG_DEFAULTS_PROTO
CONFIG_DEFAULTS_PROTO = \
"""
bev_source: 'lidar'
kitti_utils_config {
area_extents: [-40, 40, -5, 3, 0, 70]
voxel_size: 0.1
anchor_strides: [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]
bev_generator {
slices {
height_lo: -0.2
height_hi: 2.3
num_slices: 5
}
}
mini_batch_config {
density_threshold: 1
p1_config {
iou_2d_thresholds {
neg_iou_lo: 0.0
neg_iou_hi: 0.3
pos_iou_lo: 0.5
pos_iou_hi: 1.0
}
# iou_3d_thresholds {
# neg_iou_lo: 0.0
# neg_iou_hi: 0.005
# pos_iou_lo: 0.1
# pos_iou_hi: 1.0
# }
mini_batch_size: 512
}
ammf_config {
iou_2d_thresholds {
neg_iou_lo: 0.0
neg_iou_hi: 0.55
pos_iou_lo: 0.65
pos_iou_hi: 1.0
}
mini_batch_size: 1024
}
}
}
"""
#9.1
@staticmethod
def load_dataset_from_config(dataset_config_path):
dataset_config = kitti_dataset_pb2.KittiDatasetConfig()
with open(dataset_config_path, 'r') as f:
text_format.Merge(f.read(), dataset_config)
return DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
#9.2
@staticmethod
def copy_config(cfg):
return deepcopy(cfg)
#9.3
@staticmethod
def merge_defaults(cfg):
cfg_copy = DatasetBuilder.copy_config(cfg)
text_format.Merge(DatasetBuilder.CONFIG_DEFAULTS_PROTO, cfg_copy)
return cfg_copy
#9.4
@staticmethod
def build_kitti_dataset(base_cfg,
use_defaults=True,
new_cfg=None) -> KittiDataset:
"""Builds a KittiDataset object using the provided configurations
Args:
base_cfg: a base dataset configuration
use_defaults: whether to use the default config values
new_cfg: (optional) a custom dataset configuration, no default
values will be used, all config values must be provided
Returns:
KittiDataset object
"""
cfg_copy = DatasetBuilder.copy_config(base_cfg)
if use_defaults:
# Use default values
text_format.Merge(DatasetBuilder.CONFIG_DEFAULTS_PROTO, cfg_copy)
if new_cfg:
# Use new config values if provided
cfg_copy.MergeFrom(new_cfg)
return KittiDataset(cfg_copy)