def main(root):
cask = Cask(root)
demo_list_channels(cask)
demo_json_access(cask, "imu_raw")
demo_capnproto_access(cask, "imu_raw")
demo_numpy_access(cask, "segway_state")
demo_numpy_access(cask, "state")
demo_json_access(cask, "fullscan")
demo_buffer_access(cask, "fullscan")
demo_numpy_access(cask, "fullscan")
demo_load_perf(cask, "color")
demo_image(cask, "color")
demo_rgbd_stream(cask, "color", "depth")
def load_log(root):
"""
Loads a cask for a given root.
Note: a root may contain a Cask log file or a single arbitraty
named UUID subfolder containig a single Cask log file
"""
log_paths = glob.glob(os.path.join(root, '**', 'kv'), recursive=True)
if not log_paths:
logging.error("Log path %s doesn't contain any Isaac logs." % root)
return None
if len(log_paths) > 1:
logging.error(
"Log path %s should only contain a single Isaac log."
"Please remove these logs and re-run the processing stage." % root)
return None
return Cask(log_paths[0][:-3])
def create_composite_atlas(cask_root, joints):
'''Creates composite atlas cask with waypoints for ur10.'''
if len(joints) != 6:
raise ValueError("UR10 should have 6 joints, got {}".format(len(joints)))
cask = Cask(cask_root, writable=True)
# at joint waypoints
quantities = [[x, "position", 1] for x in joints]
CART_OBSERVE_WAYPOINT = np.array(
[3.2334, -1.4856, 1.7979, 4.8503, -1.5219, 0.5585], dtype=np.dtype("float64"))
CART_ALIGN_WAYPOINT = np.array(
[3.3922, -0.8047, 1.7275, 3.7742, -1.5163, 1.8773], dtype=np.dtype("float64"))
CART_DROPOFF_WAYPOINT = np.array(
[3.3942, -0.6317, 1.6605, 3.6562, -1.5213, 1.8773], dtype=np.dtype("float64"))
DOLLY_OBSERVE_WAYPOINT = np.array(
[6.0535, -1.0559, 1.2437, 5.2903, -1.7218, 0.3887], dtype=np.dtype("float64"))
DOLLY_ALIGN_WAYPOINT = np.array(
[5.8864, -0.5826, 1.5502, 3.8685, -1.5238, 1.1526], dtype=np.dtype("float64"))
DOLLY_DROPOFF_WAYPOINT = np.array(
[5.8894, -0.3856, 1.4512, 3.7055, -1.5238, 1.1526], dtype=np.dtype("float64"))
cask.write_message(create_composite_waypoint("cart_observe", quantities, CART_OBSERVE_WAYPOINT))
cask.write_message(create_composite_waypoint("cart_align", quantities, CART_ALIGN_WAYPOINT))
cask.write_message(create_composite_waypoint("cart_dropoff", quantities, CART_DROPOFF_WAYPOINT))
cask.write_message(
create_composite_waypoint("dolly_observe", quantities, DOLLY_OBSERVE_WAYPOINT))
cask.write_message(create_composite_waypoint("dolly_align", quantities, DOLLY_ALIGN_WAYPOINT))
cask.write_message(
create_composite_waypoint("dolly_dropoff", quantities, DOLLY_DROPOFF_WAYPOINT))
quantities = [[x, "none", 1] for x in ["pump", "valve", "gripper"]]
SUCTION_ON_WAYPOINT = np.array([1.0, 0.0, 1.0], dtype=np.dtype("float64"))
SUCTION_OFF_WAYPOINT = np.array([0.0, 1.0, 0.0], dtype=np.dtype("float64"))
VALVE_OFF_WAYPOINT = np.array([0.0, 0.0, 0.0], dtype=np.dtype("float64"))
cask.write_message(create_composite_waypoint("suction_on", quantities, SUCTION_ON_WAYPOINT))
cask.write_message(create_composite_waypoint("suction_off", quantities, SUCTION_OFF_WAYPOINT))
cask.write_message(create_composite_waypoint("valve_off", quantities, VALVE_OFF_WAYPOINT))
def main(args):
# Create uuid
eval_uuid = uuid.uuid4()
# ----------------------- LOAD CONFIG --------------------------------------
# load evaluation config
config = {}
config_path = os.fspath(Path(args.config).resolve())
with open(config_path) as f:
config = json.load(f)
# read cask paths
image_cask_dir_path = args.image_cask_dir
gt_cask_dir_path = args.gt_cask_dir
pred_cask_dir_path = args.pred_cask_dir
results_dir = args.results_dir
image_cask_list = multi_cask_processing.load_roots(image_cask_dir_path, "")
gt_cask_list = multi_cask_processing.load_roots(gt_cask_dir_path, "")
pred_cask_list = multi_cask_processing.load_roots(pred_cask_dir_path, "")
pred_cask_root_list = [os.path.split(root)[0] for root in pred_cask_list]
gt_image_cask_uuids = get_image_cask_uuid_raci_metadata(gt_cask_list)
pred_image_cask_uuids = get_image_cask_uuid_raci_metadata(
pred_cask_root_list)
# Image, ground truth, and prediction data for full dataset
images = []
gt_detections = []
pred_detections = []
# load other evaluation config params
iou_thresholds = config['iou_thresholds']
labels = config['labels']
num_labels = len(labels)
num_thresholds = len(iou_thresholds)
# ----------------------- READ DATASET FROM SET OF CASKS ---------------------
for image_cask_path in image_cask_list:
_, image_cask_tail = os.path.split(image_cask_path)
gt_cask_idx = None
pred_cask_idx = None
try:
# get associated gt cask and pred casks
gt_cask_idx = gt_image_cask_uuids.index(image_cask_tail)
pred_cask_idx = pred_image_cask_uuids.index(image_cask_tail)
except (ValueError):
print("Associated ground truth or predicted cask is not \
available for image cask {}".format(image_cask_path))
continue
gt_cask_path = gt_cask_list[gt_cask_idx]
pred_cask_path = pred_cask_list[pred_cask_idx]
#read image cask
image_cask = Cask(image_cask_path)
images_channel = image_cask[config['image_channel']]
#read ground truth cask
gt_cask = Cask(gt_cask_path)
gt_detections_channel = gt_cask[config['gt_channel']]
# read predictions cask
pred_cask = Cask(pred_cask_path)
pred_detections_channel = pred_cask[config['pred_channel']]
# Sync image, gt detection message, and predicted detection message channels by acqtime
prev_image_idx = 0
prev_gt_idx = 0
for prediction in pred_detections_channel:
image_idx = get_message_idx(images_channel, prediction.acqtime,
prev_image_idx)
gt_idx = get_message_idx(gt_detections_channel, prediction.acqtime,
prev_gt_idx)
if (image_idx != -1 and gt_idx != -1):
images.append(images_channel[image_idx])
gt_detections.append(gt_detections_channel[gt_idx])
pred_detections.append(prediction)
prev_image_idx = image_idx
prev_gt_idx = gt_idx
num_samples = len(
images) #also same as len(gt_detections) and len(pred_detections)
# ----------------------- COMPUTE METRICS --------------------------------------
# use bboxes matched with specified IOU threshold to determine outliers
outlier_iou_area_threshold = config['outlier_iou_area_threshold']
# matches with IOU under the area min have large bbox error
large_bbox_iou_area_min = config['large_bbox_iou_area_min']
# Lists to collect outlier indices
large_bbox_outliers_idxs = []
false_positive_idxs = []
false_negative_idxs = []
# Lists to collect (and later average) area/height/width IOUs
all_ious_area = []
all_ious_height = []
all_ious_width = []
# Compute confusion matrices: one per IOU threshold
confusion_matrices = {
iou_threshold: np.empty((0))
for iou_threshold in iou_thresholds
}
for idx in range(num_samples):
gt_bboxes = gt_detections[idx].proto.boundingBoxes
gt_predictions = gt_detections[idx].proto.predictions
pred_bboxes = pred_detections[idx].proto.boundingBoxes
pred_predictions = pred_detections[idx].proto.predictions
# match gt bboxes with predicted bboxes for each IOU
for iou_threshold in iou_thresholds:
gt_matched_idxs, pred_matched_idxs, matched_ious_width, matched_ious_height, \
matched_ious_area = match_bboxes(gt_bboxes, pred_bboxes, iou_threshold)
all_ious_width += matched_ious_width
all_ious_height += matched_ious_height
all_ious_area += matched_ious_area
# collect outliers with large bbox error
if iou_threshold == outlier_iou_area_threshold and not all(
i > large_bbox_iou_area_min for i in matched_ious_area):
large_bbox_outliers_idxs.append(idx)
# compute confusion matrices
confusion_matrix_sample = matches_to_confusion_matrix(
gt_matched_idxs, pred_matched_idxs, gt_predictions,
pred_predictions, labels)
# collect false positive idxs
if iou_threshold == outlier_iou_area_threshold and len(
pred_bboxes) != len(pred_matched_idxs):
false_positive_idxs.append(idx)
# collect false negative idxs
if iou_threshold == outlier_iou_area_threshold and len(
gt_bboxes) != len(gt_matched_idxs):
false_negative_idxs.append(idx)
# accumulate confusion matrices across samples
if not confusion_matrices[iou_threshold].any():
confusion_matrices[iou_threshold] = confusion_matrix_sample
else:
confusion_matrices[iou_threshold] += confusion_matrix_sample
# Compute precision and recall metrics for each class and each IoU from the confusion matrix
per_class_precisions = np.zeros((num_labels, num_thresholds))
per_class_recalls = np.zeros((num_labels, num_thresholds))
for iou_idx in range(num_thresholds):
confusion_matrix = confusion_matrices[iou_thresholds[iou_idx]]
precisions = np.diagonal(confusion_matrix) / np.sum(confusion_matrix,
axis=0)
recalls = np.diagonal(confusion_matrix) / np.sum(confusion_matrix,
axis=1)
per_class_precisions[:, iou_idx] = precisions[:-1]
per_class_recalls[:, iou_idx] = recalls[:-1]
# compute average IOUs for area as well as height and width dimensions
mean_iou_width = sum(all_ious_width) / len(all_ious_width)
mean_iou_height = sum(all_ious_height) / len(all_ious_height)
mean_iou_area = sum(all_ious_area) / len(all_ious_area)
# ----------------------- COMPUTE KPIS FROM METRICS ---------------------------
# COCO 2017 challenge mAP is computed over all IOUs over all classes
# https://cocodataset.org/#detection-eval
KPI_mAP = np.mean(per_class_precisions)
KPI_mAR = np.mean(per_class_recalls)
# PASCAL VOC2007 challenge mAP is computed =over a single IOU over all classes
# http://host.robots.ox.ac.uk/pascal/VOC/
mAP_per_IOU = np.mean(per_class_precisions, axis=0)
mAR_per_IOU = np.mean(per_class_recalls, axis=0)
KPI_mAP_lowest_IOU = mAP_per_IOU[0]
KPI_mAR_lowest_IOU = mAR_per_IOU[0]
# Check if the KPIs meet the minimum thresholds and determine pass/fail
kpi_pass = (KPI_mAP >= config["KPI_threshold_mAP"] and KPI_mAR >= config["KPI_threshold_mAR"] \
and KPI_mAP_lowest_IOU >= config["KPI_threshold_mAP_lowest_IOU"] \
and KPI_mAR_lowest_IOU >= config["KPI_threshold_mAR_lowest_IOU"])
# ----------------------- OUTPUT ALL RESULTS --------------------------------------
evaluation_report = config
evaluation_report["date"] = str(
datetime.datetime.now().strftime("%Y-%m-%d"))
# Construct JSON output
results = {}
results["num_frames"] = num_samples
# Write object detection evaluation metrics
statistics = []
for class_idx in range(num_labels):
per_class_stats = {}
per_class_stats["class_name"] = labels[class_idx]
per_class_stats["precisions"] = per_class_precisions[
class_idx, :].tolist()
per_class_stats["recalls"] = per_class_recalls[class_idx, :].tolist()
statistics.append(per_class_stats)
results["per_class_stats"] = statistics
# Write confusion matrices and mean IOUs
conf_matrices_stats = []
for iou_idx in range(num_thresholds):
conf_matrix = {}
conf_matrix["confusion_matrix"] = confusion_matrices[
iou_thresholds[iou_idx]].tolist()
if iou_thresholds[iou_idx] == outlier_iou_area_threshold:
conf_matrix["mean_iou_width"] = mean_iou_width
conf_matrix["mean_iou_height"] = mean_iou_height
conf_matrix["mean_iou_area"] = mean_iou_area
conf_matrices_stats.append(conf_matrix)
results["confusion_matrices"] = conf_matrices_stats
# Write outlier indices
outliers = {}
outliers["large_bbox_error_idxs"] = large_bbox_outliers_idxs
outliers["false_positive_idxs"] = false_positive_idxs
outliers["false_negative_idxs"] = false_negative_idxs
results["outliers"] = outliers
# Write results and KPIs
evaluation_report["results"] = results
evaluation_report["KPI_mAP"] = KPI_mAP
evaluation_report["KPI_mAR"] = KPI_mAR
evaluation_report["KPI_mAP_lowest_IOU"] = KPI_mAP_lowest_IOU
evaluation_report["KPI_mAR_lowest_IOU"] = KPI_mAR_lowest_IOU
evaluation_report["KPI_pass"] = bool(kpi_pass)
# Write the JSON to file
if not os.path.exists(results_dir):
os.makedirs(results_dir)
json_output_path = os.path.join(
results_dir, "object_detection_evaluation_" + str(eval_uuid) + ".json")
with open(json_output_path, 'w') as f:
json.dump(evaluation_report, f, indent=2)
# ----------------------- VISUALIZE OUTLIERS --------------------------------------
if args.save_outliers:
large_bbox_outliers_dir = os.path.join(results_dir,
"large_bbox_error_outliers")
if not os.path.exists(large_bbox_outliers_dir):
os.makedirs(large_bbox_outliers_dir)
for outlier_idx in large_bbox_outliers_idxs:
save_sample(outlier_idx, images[outlier_idx], gt_detections[outlier_idx], \
pred_detections[outlier_idx], "large bbox error outlier", large_bbox_outliers_dir)
false_positives_dir = os.path.join(results_dir, "false_positives")
if not os.path.exists(false_positives_dir):
os.makedirs(false_positives_dir)
for outlier_idx in false_positive_idxs:
save_sample(outlier_idx, images[outlier_idx], gt_detections[outlier_idx], \
pred_detections[outlier_idx], "false positive outlier", false_positives_dir)
false_negatives_dir = os.path.join(results_dir, "false_negatives")
if not os.path.exists(false_negatives_dir):
os.makedirs(false_negatives_dir)
for outlier_idx in false_negative_idxs:
save_sample(outlier_idx, images[outlier_idx], gt_detections[outlier_idx], \
pred_detections[outlier_idx], "false negative outlier", false_negatives_dir)
print("Evalulation complete. Results saved to: " + json_output_path)
def create_composite_atlas_ur10(cask_root, joints):
'''Creates composite atlas cask with waypoints for ur10. Tested with ovkit sim.'''
if len(joints) != 6:
raise ValueError("UR10 should have 6 joints, got {}".format(
len(joints)))
cask = Cask(cask_root, writable=True)
# joint waypoints
quantities = [[x, "position", 1] for x in joints]
HOME_POSE_WAYPOINT = np.array(
[1.3504, -1.4784, 1.6887, -1.7811, -1.5708, 1.3488],
dtype=np.dtype("float64"))
VIEW_POSE_WAYPOINT = np.array(
[2.1358, -1.4784, 1.6887, -1.7811, -1.5708, 0.5635],
dtype=np.dtype("float64"))
APPROACH_POSE_WAYPOINT = np.array(
[-0.2966, -1.062, 1.251, -1.38, -1.716, 0.217],
dtype=np.dtype("float64"))
cask.write_message(
pick_and_place.create_composite_waypoint("home_pose", quantities,
HOME_POSE_WAYPOINT))
cask.write_message(
pick_and_place.create_composite_waypoint("view_pose", quantities,
VIEW_POSE_WAYPOINT))
cask.write_message(
pick_and_place.create_composite_waypoint("approach_pose", quantities,
APPROACH_POSE_WAYPOINT))
# gripper waypoints
quantities = [[x, "none", 1] for x in ["pump", "valve", "gripper"]]
SUCTION_ON_WAYPOINT = np.array([1.0, 0.0, 1.0], dtype=np.dtype("float64"))
SUCTION_OFF_WAYPOINT = np.array([0.0, 1.0, 0.0], dtype=np.dtype("float64"))
VALVE_OFF_WAYPOINT = np.array([0.0, 0.0, 0.0], dtype=np.dtype("float64"))
cask.write_message(
pick_and_place.create_composite_waypoint("suction_on", quantities,
SUCTION_ON_WAYPOINT))
cask.write_message(
pick_and_place.create_composite_waypoint("suction_off", quantities,
SUCTION_OFF_WAYPOINT))
cask.write_message(
pick_and_place.create_composite_waypoint("valve_off", quantities,
VALVE_OFF_WAYPOINT))
dts = []
for i in range(len(channel) - 1):
dt_ns = channel[i + 1].pubtime - channel[i].pubtime
dts.append(dt_ns / 1000000.0)
return dts
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Time difference historgram for cask channel')
parser.add_argument('--root',
dest='root',
help='The directory in which log files will be stored')
parser.add_argument('--channel',
dest='channel',
default='color',
help='The channel for which to show time differences')
args = parser.parse_args()
cask = Cask(args.root)
dts = channel_dts(cask[args.channel])
print(numpy.histogram(dts))
plt.title(args.channel)
plt.hist(dts, bins='auto')
plt.ylabel("Number of messages")
plt.xlabel("Time delta between messages (ms)")
plt.show()
def create_composite_atlas_franka(cask_root, joints):
'''Creates composite atlas cask with waypoints for franka. Tested with ovkit sim.'''
if len(joints) != 7:
raise ValueError("Franka should have 7 joints, got {}".format(
len(joints)))
cask = Cask(cask_root, writable=True)
# joint waypoints
quantities = [[x, "position", 1] for x in joints]
HOME_POSE_WAYPOINT = np.array(
[-0.6392, -0.3524, 0.5867, -2.3106, 0.2122, 2.0046, 0.6137],
dtype=np.dtype("float64"))
cask.write_message(
create_composite_waypoint("home_pose", quantities, HOME_POSE_WAYPOINT))
# The home pose is re-used as view and approach pose to be consistent with the waypoints used by
# the UR10.
cask.write_message(
create_composite_waypoint("view_pose", quantities, HOME_POSE_WAYPOINT))
cask.write_message(
create_composite_waypoint("approach_pose", quantities,
HOME_POSE_WAYPOINT))
# gripper waypoints
quantities = [[x, "none", 1] for x in ["gripper"]]
GRIPPER_OPEN_WAYPOINT = np.array([0.0], dtype=np.dtype("float64"))
GRIPPER_CLOSE_WAYPOINT = np.array([1.0], dtype=np.dtype("float64"))
cask.write_message(
create_composite_waypoint("gripper_open", quantities,
GRIPPER_OPEN_WAYPOINT))
cask.write_message(
create_composite_waypoint("gripper_close", quantities,
GRIPPER_CLOSE_WAYPOINT))
def main(args):
""" The entry point of the application.
The function reads the json file containing the ground truth and predicted pose labels,
computes evaluation metrics and save the plots of translationa and rotation errors.
"""
# Create uuid
eval_uuid = uuid.uuid4()
# ----------------------- LOAD CONFIG --------------------------------------
# load evaluation config
config = {}
config_path = os.fspath(Path(args.config).resolve())
with open(config_path) as f:
config = json.load(f)
# read cask paths
image_cask_dir_path = args.image_cask_dir
gt_cask_dir_path = args.gt_cask_dir
pred_cask_dir_path = args.predicted_cask_dir
image_cask_list = multi_cask_processing.load_roots(image_cask_dir_path, "")
gt_poses_cask_list = multi_cask_processing.load_roots(gt_cask_dir_path, "")
predicted_poses_cask_list = multi_cask_processing.load_roots(pred_cask_dir_path, "")
pred_cask_root_list = [os.path.split(root)[0] for root in predicted_poses_cask_list]
gt_cask_root_list = [os.path.split(root)[0] for root in gt_poses_cask_list]
pred_image_cask_uuids = get_image_cask_uuid_raci_metadata(pred_cask_root_list)
assert (pred_image_cask_uuids
is not None), "Unable to find the metadata files for prediction casks."
# Check for image cask uuids in the ground truth cask list (for simulation) or
# roots of the cask list (for real data ground truth casks)
gt_image_cask_uuids = get_image_cask_uuid_raci_metadata(gt_poses_cask_list)
if (gt_image_cask_uuids is None):
gt_image_cask_uuids = get_image_cask_uuid_raci_metadata(gt_cask_root_list)
assert (gt_image_cask_uuids
is not None), "Unable to find the metadata files for ground truth casks."
# List of predicted poses
images = []
predicted_poses = []
gt_poses = []
predicted_detections = []
# Read all the json objects into a string
if (not (len(predicted_poses_cask_list) == len(gt_poses_cask_list)
or not (len(predicted_poses_cask_list) == len(image_cask_list)))):
raiseError("Length of image, ground truth and predicted labels list must be equal")
# ------------------------- AGGREGATE IMAGE, PREDICTIONS, LABELS DATA --------------------
for image_cask_path in image_cask_list:
_, image_cask_tail = os.path.split(image_cask_path)
try:
# get associated gt cask and pred casks
gt_cask_idx = gt_image_cask_uuids.index(image_cask_tail)
pred_cask_idx = pred_image_cask_uuids.index(image_cask_tail)
except (ValueError):
print("Associated ground truth or predicted cask is not \
available for image cask {}".format(image_cask_path))
continue
gt_cask_path = gt_poses_cask_list[gt_cask_idx]
pred_cask_path = predicted_poses_cask_list[pred_cask_idx]
#read image cask
images_channel = Cask(image_cask_path)[config['image_channel']]
#read ground truth cask
gt_poses_channel = Cask(gt_cask_path)[config['gt_pose_channel']]
# read predictions cask
pred_poses_channel = Cask(pred_cask_path)[config['pred_pose_channel']]
if (args.use_2d_detections):
# read predcited detections cask if the channel is not empty
pred_detections_channel = Cask(pred_cask_path)[config['pred_detection_channel']]
# Reset indices before synching casks
prev_image_idx = 0
prev_gt_pose_idx = 0
prev_pred_detection_idx = 0
# ----------------------- SYNC CASKS --------------------------------------
# Sync image, gt pose message, and predicted pose message channels by acqtime
for prediction_pose in pred_poses_channel:
#TODO:[Sravya] - Extend the evaluation to multiple objects in an image scenario
if (len(prediction_pose.proto.poses) == 0 or len(prediction_pose.proto.poses) > 1):
continue
acqtime = prediction_pose.acqtime
gt_pose_idx = get_message_idx(gt_poses_channel, acqtime, prev_gt_pose_idx)
image_idx = get_message_idx(images_channel, acqtime, prev_image_idx)
# Skip the message if the matching timestamp is not found in either casks
if (image_idx != -1 and gt_pose_idx != -1):
if (len(gt_poses_channel[gt_pose_idx].proto.poses) == 0):
continue
images.append(images_channel[image_idx].tensor)
gt_poses.append(pose_capnp_to_list(gt_poses_channel[gt_pose_idx].proto.poses[0]))
predicted_poses.append(pose_capnp_to_list(prediction_pose.proto.poses[0]))
prev_image_idx = image_idx
prev_gt_pose_idx = gt_pose_idx
if (args.use_2d_detections):
detection_idx = get_message_idx(pred_detections_channel, acqtime,
prev_pred_detection_idx)
prev_pred_detection_idx = detection_idx
# If detection sample is missing, add a zero size bbox instead of
# ignoring the sample.
if ((detection_idx < 0)
or (not pred_detections_channel[detection_idx].proto.boundingBoxes)):
predicted_detections.append([0, 0, 0, 0])
continue
bbox = pred_detections_channel[detection_idx].proto.boundingBoxes[0]
predicted_detections.append([bbox.min.x, bbox.min.y, bbox.max.x, bbox.max.y])
# --------- Compute and plot evaluation metrics and outliers --------
# Parse json objects and populate the pose statistics
predicted_poses = np.asarray(predicted_poses)
gt_poses = np.asarray(gt_poses)
num_samples = gt_poses.shape[0]
if (num_samples == 0):
return
if (args.use_2d_detections):
predicted_detections = np.asarray(predicted_detections)
# Setup folders to save eval metrics/plots.
results_dir = args.results_dir
make_clean_dir(results_dir)
# Translation error along x, y a nd z axes in meters in the depth region of interest.
translation_err = compute_roi_abs_translation_error(predicted_poses, gt_poses)
# Rotation error between quaternions of ground truth and predicted pose (in deg)
rotation_err = compute_roi_rotation_error(predicted_poses, gt_poses, config['symmetry_axis'],
config['num_rotation_symmetry'])
# ----------------------- COMPUTE AND OUTPUT KPIS --------------------------------------
# Indices in the depth range of interest
roi = np.where((gt_poses[:, 6] > config['depth_roi'][0])
& (gt_poses[:, 6] <= config['depth_roi'][1]))[0].tolist()
# Median metrics in the region of interest in terms of distance from camera
median_metrics = compute_median_pose_errors(rotation_err[roi], translation_err[roi, :])
median_metrics = [round(elem, 3) for elem in median_metrics]
translation_metrics_txt = "Median translation error in the depth roi [{}, {}] in m = {}\n".\
format(config['depth_roi'][0], config['depth_roi'][1], median_metrics[1:])
rotation_metrics_txt = "Median rotation error in the depth roi [{}, {}] = {} deg \n".\
format(config['depth_roi'][0], config['depth_roi'][1], median_metrics[0])
# Mean Precision in Pose Estimation metric
# Reference: https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=6130367
accuracy = compute_accuracy(translation_err[roi, :], rotation_err[roi],
config['translation_err_threshold'],
config['rotation_err_threshold'])
accuracy = round(accuracy, 2)
# Check if the KPIs meet the minimum thresholds and determine pass/fail
kpi_pass = (accuracy >= config['KPI_accuracy_threshold']
and median_metrics[0] <= config['rotation_err_threshold']
and median_metrics[1] <= config['translation_err_threshold'][0]
and median_metrics[2] <= config['translation_err_threshold'][1]
and median_metrics[3] <= config['translation_err_threshold'][2])
# Write KPI metrics to JSON file
evaluation_report = config
evaluation_report["translation_x_err_threshold"] = config['translation_err_threshold'][0]
evaluation_report["translation_y_err_threshold"] = config['translation_err_threshold'][1]
evaluation_report["translation_z_err_threshold"] = config['translation_err_threshold'][2]
evaluation_report["date"] = str(datetime.datetime.now().strftime("%Y-%m-%d"))
evaluation_report["num_frames"] = num_samples
evaluation_report["KPI_roi_median_translation_x_error"] = median_metrics[1]
evaluation_report["KPI_roi_median_translation_y_error"] = median_metrics[2]
evaluation_report["KPI_roi_median_translation_z_error"] = median_metrics[3]
evaluation_report["KPI_roi_median_rotation_error_deg"] = median_metrics[0]
evaluation_report["KPI_roi_accuracy"] = accuracy
evaluation_report["KPI_pass"] = bool(kpi_pass)
# Write the JSON to file
json_output_path = os.path.join(results_dir,
"pose_estimation_evaluation_" + str(eval_uuid) + ".json")
with open(json_output_path, 'w') as f:
json.dump(evaluation_report, f, indent=2)
# Save outlier images (above the error thresholds above) as animation
# List of indices common across roi, translation and rotation positive indices
positive_ind = (np.where((gt_poses[:, 6] > config['depth_roi'][0])
& (gt_poses[:, 6] <= config['depth_roi'][1])
& (translation_err[:, 0] < config['translation_err_threshold'][0])
& (translation_err[:, 1] < config['translation_err_threshold'][1])
& (translation_err[:, 2] < config['translation_err_threshold'][2])
& (rotation_err < config['rotation_err_threshold'])))[0].tolist()
outlier_ind = [x for x in roi if x not in positive_ind]
non_relevant_ind = [x for x in range(num_samples) if x not in outlier_ind]
for index in sorted(non_relevant_ind, reverse=True):
del images[index]
camera_matrix = get_camera_matrix(config["camera_focal"], config["image_center"])
save_animation_path = os.path.join("{}/{}.mp4".format(results_dir,
'outlier_images_with_predicted_poses'))
for i in range(len(images)):
# Update the image pixels with 3D bbox of the pose
images[i] = visualize_3Dboundingbox(config["object_size"], config["object_center"],
camera_matrix, predicted_poses[outlier_ind[i], :],
images[i])
if (args.use_2d_detections):
images[i] = visualize_2Dboundingbox(predicted_detections[outlier_ind[i], :],
images[i])
# Save the outlier images with 2D and 3D predicted bounding boxes as animation
save_animation_from_image_list(images, save_animation_path)
# Box-whisker plot to analyze the distribution of errors at various camera distances)
# Bins of errors based on distance from camera
[rotation_err_bins, trans_err_x_bins, trans_err_y_bins, trans_err_z_bins,
dist_values_bins] = compute_pose_error_depth_bins(rotation_err, translation_err, gt_poses,
config['box_plot_camera_distance_bins'])
plt.figure()
plt.boxplot(trans_err_x_bins)
plt.ylim([0, config['translation_box_plot_y_limit'][0]])
plt.title('Absolute of x position error for different camera distance intervals')
plt.xlabel('Distance from camera in m')
plt.ylabel('Absolute x position error in m')
plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'box_plot_abs_x_err')))
plt.figure()
plt.boxplot(trans_err_y_bins)
plt.ylim([0, config['translation_box_plot_y_limit'][1]])
plt.title('Absolute of y position error for different camera distance intervals')
plt.xlabel('Distance from camera in m')
plt.ylabel('Absolute y position error in m')
plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'box_plot_abs_y_err')))
plt.figure()
plt.boxplot(trans_err_z_bins)
plt.ylim([0, config['translation_box_plot_y_limit'][2]])
plt.title('Absolute of z error for different camera distance intervals')
plt.xlabel('Distance from camera in m')
plt.ylabel('Absolute z position error in m')
plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'box_plot_abs_z_err')))
plt.figure()
plt.boxplot(rotation_err_bins)
plt.ylim([0, config['rotation_box_plot_y_limit']])
plt.title('Rotation error in deg for different camera distance intervals')
plt.xlabel('distance from camera in m')
plt.ylabel('rotation error angle in deg')
plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'box_plot_rotation_err')))
# Rotation angle threshold vs accuracy
# Ref: PoseCNN: https://arxiv.org/pdf/1711.00199.pdf
z_min = config['depth_roi'][0]
z_max = config['depth_roi'][1]
rot_threshold_bins = np.linspace(0, 180, 180)
# Total number of samples within the camera distances range
rotation_accuracy = []
for rot_threshold in rot_threshold_bins:
positive_ind = (np.where((gt_poses[:, 6] > z_min) & (gt_poses[:, 6] <= z_max)
& (rotation_err < rot_threshold)))[0].tolist()
rotation_accuracy.append(len(positive_ind) / len(roi))
# Translation threshold vs accuracy
# Ref: PoseCNN: https://arxiv.org/pdf/1711.00199.pdf
# Plotting accuracy till translation error threshold of 30% body diameter
body_diameter = np.linalg.norm(config['object_size'])
trans_threshold_bins = np.linspace(0, 0.3 * body_diameter, 100)
# Total number of samples within the camera distances range
translation_accuracy = []
for trans_threshold in trans_threshold_bins:
positive_ind = (np.where((gt_poses[:, 6] > z_min) & (gt_poses[:, 6] <= z_max)
& (translation_err[:, 0] < trans_threshold)
& (translation_err[:, 1] < trans_threshold)
& (translation_err[:, 2] < trans_threshold)))[0].tolist()
translation_accuracy.append(len(positive_ind) / len(roi))
# Plot of accuracy vs rotation threshold
plt.figure()
plt.plot(rot_threshold_bins, rotation_accuracy)
plt.ylim([0, 1.0])
plt.xlabel('Rotationangle threshold (in deg)')
plt.ylabel('Accuracy')
plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'rotation_accuracy')))
# Plot of accuracy vs translation threshold
plt.figure()
plt.plot(trans_threshold_bins, translation_accuracy)
plt.xlabel('Translation threshold (in m)')
plt.ylabel('Accuracy')
plt.ylim([0, 1.0])
plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'translation_accuracy')))
help='Path to the cask file to read with deprecated message types.')
parser.add_argument(
'--output_cask_path',
required=True,
help='Path to the cask file to write with new message types.')
args = parser.parse_args()
# Create and start an application that has a node with a MessageLedger
app = Application()
app.add('node')
app.start()
node = app.nodes['node']
component = node['MessageLedger']
# Create cask objects for input and output
input_cask = Cask(args.input_cask_path)
output_cask = Cask(args.output_cask_path, writable=True)
# For each input channel, either convert or pass through the messages
for input_channel in input_cask.channels:
series = input_cask[input_channel.name]
if series[0].type_id == Message.create_message_builder(
'ColorCameraProto').proto.schema.node.id:
image_channel = output_cask.open_channel(component,
input_channel.name)
intrinsics_channel = output_cask.open_channel(
component, "{}_intrinsics".format(input_channel.name))
for input_message in tqdm(
series,
'> Splitting ColorCameraProto type messages of input channel "{0}" to write '
'ImageProto and CameraIntrinsicsProto type messages to channels named "{0}" '
'and "{0}_intrinsics"'.format(input_channel.name)):
def create_composite_atlas_ur10(cask_root, joints):
'''Creates composite atlas cask with waypoints for ur10. Tested with ovkit sim'''
if len(joints) != 6:
raise ValueError("UR10 should have 6 joints, got {}".format(
len(joints)))
cask = Cask(cask_root, writable=True)
# at joint waypoints
quantities = [[x, "position", 1] for x in joints]
CART_OBSERVE_WAYPOINT = np.array(
[3.460, -1.552, 1.810, 4.902, -1.494, 0.796],
dtype=np.dtype("float64"))
CART_ALIGN_WAYPOINT = np.array(
[3.4169, -0.5545, 1.7097, 3.5572, -1.5708, 1.8461],
dtype=np.dtype("float64"))
CART_DROPOFF_WAYPOINT = np.array(
[3.4169, -0.4016, 1.6376, 3.4764, -1.5708, 1.8461],
dtype=np.dtype("float64"))
DOLLY_OBSERVE_WAYPOINT = np.array(
[5.9866, -1.062, 1.251, 5.164, -1.716, 0.217],
dtype=np.dtype("float64"))
DOLLY_ALIGN_WAYPOINT = np.array(
[5.9866, -0.3981, 1.3259, 3.7847, -1.5708, 1.2853],
dtype=np.dtype("float64"))
DOLLY_DROPOFF_WAYPOINT = np.array(
[5.9866, -0.259, 1.2457, 3.7257, -1.5708, 1.2856],
dtype=np.dtype("float64"))
cask.write_message(
create_composite_waypoint("cart_observe", quantities,
CART_OBSERVE_WAYPOINT))
cask.write_message(
create_composite_waypoint("cart_align", quantities,
CART_ALIGN_WAYPOINT))
cask.write_message(
create_composite_waypoint("cart_dropoff", quantities,
CART_DROPOFF_WAYPOINT))
cask.write_message(
create_composite_waypoint("dolly_observe", quantities,
DOLLY_OBSERVE_WAYPOINT))
cask.write_message(
create_composite_waypoint("dolly_align", quantities,
DOLLY_ALIGN_WAYPOINT))
cask.write_message(
create_composite_waypoint("dolly_dropoff", quantities,
DOLLY_DROPOFF_WAYPOINT))
quantities = [[x, "none", 1] for x in ["pump", "valve", "gripper"]]
SUCTION_ON_WAYPOINT = np.array([1.0, 0.0, 1.0], dtype=np.dtype("float64"))
SUCTION_OFF_WAYPOINT = np.array([0.0, 1.0, 0.0], dtype=np.dtype("float64"))
VALVE_OFF_WAYPOINT = np.array([0.0, 0.0, 0.0], dtype=np.dtype("float64"))
cask.write_message(
create_composite_waypoint("suction_on", quantities,
SUCTION_ON_WAYPOINT))
cask.write_message(
create_composite_waypoint("suction_off", quantities,
SUCTION_OFF_WAYPOINT))
cask.write_message(
create_composite_waypoint("valve_off", quantities, VALVE_OFF_WAYPOINT))