def main(root): cask = Cask(root) demo_list_channels(cask) demo_json_access(cask, "imu_raw") demo_capnproto_access(cask, "imu_raw") demo_numpy_access(cask, "segway_state") demo_numpy_access(cask, "state") demo_json_access(cask, "fullscan") demo_buffer_access(cask, "fullscan") demo_numpy_access(cask, "fullscan") demo_load_perf(cask, "color") demo_image(cask, "color") demo_rgbd_stream(cask, "color", "depth")
def load_log(root): """ Loads a cask for a given root. Note: a root may contain a Cask log file or a single arbitraty named UUID subfolder containig a single Cask log file """ log_paths = glob.glob(os.path.join(root, '**', 'kv'), recursive=True) if not log_paths: logging.error("Log path %s doesn't contain any Isaac logs." % root) return None if len(log_paths) > 1: logging.error( "Log path %s should only contain a single Isaac log." "Please remove these logs and re-run the processing stage." % root) return None return Cask(log_paths[0][:-3])
def create_composite_atlas(cask_root, joints): '''Creates composite atlas cask with waypoints for ur10.''' if len(joints) != 6: raise ValueError("UR10 should have 6 joints, got {}".format(len(joints))) cask = Cask(cask_root, writable=True) # at joint waypoints quantities = [[x, "position", 1] for x in joints] CART_OBSERVE_WAYPOINT = np.array( [3.2334, -1.4856, 1.7979, 4.8503, -1.5219, 0.5585], dtype=np.dtype("float64")) CART_ALIGN_WAYPOINT = np.array( [3.3922, -0.8047, 1.7275, 3.7742, -1.5163, 1.8773], dtype=np.dtype("float64")) CART_DROPOFF_WAYPOINT = np.array( [3.3942, -0.6317, 1.6605, 3.6562, -1.5213, 1.8773], dtype=np.dtype("float64")) DOLLY_OBSERVE_WAYPOINT = np.array( [6.0535, -1.0559, 1.2437, 5.2903, -1.7218, 0.3887], dtype=np.dtype("float64")) DOLLY_ALIGN_WAYPOINT = np.array( [5.8864, -0.5826, 1.5502, 3.8685, -1.5238, 1.1526], dtype=np.dtype("float64")) DOLLY_DROPOFF_WAYPOINT = np.array( [5.8894, -0.3856, 1.4512, 3.7055, -1.5238, 1.1526], dtype=np.dtype("float64")) cask.write_message(create_composite_waypoint("cart_observe", quantities, CART_OBSERVE_WAYPOINT)) cask.write_message(create_composite_waypoint("cart_align", quantities, CART_ALIGN_WAYPOINT)) cask.write_message(create_composite_waypoint("cart_dropoff", quantities, CART_DROPOFF_WAYPOINT)) cask.write_message( create_composite_waypoint("dolly_observe", quantities, DOLLY_OBSERVE_WAYPOINT)) cask.write_message(create_composite_waypoint("dolly_align", quantities, DOLLY_ALIGN_WAYPOINT)) cask.write_message( create_composite_waypoint("dolly_dropoff", quantities, DOLLY_DROPOFF_WAYPOINT)) quantities = [[x, "none", 1] for x in ["pump", "valve", "gripper"]] SUCTION_ON_WAYPOINT = np.array([1.0, 0.0, 1.0], dtype=np.dtype("float64")) SUCTION_OFF_WAYPOINT = np.array([0.0, 1.0, 0.0], dtype=np.dtype("float64")) VALVE_OFF_WAYPOINT = np.array([0.0, 0.0, 0.0], dtype=np.dtype("float64")) cask.write_message(create_composite_waypoint("suction_on", quantities, SUCTION_ON_WAYPOINT)) cask.write_message(create_composite_waypoint("suction_off", quantities, SUCTION_OFF_WAYPOINT)) cask.write_message(create_composite_waypoint("valve_off", quantities, VALVE_OFF_WAYPOINT))
def main(args): # Create uuid eval_uuid = uuid.uuid4() # ----------------------- LOAD CONFIG -------------------------------------- # load evaluation config config = {} config_path = os.fspath(Path(args.config).resolve()) with open(config_path) as f: config = json.load(f) # read cask paths image_cask_dir_path = args.image_cask_dir gt_cask_dir_path = args.gt_cask_dir pred_cask_dir_path = args.pred_cask_dir results_dir = args.results_dir image_cask_list = multi_cask_processing.load_roots(image_cask_dir_path, "") gt_cask_list = multi_cask_processing.load_roots(gt_cask_dir_path, "") pred_cask_list = multi_cask_processing.load_roots(pred_cask_dir_path, "") pred_cask_root_list = [os.path.split(root)[0] for root in pred_cask_list] gt_image_cask_uuids = get_image_cask_uuid_raci_metadata(gt_cask_list) pred_image_cask_uuids = get_image_cask_uuid_raci_metadata( pred_cask_root_list) # Image, ground truth, and prediction data for full dataset images = [] gt_detections = [] pred_detections = [] # load other evaluation config params iou_thresholds = config['iou_thresholds'] labels = config['labels'] num_labels = len(labels) num_thresholds = len(iou_thresholds) # ----------------------- READ DATASET FROM SET OF CASKS --------------------- for image_cask_path in image_cask_list: _, image_cask_tail = os.path.split(image_cask_path) gt_cask_idx = None pred_cask_idx = None try: # get associated gt cask and pred casks gt_cask_idx = gt_image_cask_uuids.index(image_cask_tail) pred_cask_idx = pred_image_cask_uuids.index(image_cask_tail) except (ValueError): print("Associated ground truth or predicted cask is not \ available for image cask {}".format(image_cask_path)) continue gt_cask_path = gt_cask_list[gt_cask_idx] pred_cask_path = pred_cask_list[pred_cask_idx] #read image cask image_cask = Cask(image_cask_path) images_channel = image_cask[config['image_channel']] #read ground truth cask gt_cask = Cask(gt_cask_path) gt_detections_channel = gt_cask[config['gt_channel']] # read predictions cask pred_cask = Cask(pred_cask_path) pred_detections_channel = pred_cask[config['pred_channel']] # Sync image, gt detection message, and predicted detection message channels by acqtime prev_image_idx = 0 prev_gt_idx = 0 for prediction in pred_detections_channel: image_idx = get_message_idx(images_channel, prediction.acqtime, prev_image_idx) gt_idx = get_message_idx(gt_detections_channel, prediction.acqtime, prev_gt_idx) if (image_idx != -1 and gt_idx != -1): images.append(images_channel[image_idx]) gt_detections.append(gt_detections_channel[gt_idx]) pred_detections.append(prediction) prev_image_idx = image_idx prev_gt_idx = gt_idx num_samples = len( images) #also same as len(gt_detections) and len(pred_detections) # ----------------------- COMPUTE METRICS -------------------------------------- # use bboxes matched with specified IOU threshold to determine outliers outlier_iou_area_threshold = config['outlier_iou_area_threshold'] # matches with IOU under the area min have large bbox error large_bbox_iou_area_min = config['large_bbox_iou_area_min'] # Lists to collect outlier indices large_bbox_outliers_idxs = [] false_positive_idxs = [] false_negative_idxs = [] # Lists to collect (and later average) area/height/width IOUs all_ious_area = [] all_ious_height = [] all_ious_width = [] # Compute confusion matrices: one per IOU threshold confusion_matrices = { iou_threshold: np.empty((0)) for iou_threshold in iou_thresholds } for idx in range(num_samples): gt_bboxes = gt_detections[idx].proto.boundingBoxes gt_predictions = gt_detections[idx].proto.predictions pred_bboxes = pred_detections[idx].proto.boundingBoxes pred_predictions = pred_detections[idx].proto.predictions # match gt bboxes with predicted bboxes for each IOU for iou_threshold in iou_thresholds: gt_matched_idxs, pred_matched_idxs, matched_ious_width, matched_ious_height, \ matched_ious_area = match_bboxes(gt_bboxes, pred_bboxes, iou_threshold) all_ious_width += matched_ious_width all_ious_height += matched_ious_height all_ious_area += matched_ious_area # collect outliers with large bbox error if iou_threshold == outlier_iou_area_threshold and not all( i > large_bbox_iou_area_min for i in matched_ious_area): large_bbox_outliers_idxs.append(idx) # compute confusion matrices confusion_matrix_sample = matches_to_confusion_matrix( gt_matched_idxs, pred_matched_idxs, gt_predictions, pred_predictions, labels) # collect false positive idxs if iou_threshold == outlier_iou_area_threshold and len( pred_bboxes) != len(pred_matched_idxs): false_positive_idxs.append(idx) # collect false negative idxs if iou_threshold == outlier_iou_area_threshold and len( gt_bboxes) != len(gt_matched_idxs): false_negative_idxs.append(idx) # accumulate confusion matrices across samples if not confusion_matrices[iou_threshold].any(): confusion_matrices[iou_threshold] = confusion_matrix_sample else: confusion_matrices[iou_threshold] += confusion_matrix_sample # Compute precision and recall metrics for each class and each IoU from the confusion matrix per_class_precisions = np.zeros((num_labels, num_thresholds)) per_class_recalls = np.zeros((num_labels, num_thresholds)) for iou_idx in range(num_thresholds): confusion_matrix = confusion_matrices[iou_thresholds[iou_idx]] precisions = np.diagonal(confusion_matrix) / np.sum(confusion_matrix, axis=0) recalls = np.diagonal(confusion_matrix) / np.sum(confusion_matrix, axis=1) per_class_precisions[:, iou_idx] = precisions[:-1] per_class_recalls[:, iou_idx] = recalls[:-1] # compute average IOUs for area as well as height and width dimensions mean_iou_width = sum(all_ious_width) / len(all_ious_width) mean_iou_height = sum(all_ious_height) / len(all_ious_height) mean_iou_area = sum(all_ious_area) / len(all_ious_area) # ----------------------- COMPUTE KPIS FROM METRICS --------------------------- # COCO 2017 challenge mAP is computed over all IOUs over all classes # https://cocodataset.org/#detection-eval KPI_mAP = np.mean(per_class_precisions) KPI_mAR = np.mean(per_class_recalls) # PASCAL VOC2007 challenge mAP is computed =over a single IOU over all classes # http://host.robots.ox.ac.uk/pascal/VOC/ mAP_per_IOU = np.mean(per_class_precisions, axis=0) mAR_per_IOU = np.mean(per_class_recalls, axis=0) KPI_mAP_lowest_IOU = mAP_per_IOU[0] KPI_mAR_lowest_IOU = mAR_per_IOU[0] # Check if the KPIs meet the minimum thresholds and determine pass/fail kpi_pass = (KPI_mAP >= config["KPI_threshold_mAP"] and KPI_mAR >= config["KPI_threshold_mAR"] \ and KPI_mAP_lowest_IOU >= config["KPI_threshold_mAP_lowest_IOU"] \ and KPI_mAR_lowest_IOU >= config["KPI_threshold_mAR_lowest_IOU"]) # ----------------------- OUTPUT ALL RESULTS -------------------------------------- evaluation_report = config evaluation_report["date"] = str( datetime.datetime.now().strftime("%Y-%m-%d")) # Construct JSON output results = {} results["num_frames"] = num_samples # Write object detection evaluation metrics statistics = [] for class_idx in range(num_labels): per_class_stats = {} per_class_stats["class_name"] = labels[class_idx] per_class_stats["precisions"] = per_class_precisions[ class_idx, :].tolist() per_class_stats["recalls"] = per_class_recalls[class_idx, :].tolist() statistics.append(per_class_stats) results["per_class_stats"] = statistics # Write confusion matrices and mean IOUs conf_matrices_stats = [] for iou_idx in range(num_thresholds): conf_matrix = {} conf_matrix["confusion_matrix"] = confusion_matrices[ iou_thresholds[iou_idx]].tolist() if iou_thresholds[iou_idx] == outlier_iou_area_threshold: conf_matrix["mean_iou_width"] = mean_iou_width conf_matrix["mean_iou_height"] = mean_iou_height conf_matrix["mean_iou_area"] = mean_iou_area conf_matrices_stats.append(conf_matrix) results["confusion_matrices"] = conf_matrices_stats # Write outlier indices outliers = {} outliers["large_bbox_error_idxs"] = large_bbox_outliers_idxs outliers["false_positive_idxs"] = false_positive_idxs outliers["false_negative_idxs"] = false_negative_idxs results["outliers"] = outliers # Write results and KPIs evaluation_report["results"] = results evaluation_report["KPI_mAP"] = KPI_mAP evaluation_report["KPI_mAR"] = KPI_mAR evaluation_report["KPI_mAP_lowest_IOU"] = KPI_mAP_lowest_IOU evaluation_report["KPI_mAR_lowest_IOU"] = KPI_mAR_lowest_IOU evaluation_report["KPI_pass"] = bool(kpi_pass) # Write the JSON to file if not os.path.exists(results_dir): os.makedirs(results_dir) json_output_path = os.path.join( results_dir, "object_detection_evaluation_" + str(eval_uuid) + ".json") with open(json_output_path, 'w') as f: json.dump(evaluation_report, f, indent=2) # ----------------------- VISUALIZE OUTLIERS -------------------------------------- if args.save_outliers: large_bbox_outliers_dir = os.path.join(results_dir, "large_bbox_error_outliers") if not os.path.exists(large_bbox_outliers_dir): os.makedirs(large_bbox_outliers_dir) for outlier_idx in large_bbox_outliers_idxs: save_sample(outlier_idx, images[outlier_idx], gt_detections[outlier_idx], \ pred_detections[outlier_idx], "large bbox error outlier", large_bbox_outliers_dir) false_positives_dir = os.path.join(results_dir, "false_positives") if not os.path.exists(false_positives_dir): os.makedirs(false_positives_dir) for outlier_idx in false_positive_idxs: save_sample(outlier_idx, images[outlier_idx], gt_detections[outlier_idx], \ pred_detections[outlier_idx], "false positive outlier", false_positives_dir) false_negatives_dir = os.path.join(results_dir, "false_negatives") if not os.path.exists(false_negatives_dir): os.makedirs(false_negatives_dir) for outlier_idx in false_negative_idxs: save_sample(outlier_idx, images[outlier_idx], gt_detections[outlier_idx], \ pred_detections[outlier_idx], "false negative outlier", false_negatives_dir) print("Evalulation complete. Results saved to: " + json_output_path)
def create_composite_atlas_ur10(cask_root, joints): '''Creates composite atlas cask with waypoints for ur10. Tested with ovkit sim.''' if len(joints) != 6: raise ValueError("UR10 should have 6 joints, got {}".format( len(joints))) cask = Cask(cask_root, writable=True) # joint waypoints quantities = [[x, "position", 1] for x in joints] HOME_POSE_WAYPOINT = np.array( [1.3504, -1.4784, 1.6887, -1.7811, -1.5708, 1.3488], dtype=np.dtype("float64")) VIEW_POSE_WAYPOINT = np.array( [2.1358, -1.4784, 1.6887, -1.7811, -1.5708, 0.5635], dtype=np.dtype("float64")) APPROACH_POSE_WAYPOINT = np.array( [-0.2966, -1.062, 1.251, -1.38, -1.716, 0.217], dtype=np.dtype("float64")) cask.write_message( pick_and_place.create_composite_waypoint("home_pose", quantities, HOME_POSE_WAYPOINT)) cask.write_message( pick_and_place.create_composite_waypoint("view_pose", quantities, VIEW_POSE_WAYPOINT)) cask.write_message( pick_and_place.create_composite_waypoint("approach_pose", quantities, APPROACH_POSE_WAYPOINT)) # gripper waypoints quantities = [[x, "none", 1] for x in ["pump", "valve", "gripper"]] SUCTION_ON_WAYPOINT = np.array([1.0, 0.0, 1.0], dtype=np.dtype("float64")) SUCTION_OFF_WAYPOINT = np.array([0.0, 1.0, 0.0], dtype=np.dtype("float64")) VALVE_OFF_WAYPOINT = np.array([0.0, 0.0, 0.0], dtype=np.dtype("float64")) cask.write_message( pick_and_place.create_composite_waypoint("suction_on", quantities, SUCTION_ON_WAYPOINT)) cask.write_message( pick_and_place.create_composite_waypoint("suction_off", quantities, SUCTION_OFF_WAYPOINT)) cask.write_message( pick_and_place.create_composite_waypoint("valve_off", quantities, VALVE_OFF_WAYPOINT))
dts = [] for i in range(len(channel) - 1): dt_ns = channel[i + 1].pubtime - channel[i].pubtime dts.append(dt_ns / 1000000.0) return dts if __name__ == '__main__': parser = argparse.ArgumentParser( description='Time difference historgram for cask channel') parser.add_argument('--root', dest='root', help='The directory in which log files will be stored') parser.add_argument('--channel', dest='channel', default='color', help='The channel for which to show time differences') args = parser.parse_args() cask = Cask(args.root) dts = channel_dts(cask[args.channel]) print(numpy.histogram(dts)) plt.title(args.channel) plt.hist(dts, bins='auto') plt.ylabel("Number of messages") plt.xlabel("Time delta between messages (ms)") plt.show()
def create_composite_atlas_franka(cask_root, joints): '''Creates composite atlas cask with waypoints for franka. Tested with ovkit sim.''' if len(joints) != 7: raise ValueError("Franka should have 7 joints, got {}".format( len(joints))) cask = Cask(cask_root, writable=True) # joint waypoints quantities = [[x, "position", 1] for x in joints] HOME_POSE_WAYPOINT = np.array( [-0.6392, -0.3524, 0.5867, -2.3106, 0.2122, 2.0046, 0.6137], dtype=np.dtype("float64")) cask.write_message( create_composite_waypoint("home_pose", quantities, HOME_POSE_WAYPOINT)) # The home pose is re-used as view and approach pose to be consistent with the waypoints used by # the UR10. cask.write_message( create_composite_waypoint("view_pose", quantities, HOME_POSE_WAYPOINT)) cask.write_message( create_composite_waypoint("approach_pose", quantities, HOME_POSE_WAYPOINT)) # gripper waypoints quantities = [[x, "none", 1] for x in ["gripper"]] GRIPPER_OPEN_WAYPOINT = np.array([0.0], dtype=np.dtype("float64")) GRIPPER_CLOSE_WAYPOINT = np.array([1.0], dtype=np.dtype("float64")) cask.write_message( create_composite_waypoint("gripper_open", quantities, GRIPPER_OPEN_WAYPOINT)) cask.write_message( create_composite_waypoint("gripper_close", quantities, GRIPPER_CLOSE_WAYPOINT))
def main(args): """ The entry point of the application. The function reads the json file containing the ground truth and predicted pose labels, computes evaluation metrics and save the plots of translationa and rotation errors. """ # Create uuid eval_uuid = uuid.uuid4() # ----------------------- LOAD CONFIG -------------------------------------- # load evaluation config config = {} config_path = os.fspath(Path(args.config).resolve()) with open(config_path) as f: config = json.load(f) # read cask paths image_cask_dir_path = args.image_cask_dir gt_cask_dir_path = args.gt_cask_dir pred_cask_dir_path = args.predicted_cask_dir image_cask_list = multi_cask_processing.load_roots(image_cask_dir_path, "") gt_poses_cask_list = multi_cask_processing.load_roots(gt_cask_dir_path, "") predicted_poses_cask_list = multi_cask_processing.load_roots(pred_cask_dir_path, "") pred_cask_root_list = [os.path.split(root)[0] for root in predicted_poses_cask_list] gt_cask_root_list = [os.path.split(root)[0] for root in gt_poses_cask_list] pred_image_cask_uuids = get_image_cask_uuid_raci_metadata(pred_cask_root_list) assert (pred_image_cask_uuids is not None), "Unable to find the metadata files for prediction casks." # Check for image cask uuids in the ground truth cask list (for simulation) or # roots of the cask list (for real data ground truth casks) gt_image_cask_uuids = get_image_cask_uuid_raci_metadata(gt_poses_cask_list) if (gt_image_cask_uuids is None): gt_image_cask_uuids = get_image_cask_uuid_raci_metadata(gt_cask_root_list) assert (gt_image_cask_uuids is not None), "Unable to find the metadata files for ground truth casks." # List of predicted poses images = [] predicted_poses = [] gt_poses = [] predicted_detections = [] # Read all the json objects into a string if (not (len(predicted_poses_cask_list) == len(gt_poses_cask_list) or not (len(predicted_poses_cask_list) == len(image_cask_list)))): raiseError("Length of image, ground truth and predicted labels list must be equal") # ------------------------- AGGREGATE IMAGE, PREDICTIONS, LABELS DATA -------------------- for image_cask_path in image_cask_list: _, image_cask_tail = os.path.split(image_cask_path) try: # get associated gt cask and pred casks gt_cask_idx = gt_image_cask_uuids.index(image_cask_tail) pred_cask_idx = pred_image_cask_uuids.index(image_cask_tail) except (ValueError): print("Associated ground truth or predicted cask is not \ available for image cask {}".format(image_cask_path)) continue gt_cask_path = gt_poses_cask_list[gt_cask_idx] pred_cask_path = predicted_poses_cask_list[pred_cask_idx] #read image cask images_channel = Cask(image_cask_path)[config['image_channel']] #read ground truth cask gt_poses_channel = Cask(gt_cask_path)[config['gt_pose_channel']] # read predictions cask pred_poses_channel = Cask(pred_cask_path)[config['pred_pose_channel']] if (args.use_2d_detections): # read predcited detections cask if the channel is not empty pred_detections_channel = Cask(pred_cask_path)[config['pred_detection_channel']] # Reset indices before synching casks prev_image_idx = 0 prev_gt_pose_idx = 0 prev_pred_detection_idx = 0 # ----------------------- SYNC CASKS -------------------------------------- # Sync image, gt pose message, and predicted pose message channels by acqtime for prediction_pose in pred_poses_channel: #TODO:[Sravya] - Extend the evaluation to multiple objects in an image scenario if (len(prediction_pose.proto.poses) == 0 or len(prediction_pose.proto.poses) > 1): continue acqtime = prediction_pose.acqtime gt_pose_idx = get_message_idx(gt_poses_channel, acqtime, prev_gt_pose_idx) image_idx = get_message_idx(images_channel, acqtime, prev_image_idx) # Skip the message if the matching timestamp is not found in either casks if (image_idx != -1 and gt_pose_idx != -1): if (len(gt_poses_channel[gt_pose_idx].proto.poses) == 0): continue images.append(images_channel[image_idx].tensor) gt_poses.append(pose_capnp_to_list(gt_poses_channel[gt_pose_idx].proto.poses[0])) predicted_poses.append(pose_capnp_to_list(prediction_pose.proto.poses[0])) prev_image_idx = image_idx prev_gt_pose_idx = gt_pose_idx if (args.use_2d_detections): detection_idx = get_message_idx(pred_detections_channel, acqtime, prev_pred_detection_idx) prev_pred_detection_idx = detection_idx # If detection sample is missing, add a zero size bbox instead of # ignoring the sample. if ((detection_idx < 0) or (not pred_detections_channel[detection_idx].proto.boundingBoxes)): predicted_detections.append([0, 0, 0, 0]) continue bbox = pred_detections_channel[detection_idx].proto.boundingBoxes[0] predicted_detections.append([bbox.min.x, bbox.min.y, bbox.max.x, bbox.max.y]) # --------- Compute and plot evaluation metrics and outliers -------- # Parse json objects and populate the pose statistics predicted_poses = np.asarray(predicted_poses) gt_poses = np.asarray(gt_poses) num_samples = gt_poses.shape[0] if (num_samples == 0): return if (args.use_2d_detections): predicted_detections = np.asarray(predicted_detections) # Setup folders to save eval metrics/plots. results_dir = args.results_dir make_clean_dir(results_dir) # Translation error along x, y a nd z axes in meters in the depth region of interest. translation_err = compute_roi_abs_translation_error(predicted_poses, gt_poses) # Rotation error between quaternions of ground truth and predicted pose (in deg) rotation_err = compute_roi_rotation_error(predicted_poses, gt_poses, config['symmetry_axis'], config['num_rotation_symmetry']) # ----------------------- COMPUTE AND OUTPUT KPIS -------------------------------------- # Indices in the depth range of interest roi = np.where((gt_poses[:, 6] > config['depth_roi'][0]) & (gt_poses[:, 6] <= config['depth_roi'][1]))[0].tolist() # Median metrics in the region of interest in terms of distance from camera median_metrics = compute_median_pose_errors(rotation_err[roi], translation_err[roi, :]) median_metrics = [round(elem, 3) for elem in median_metrics] translation_metrics_txt = "Median translation error in the depth roi [{}, {}] in m = {}\n".\ format(config['depth_roi'][0], config['depth_roi'][1], median_metrics[1:]) rotation_metrics_txt = "Median rotation error in the depth roi [{}, {}] = {} deg \n".\ format(config['depth_roi'][0], config['depth_roi'][1], median_metrics[0]) # Mean Precision in Pose Estimation metric # Reference: https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=6130367 accuracy = compute_accuracy(translation_err[roi, :], rotation_err[roi], config['translation_err_threshold'], config['rotation_err_threshold']) accuracy = round(accuracy, 2) # Check if the KPIs meet the minimum thresholds and determine pass/fail kpi_pass = (accuracy >= config['KPI_accuracy_threshold'] and median_metrics[0] <= config['rotation_err_threshold'] and median_metrics[1] <= config['translation_err_threshold'][0] and median_metrics[2] <= config['translation_err_threshold'][1] and median_metrics[3] <= config['translation_err_threshold'][2]) # Write KPI metrics to JSON file evaluation_report = config evaluation_report["translation_x_err_threshold"] = config['translation_err_threshold'][0] evaluation_report["translation_y_err_threshold"] = config['translation_err_threshold'][1] evaluation_report["translation_z_err_threshold"] = config['translation_err_threshold'][2] evaluation_report["date"] = str(datetime.datetime.now().strftime("%Y-%m-%d")) evaluation_report["num_frames"] = num_samples evaluation_report["KPI_roi_median_translation_x_error"] = median_metrics[1] evaluation_report["KPI_roi_median_translation_y_error"] = median_metrics[2] evaluation_report["KPI_roi_median_translation_z_error"] = median_metrics[3] evaluation_report["KPI_roi_median_rotation_error_deg"] = median_metrics[0] evaluation_report["KPI_roi_accuracy"] = accuracy evaluation_report["KPI_pass"] = bool(kpi_pass) # Write the JSON to file json_output_path = os.path.join(results_dir, "pose_estimation_evaluation_" + str(eval_uuid) + ".json") with open(json_output_path, 'w') as f: json.dump(evaluation_report, f, indent=2) # Save outlier images (above the error thresholds above) as animation # List of indices common across roi, translation and rotation positive indices positive_ind = (np.where((gt_poses[:, 6] > config['depth_roi'][0]) & (gt_poses[:, 6] <= config['depth_roi'][1]) & (translation_err[:, 0] < config['translation_err_threshold'][0]) & (translation_err[:, 1] < config['translation_err_threshold'][1]) & (translation_err[:, 2] < config['translation_err_threshold'][2]) & (rotation_err < config['rotation_err_threshold'])))[0].tolist() outlier_ind = [x for x in roi if x not in positive_ind] non_relevant_ind = [x for x in range(num_samples) if x not in outlier_ind] for index in sorted(non_relevant_ind, reverse=True): del images[index] camera_matrix = get_camera_matrix(config["camera_focal"], config["image_center"]) save_animation_path = os.path.join("{}/{}.mp4".format(results_dir, 'outlier_images_with_predicted_poses')) for i in range(len(images)): # Update the image pixels with 3D bbox of the pose images[i] = visualize_3Dboundingbox(config["object_size"], config["object_center"], camera_matrix, predicted_poses[outlier_ind[i], :], images[i]) if (args.use_2d_detections): images[i] = visualize_2Dboundingbox(predicted_detections[outlier_ind[i], :], images[i]) # Save the outlier images with 2D and 3D predicted bounding boxes as animation save_animation_from_image_list(images, save_animation_path) # Box-whisker plot to analyze the distribution of errors at various camera distances) # Bins of errors based on distance from camera [rotation_err_bins, trans_err_x_bins, trans_err_y_bins, trans_err_z_bins, dist_values_bins] = compute_pose_error_depth_bins(rotation_err, translation_err, gt_poses, config['box_plot_camera_distance_bins']) plt.figure() plt.boxplot(trans_err_x_bins) plt.ylim([0, config['translation_box_plot_y_limit'][0]]) plt.title('Absolute of x position error for different camera distance intervals') plt.xlabel('Distance from camera in m') plt.ylabel('Absolute x position error in m') plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'box_plot_abs_x_err'))) plt.figure() plt.boxplot(trans_err_y_bins) plt.ylim([0, config['translation_box_plot_y_limit'][1]]) plt.title('Absolute of y position error for different camera distance intervals') plt.xlabel('Distance from camera in m') plt.ylabel('Absolute y position error in m') plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'box_plot_abs_y_err'))) plt.figure() plt.boxplot(trans_err_z_bins) plt.ylim([0, config['translation_box_plot_y_limit'][2]]) plt.title('Absolute of z error for different camera distance intervals') plt.xlabel('Distance from camera in m') plt.ylabel('Absolute z position error in m') plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'box_plot_abs_z_err'))) plt.figure() plt.boxplot(rotation_err_bins) plt.ylim([0, config['rotation_box_plot_y_limit']]) plt.title('Rotation error in deg for different camera distance intervals') plt.xlabel('distance from camera in m') plt.ylabel('rotation error angle in deg') plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'box_plot_rotation_err'))) # Rotation angle threshold vs accuracy # Ref: PoseCNN: https://arxiv.org/pdf/1711.00199.pdf z_min = config['depth_roi'][0] z_max = config['depth_roi'][1] rot_threshold_bins = np.linspace(0, 180, 180) # Total number of samples within the camera distances range rotation_accuracy = [] for rot_threshold in rot_threshold_bins: positive_ind = (np.where((gt_poses[:, 6] > z_min) & (gt_poses[:, 6] <= z_max) & (rotation_err < rot_threshold)))[0].tolist() rotation_accuracy.append(len(positive_ind) / len(roi)) # Translation threshold vs accuracy # Ref: PoseCNN: https://arxiv.org/pdf/1711.00199.pdf # Plotting accuracy till translation error threshold of 30% body diameter body_diameter = np.linalg.norm(config['object_size']) trans_threshold_bins = np.linspace(0, 0.3 * body_diameter, 100) # Total number of samples within the camera distances range translation_accuracy = [] for trans_threshold in trans_threshold_bins: positive_ind = (np.where((gt_poses[:, 6] > z_min) & (gt_poses[:, 6] <= z_max) & (translation_err[:, 0] < trans_threshold) & (translation_err[:, 1] < trans_threshold) & (translation_err[:, 2] < trans_threshold)))[0].tolist() translation_accuracy.append(len(positive_ind) / len(roi)) # Plot of accuracy vs rotation threshold plt.figure() plt.plot(rot_threshold_bins, rotation_accuracy) plt.ylim([0, 1.0]) plt.xlabel('Rotationangle threshold (in deg)') plt.ylabel('Accuracy') plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'rotation_accuracy'))) # Plot of accuracy vs translation threshold plt.figure() plt.plot(trans_threshold_bins, translation_accuracy) plt.xlabel('Translation threshold (in m)') plt.ylabel('Accuracy') plt.ylim([0, 1.0]) plt.savefig(os.path.join("{}/{}.png".format(results_dir, 'translation_accuracy')))
help='Path to the cask file to read with deprecated message types.') parser.add_argument( '--output_cask_path', required=True, help='Path to the cask file to write with new message types.') args = parser.parse_args() # Create and start an application that has a node with a MessageLedger app = Application() app.add('node') app.start() node = app.nodes['node'] component = node['MessageLedger'] # Create cask objects for input and output input_cask = Cask(args.input_cask_path) output_cask = Cask(args.output_cask_path, writable=True) # For each input channel, either convert or pass through the messages for input_channel in input_cask.channels: series = input_cask[input_channel.name] if series[0].type_id == Message.create_message_builder( 'ColorCameraProto').proto.schema.node.id: image_channel = output_cask.open_channel(component, input_channel.name) intrinsics_channel = output_cask.open_channel( component, "{}_intrinsics".format(input_channel.name)) for input_message in tqdm( series, '> Splitting ColorCameraProto type messages of input channel "{0}" to write ' 'ImageProto and CameraIntrinsicsProto type messages to channels named "{0}" ' 'and "{0}_intrinsics"'.format(input_channel.name)):
def create_composite_atlas_ur10(cask_root, joints): '''Creates composite atlas cask with waypoints for ur10. Tested with ovkit sim''' if len(joints) != 6: raise ValueError("UR10 should have 6 joints, got {}".format( len(joints))) cask = Cask(cask_root, writable=True) # at joint waypoints quantities = [[x, "position", 1] for x in joints] CART_OBSERVE_WAYPOINT = np.array( [3.460, -1.552, 1.810, 4.902, -1.494, 0.796], dtype=np.dtype("float64")) CART_ALIGN_WAYPOINT = np.array( [3.4169, -0.5545, 1.7097, 3.5572, -1.5708, 1.8461], dtype=np.dtype("float64")) CART_DROPOFF_WAYPOINT = np.array( [3.4169, -0.4016, 1.6376, 3.4764, -1.5708, 1.8461], dtype=np.dtype("float64")) DOLLY_OBSERVE_WAYPOINT = np.array( [5.9866, -1.062, 1.251, 5.164, -1.716, 0.217], dtype=np.dtype("float64")) DOLLY_ALIGN_WAYPOINT = np.array( [5.9866, -0.3981, 1.3259, 3.7847, -1.5708, 1.2853], dtype=np.dtype("float64")) DOLLY_DROPOFF_WAYPOINT = np.array( [5.9866, -0.259, 1.2457, 3.7257, -1.5708, 1.2856], dtype=np.dtype("float64")) cask.write_message( create_composite_waypoint("cart_observe", quantities, CART_OBSERVE_WAYPOINT)) cask.write_message( create_composite_waypoint("cart_align", quantities, CART_ALIGN_WAYPOINT)) cask.write_message( create_composite_waypoint("cart_dropoff", quantities, CART_DROPOFF_WAYPOINT)) cask.write_message( create_composite_waypoint("dolly_observe", quantities, DOLLY_OBSERVE_WAYPOINT)) cask.write_message( create_composite_waypoint("dolly_align", quantities, DOLLY_ALIGN_WAYPOINT)) cask.write_message( create_composite_waypoint("dolly_dropoff", quantities, DOLLY_DROPOFF_WAYPOINT)) quantities = [[x, "none", 1] for x in ["pump", "valve", "gripper"]] SUCTION_ON_WAYPOINT = np.array([1.0, 0.0, 1.0], dtype=np.dtype("float64")) SUCTION_OFF_WAYPOINT = np.array([0.0, 1.0, 0.0], dtype=np.dtype("float64")) VALVE_OFF_WAYPOINT = np.array([0.0, 0.0, 0.0], dtype=np.dtype("float64")) cask.write_message( create_composite_waypoint("suction_on", quantities, SUCTION_ON_WAYPOINT)) cask.write_message( create_composite_waypoint("suction_off", quantities, SUCTION_OFF_WAYPOINT)) cask.write_message( create_composite_waypoint("valve_off", quantities, VALVE_OFF_WAYPOINT))