def save_samples(self, sample, grasps, T_obj_stp, collision_checker, grasp_metrics, stable_pose): self.tensor_datapoint['image_labels'] = self.cur_image_label T_stp_camera = sample.camera.object_to_camera_pose self.T_obj_camera = T_stp_camera * T_obj_stp.as_frames('obj', T_stp_camera.from_frame) aligned_grasps = self.align_grasps_with_camera(grasps) depth_im_table = sample.renders[RenderMode.DEPTH_SCENE].image cx = depth_im_table.center[1] cy = depth_im_table.center[0] camera_pose = self.get_camera_pose(sample) self.tensor_datapoint['camera_poses'] = camera_pose shifted_camera_intr = sample.camera.camera_intr self.tensor_datapoint['camera_intrs'] = np.r_[shifted_camera_intr.fx, shifted_camera_intr.fy, shifted_camera_intr.cx, shifted_camera_intr.cy] self.save_orig_image(depth_im_table, camera_pose, stable_pose, shifted_camera_intr) grid_space = 200 grids = 3 centre = np.empty((grids, grids, 2)) saved = np.ones((grids, grids)) * -1 depth_images = np.empty((grids, grids, 32, 32, 1)) # Crop images in a grid around the centre for x_grid in range(grids): x_pos = grid_space / 4 - x_grid * grid_space / (grids + 1) for y_grid in range(grids): y_pos = grid_space / 4 - y_grid * grid_space / (grids + 1) depth_im = self.prepare_images(depth_im_table, x_pos, y_pos) self._show_image(depth_im) depth_images[x_grid, y_grid, :, :, :] = depth_im centre[x_grid, y_grid, 0] = x_pos centre[x_grid, y_grid, 1] = y_pos for cnt, aligned_grasp in enumerate(aligned_grasps): collision_free = self.is_grasp_collision_free(aligned_grasp, collision_checker) # Project grasp coordinates in image grasp_2d = aligned_grasp.project_camera(self.T_obj_camera, shifted_camera_intr) grasp_point = aligned_grasp.close_fingers(self.obj, vis=False, check_approach=False) if not grasp_point[0]: Warning("Could not find contact points") continue # Get grasp width in pixel from endpoints p_1 = np.dot(self.T_obj_camera.matrix, np.append(grasp_point[1][0].point, 1).T).T[:3] p_2 = np.dot(self.T_obj_camera.matrix, np.append(grasp_point[1][1].point, 1).T).T[:3] grasp_width_px = self.width_px(shifted_camera_intr, p_1, p_2) grasp_width = aligned_grasp.width_from_endpoints(grasp_point[1][0].point, grasp_point[1][1].point) pos_x = cx - grasp_2d.center.x pos_y = cy - grasp_2d.center.y distance = np.abs((centre[:, :, 0] - pos_x)**2 + (centre[:, :, 1] - pos_y)**2) idx = np.where(distance == np.amin(distance)) idx_x = idx[0][0] idx_y = idx[1][0] grasp_center_x = (300 - grasp_2d.center.x - centre[idx_x, idx_y, 0])//3 grasp_center_y = (300 - grasp_2d.center.y - centre[idx_x, idx_y, 1])//3 if saved[idx_x, idx_y] == -1: np.save(self.image_dir + "/depth_im_{:07d}.npy".format(self.cur_image_file), depth_images[idx_x, idx_y, :, :, :]) saved[idx_x, idx_y] = self.cur_image_file self.tensor_datapoint['image_files'] = self.cur_image_file self.cur_image_file += 1 else: self.tensor_datapoint['image_files'] = saved[idx_x, idx_y] T_grasp_camera = aligned_grasp.gripper_pose(self.gripper).inverse() * self.T_obj_camera.inverse() if VISUALISE_3D: z_axis = T_grasp_camera.z_axis theta = np.rad2deg(np.arccos((z_axis[2]) / (np.sqrt((z_axis[0] ** 2 + z_axis[1] ** 2 + z_axis[2] ** 2))))) vis.figure() T_obj_world = vis.mesh_stable_pose(self.obj.mesh.trimesh, stable_pose.T_obj_world, style='surface', dim=0.5, plot_table=True) T_camera_world = T_obj_world * self.T_obj_camera.inverse() vis.gripper(self.gripper, aligned_grasp, T_obj_world, color=(0.3, 0.3, 0.3), T_camera_world=T_camera_world) print(theta) vis.show() hand_pose = self.get_hand_pose(grasp_center_x, grasp_center_y, grasp_2d, T_grasp_camera, grasp_width, grasp_width_px) self.add_datapoint(hand_pose, collision_free, aligned_grasp, grasp_metrics) self.cur_image_label += 1
def save_samples(self, sample, grasps, T_obj_stp, collision_checker, grasp_metrics, stable_pose): self.tensor_datapoint['image_labels'] = self.cur_image_label T_stp_camera = sample.camera.object_to_camera_pose self.T_obj_camera = T_stp_camera * T_obj_stp.as_frames( 'obj', T_stp_camera.from_frame) depth_im_table = sample.renders[RenderMode.DEPTH_SCENE].image camera_pose = self.get_camera_pose(sample) shifted_camera_intr = sample.camera.camera_intr aligned_grasps = self.align_grasps_with_camera(grasps) cx = depth_im_table.center[1] cy = depth_im_table.center[0] self.tensor_datapoint['camera_intrs'] = np.r_[shifted_camera_intr.fx, shifted_camera_intr.fy, shifted_camera_intr.cx, shifted_camera_intr.cy] for aligned_grasp in aligned_grasps: if not self.is_grasp_aligned(aligned_grasp): continue collision_free = self.is_grasp_collision_free( aligned_grasp, collision_checker) grasp_2d = aligned_grasp.project_camera(self.T_obj_camera, shifted_camera_intr) depth_im_tf = self.prepare_images(depth_im_table, grasp_2d, cx, cy) T_grasp_camera = aligned_grasp.gripper_pose( self.gripper).inverse() * self.T_obj_camera.inverse() if VISUALISE_3D: z_axis = T_grasp_camera.z_axis theta = np.rad2deg( np.arccos((z_axis[2]) / (np.sqrt( (z_axis[0]**2 + z_axis[1]**2 + z_axis[2]**2))))) vis.figure() T_obj_world = vis.mesh_stable_pose(self.obj.mesh.trimesh, stable_pose.T_obj_world, style='surface', dim=0.5, plot_table=True) T_camera_world = T_obj_world * self.T_obj_camera.inverse() vis.gripper(self.gripper, aligned_grasp, T_obj_world, color=(0.3, 0.3, 0.3), T_camera_world=T_camera_world) vis.show() theta = grasp_2d.angle R = np.array([[np.cos(theta), -np.sin(theta), 0], [np.sin(theta), np.cos(theta), 0], [0, 0, 1]]) rot = RigidTransform(rotation=R, to_frame='camera', from_frame='camera') new_quaternion = T_grasp_camera * rot hand_pose = self.get_hand_pose(grasp_2d, new_quaternion.quaternion) self.add_datapoint(depth_im_tf, hand_pose, collision_free, camera_pose, aligned_grasp, grasp_metrics) self.cur_image_label += 1
def generate_gqcnn_dataset(dataset_path, database, target_object_keys, env_rv_params, gripper_name, config): """ Generates a GQ-CNN TensorDataset for training models with new grippers, quality metrics, objects, and cameras. Parameters ---------- dataset_path : str path to save the dataset to database : :obj:`Hdf5Database` Dex-Net database containing the 3D meshes, grasps, and grasp metrics target_object_keys : :obj:`OrderedDict` dictionary mapping dataset names to target objects (either 'all' or a list of specific object keys) env_rv_params : :obj:`OrderedDict` parameters of the camera and object random variables used in sampling (see meshpy.UniformPlanarWorksurfaceImageRandomVariable for more info) gripper_name : str name of the gripper to use config : :obj:`autolab_core.YamlConfig` other parameters for dataset generation Notes ----- Required parameters of config are specified in Other Parameters Other Parameters ---------------- images_per_stable_pose : int number of object and camera poses to sample for each stable pose stable_pose_min_p : float minimum probability of occurrence for a stable pose to be used in data generation (used to prune bad stable poses gqcnn/crop_width : int width, in pixels, of crop region around each grasp center, before resize (changes the size of the region seen by the GQ-CNN) gqcnn/crop_height : int height, in pixels, of crop region around each grasp center, before resize (changes the size of the region seen by the GQ-CNN) gqcnn/final_width : int width, in pixels, of final transformed grasp image for input to the GQ-CNN (defaults to 32) gqcnn/final_height : int height, in pixels, of final transformed grasp image for input to the GQ-CNN (defaults to 32) table_alignment/max_approach_table_angle : float max angle between the grasp axis and the table normal when the grasp approach is maximally aligned with the table normal table_alignment/max_approach_offset : float max deviation from perpendicular approach direction to use in grasp collision checking table_alignment/num_approach_offset_samples : int number of approach samples to use in collision checking collision_checking/table_offset : float max allowable interpenetration between the gripper and table to be considered collision free collision_checking/table_mesh_filename : str path to a table mesh for collision checking (default data/meshes/table.obj) collision_checking/approach_dist : float distance, in meters, between the approach pose and final grasp pose along the grasp axis collision_checking/delta_approach : float amount, in meters, to discretize the straight-line path from the gripper approach pose to the final grasp pose tensors/datapoints_per_file : int number of datapoints to store in each unique tensor file on disk tensors/fields : :obj:`dict` dictionary mapping field names to dictionaries specifying the data type, height, width, and number of channels for each tensor debug : bool True (or 1) if the random seed should be set to enforce deterministic behavior, False (0) otherwise vis/candidate_grasps : bool True (or 1) if the collision free candidate grasps should be displayed in 3D (for debugging) vis/rendered_images : bool True (or 1) if the rendered images for each stable pose should be displayed (for debugging) vis/grasp_images : bool True (or 1) if the transformed grasp images should be displayed (for debugging) """ # read data gen params output_dir = dataset_path gripper = RobotGripper.load(gripper_name) image_samples_per_stable_pose = config['images_per_stable_pose'] stable_pose_min_p = config['stable_pose_min_p'] # read gqcnn params gqcnn_params = config['gqcnn'] im_crop_height = gqcnn_params['crop_height'] im_crop_width = gqcnn_params['crop_width'] im_final_height = gqcnn_params['final_height'] im_final_width = gqcnn_params['final_width'] cx_crop = float(im_crop_width) / 2 cy_crop = float(im_crop_height) / 2 # open database dataset_names = target_object_keys.keys() datasets = [database.dataset(dn) for dn in dataset_names] # set target objects for dataset in datasets: if target_object_keys[dataset.name] == 'all': target_object_keys[dataset.name] = dataset.object_keys # setup grasp params table_alignment_params = config['table_alignment'] min_grasp_approach_offset = -np.deg2rad(table_alignment_params['max_approach_offset']) max_grasp_approach_offset = np.deg2rad(table_alignment_params['max_approach_offset']) max_grasp_approach_table_angle = np.deg2rad(table_alignment_params['max_approach_table_angle']) num_grasp_approach_samples = table_alignment_params['num_approach_offset_samples'] phi_offsets = [] if max_grasp_approach_offset == min_grasp_approach_offset: phi_inc = 1 elif num_grasp_approach_samples == 1: phi_inc = max_grasp_approach_offset - min_grasp_approach_offset + 1 else: phi_inc = (max_grasp_approach_offset - min_grasp_approach_offset) / (num_grasp_approach_samples - 1) phi = min_grasp_approach_offset while phi <= max_grasp_approach_offset: phi_offsets.append(phi) phi += phi_inc # setup collision checking coll_check_params = config['collision_checking'] approach_dist = coll_check_params['approach_dist'] delta_approach = coll_check_params['delta_approach'] table_offset = coll_check_params['table_offset'] table_mesh_filename = coll_check_params['table_mesh_filename'] if not os.path.isabs(table_mesh_filename): table_mesh_filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', table_mesh_filename) table_mesh = ObjFile(table_mesh_filename).read() # set tensor dataset config tensor_config = config['tensors'] tensor_config['fields']['depth_ims_tf_table']['height'] = im_final_height tensor_config['fields']['depth_ims_tf_table']['width'] = im_final_width tensor_config['fields']['obj_masks']['height'] = im_final_height tensor_config['fields']['obj_masks']['width'] = im_final_width # add available metrics (assuming same are computed for all objects) metric_names = [] dataset = datasets[0] obj_keys = dataset.object_keys if len(obj_keys) == 0: raise ValueError('No valid objects in dataset %s' %(dataset.name)) obj = dataset[obj_keys[0]] grasps = dataset.grasps(obj.key, gripper=gripper.name) grasp_metrics = dataset.grasp_metrics(obj.key, grasps, gripper=gripper.name) metric_names = grasp_metrics[grasp_metrics.keys()[0]].keys() for metric_name in metric_names: tensor_config['fields'][metric_name] = {} tensor_config['fields'][metric_name]['dtype'] = 'float32' # init tensor dataset tensor_dataset = TensorDataset(output_dir, tensor_config) tensor_datapoint = tensor_dataset.datapoint_template # setup log file experiment_log_filename = os.path.join(output_dir, 'dataset_generation.log') formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s') hdlr = logging.FileHandler(experiment_log_filename) hdlr.setFormatter(formatter) logging.getLogger().addHandler(hdlr) root_logger = logging.getLogger() # copy config out_config_filename = os.path.join(output_dir, 'dataset_generation.json') ordered_dict_config = collections.OrderedDict() for key in config.keys(): ordered_dict_config[key] = config[key] with open(out_config_filename, 'w') as outfile: json.dump(ordered_dict_config, outfile) # 1. Precompute the set of valid grasps for each stable pose: # i) Perpendicular to the table # ii) Collision-free along the approach direction # load grasps if they already exist grasp_cache_filename = os.path.join(output_dir, CACHE_FILENAME) if os.path.exists(grasp_cache_filename): logging.info('Loading grasp candidates from file') candidate_grasps_dict = pkl.load(open(grasp_cache_filename, 'rb')) # otherwise re-compute by reading from the database and enforcing constraints else: # create grasps dict candidate_grasps_dict = {} # loop through datasets and objects for dataset in datasets: logging.info('Reading dataset %s' %(dataset.name)) for obj in dataset: if obj.key not in target_object_keys[dataset.name]: continue # init candidate grasp storage candidate_grasps_dict[obj.key] = {} # setup collision checker collision_checker = GraspCollisionChecker(gripper) collision_checker.set_graspable_object(obj) # read in the stable poses of the mesh stable_poses = dataset.stable_poses(obj.key) for i, stable_pose in enumerate(stable_poses): # render images if stable pose is valid if stable_pose.p > stable_pose_min_p: candidate_grasps_dict[obj.key][stable_pose.id] = [] # setup table in collision checker T_obj_stp = stable_pose.T_obj_table.as_frames('obj', 'stp') T_obj_table = obj.mesh.get_T_surface_obj(T_obj_stp, delta=table_offset).as_frames('obj', 'table') T_table_obj = T_obj_table.inverse() collision_checker.set_table(table_mesh_filename, T_table_obj) # read grasp and metrics grasps = dataset.grasps(obj.key, gripper=gripper.name) logging.info('Aligning %d grasps for object %s in stable %s' %(len(grasps), obj.key, stable_pose.id)) # align grasps with the table aligned_grasps = [grasp.perpendicular_table(stable_pose) for grasp in grasps] # check grasp validity logging.info('Checking collisions for %d grasps for object %s in stable %s' %(len(grasps), obj.key, stable_pose.id)) for aligned_grasp in aligned_grasps: # check angle with table plane and skip unaligned grasps _, grasp_approach_table_angle, _ = aligned_grasp.grasp_angles_from_stp_z(stable_pose) perpendicular_table = (np.abs(grasp_approach_table_angle) < max_grasp_approach_table_angle) if not perpendicular_table: continue # check whether any valid approach directions are collision free collision_free = False for phi_offset in phi_offsets: rotated_grasp = aligned_grasp.grasp_y_axis_offset(phi_offset) collides = collision_checker.collides_along_approach(rotated_grasp, approach_dist, delta_approach) if not collides: collision_free = True break # store if aligned to table candidate_grasps_dict[obj.key][stable_pose.id].append(GraspInfo(aligned_grasp, collision_free)) # visualize if specified if collision_free and config['vis']['candidate_grasps']: logging.info('Grasp %d' %(aligned_grasp.id)) vis.figure() vis.gripper_on_object(gripper, aligned_grasp, obj, stable_pose.T_obj_world) vis.show() # save to file logging.info('Saving to file') pkl.dump(candidate_grasps_dict, open(grasp_cache_filename, 'wb')) # 2. Render a dataset of images and associate the gripper pose with image coordinates for each grasp in the Dex-Net database # setup variables obj_category_map = {} pose_category_map = {} cur_pose_label = 0 cur_obj_label = 0 cur_image_label = 0 # render images for each stable pose of each object in the dataset render_modes = [RenderMode.SEGMASK, RenderMode.DEPTH_SCENE] for dataset in datasets: logging.info('Generating data for dataset %s' %(dataset.name)) # iterate through all object keys object_keys = dataset.object_keys for obj_key in object_keys: obj = dataset[obj_key] if obj.key not in target_object_keys[dataset.name]: continue # read in the stable poses of the mesh stable_poses = dataset.stable_poses(obj.key) for i, stable_pose in enumerate(stable_poses): # render images if stable pose is valid if stable_pose.p > stable_pose_min_p: # log progress logging.info('Rendering images for object %s in %s' %(obj.key, stable_pose.id)) # add to category maps if obj.key not in obj_category_map.keys(): obj_category_map[obj.key] = cur_obj_label pose_category_map['%s_%s' %(obj.key, stable_pose.id)] = cur_pose_label # read in candidate grasps and metrics candidate_grasp_info = candidate_grasps_dict[obj.key][stable_pose.id] candidate_grasps = [g.grasp for g in candidate_grasp_info] grasp_metrics = dataset.grasp_metrics(obj.key, candidate_grasps, gripper=gripper.name) # compute object pose relative to the table T_obj_stp = stable_pose.T_obj_table.as_frames('obj', 'stp') T_obj_stp = obj.mesh.get_T_surface_obj(T_obj_stp) # sample images from random variable T_table_obj = RigidTransform(from_frame='table', to_frame='obj') scene_objs = {'table': SceneObject(table_mesh, T_table_obj)} urv = UniformPlanarWorksurfaceImageRandomVariable(obj.mesh, render_modes, 'camera', env_rv_params, stable_pose=stable_pose, scene_objs=scene_objs) render_start = time.time() render_samples = urv.rvs(size=image_samples_per_stable_pose) render_stop = time.time() logging.info('Rendering images took %.3f sec' %(render_stop - render_start)) # visualize if config['vis']['rendered_images']: d = int(np.ceil(np.sqrt(image_samples_per_stable_pose))) # binary vis2d.figure() for j, render_sample in enumerate(render_samples): vis2d.subplot(d,d,j+1) vis2d.imshow(render_sample.renders[RenderMode.SEGMASK].image) # depth table vis2d.figure() for j, render_sample in enumerate(render_samples): vis2d.subplot(d,d,j+1) vis2d.imshow(render_sample.renders[RenderMode.DEPTH_SCENE].image) vis2d.show() # tally total amount of data num_grasps = len(candidate_grasps) num_images = image_samples_per_stable_pose num_save = num_images * num_grasps logging.info('Saving %d datapoints' %(num_save)) # for each candidate grasp on the object compute the projection # of the grasp into image space for render_sample in render_samples: # read images binary_im = render_sample.renders[RenderMode.SEGMASK].image depth_im_table = render_sample.renders[RenderMode.DEPTH_SCENE].image # read camera params T_stp_camera = render_sample.camera.object_to_camera_pose shifted_camera_intr = render_sample.camera.camera_intr # read pixel offsets cx = depth_im_table.center[1] cy = depth_im_table.center[0] # compute intrinsics for virtual camera of the final # cropped and rescaled images camera_intr_scale = float(im_final_height) / float(im_crop_height) cropped_camera_intr = shifted_camera_intr.crop(im_crop_height, im_crop_width, cy, cx) final_camera_intr = cropped_camera_intr.resize(camera_intr_scale) # create a thumbnail for each grasp for grasp_info in candidate_grasp_info: # read info grasp = grasp_info.grasp collision_free = grasp_info.collision_free # get the gripper pose T_obj_camera = T_stp_camera * T_obj_stp.as_frames('obj', T_stp_camera.from_frame) grasp_2d = grasp.project_camera(T_obj_camera, shifted_camera_intr) # center images on the grasp, rotate to image x axis dx = cx - grasp_2d.center.x dy = cy - grasp_2d.center.y translation = np.array([dy, dx]) binary_im_tf = binary_im.transform(translation, grasp_2d.angle) depth_im_tf_table = depth_im_table.transform(translation, grasp_2d.angle) # crop to image size binary_im_tf = binary_im_tf.crop(im_crop_height, im_crop_width) depth_im_tf_table = depth_im_tf_table.crop(im_crop_height, im_crop_width) # resize to image size binary_im_tf = binary_im_tf.resize((im_final_height, im_final_width), interp='nearest') depth_im_tf_table = depth_im_tf_table.resize((im_final_height, im_final_width)) # visualize the transformed images if config['vis']['grasp_images']: grasp_center = Point(depth_im_tf_table.center, frame=final_camera_intr.frame) tf_grasp_2d = Grasp2D(grasp_center, 0, grasp_2d.depth, width=gripper.max_width, camera_intr=final_camera_intr) # plot 2D grasp image vis2d.figure() vis2d.subplot(2,2,1) vis2d.imshow(binary_im) vis2d.grasp(grasp_2d) vis2d.subplot(2,2,2) vis2d.imshow(depth_im_table) vis2d.grasp(grasp_2d) vis2d.subplot(2,2,3) vis2d.imshow(binary_im_tf) vis2d.grasp(tf_grasp_2d) vis2d.subplot(2,2,4) vis2d.imshow(depth_im_tf_table) vis2d.grasp(tf_grasp_2d) vis2d.title('Coll Free? %d'%(grasp_info.collision_free)) vis2d.show() # plot 3D visualization vis.figure() T_obj_world = vis.mesh_stable_pose(obj.mesh, stable_pose.T_obj_world, style='surface', dim=0.5) vis.gripper(gripper, grasp, T_obj_world, color=(0.3,0.3,0.3)) vis.show() # form hand pose array hand_pose = np.r_[grasp_2d.center.y, grasp_2d.center.x, grasp_2d.depth, grasp_2d.angle, grasp_2d.center.y - shifted_camera_intr.cy, grasp_2d.center.x - shifted_camera_intr.cx, grasp_2d.width_px] # store to data buffers tensor_datapoint['depth_ims_tf_table'] = depth_im_tf_table.raw_data tensor_datapoint['obj_masks'] = binary_im_tf.raw_data tensor_datapoint['hand_poses'] = hand_pose tensor_datapoint['collision_free'] = collision_free tensor_datapoint['obj_labels'] = cur_obj_label tensor_datapoint['pose_labels'] = cur_pose_label tensor_datapoint['image_labels'] = cur_image_label for metric_name, metric_val in grasp_metrics[grasp.id].iteritems(): coll_free_metric = (1 * collision_free) * metric_val tensor_datapoint[metric_name] = coll_free_metric tensor_dataset.add(tensor_datapoint) # update image label cur_image_label += 1 # update pose label cur_pose_label += 1 # force clean up gc.collect() # update object label cur_obj_label += 1 # force clean up gc.collect() # save last file tensor_dataset.flush() # save category mappings obj_cat_filename = os.path.join(output_dir, 'object_category_map.json') json.dump(obj_category_map, open(obj_cat_filename, 'w')) pose_cat_filename = os.path.join(output_dir, 'pose_category_map.json') json.dump(pose_category_map, open(pose_cat_filename, 'w'))
push_idx = None # push_idx = 33 sample_id = 0 num_toppled, pose_angles, actual_poses = [], [], [] print '\n\n\n\n\n\nSTARTING DIFFERENT PUSH', push_num while sample_id < 10: sample_id += 1 # sim_point_cloud_masked = get_sim_point_cloud(depth_scene, env.state.obj) usr_input = 'n' while usr_input != 'y' and usr_input != 'a': usr_input = utils.keyboard_input('\n\nPut the object in position y: continue, v: vis pose, a: abort push[y/v/a]:') if usr_input == 'v': vis3d.figure() env.state.obj.T_obj_world = orig_pose env.render_3d_scene() vis3d.show(starting_camera_pose=CAMERA_POSE) if usr_input == 'a': print 'trying different push' push_idx = None sample_id = 0 num_toppled, pose_angles, actual_poses = [], [], [] continue usr_input = 'n' while usr_input != 'y': s2r = sim_to_real_tf(sim_point_cloud_masked, vis=True) usr_input = utils.keyboard_input('Did the pose registration work? [y/n]:') env.state.obj.T_obj_world = s2r*orig_pose # Plan topple action action = policy.action(env.state, push_idx)
def showgraspfrompicklewithcandidate(self, object_name,metric_name,onlynagative,onlypositive,openravechecker): object = self.dataset[object_name] labelgrasps,labelmetrics,candidate_labelgrasps,candidate_labelmetrics=self.readdatafrompicklewithcandidate(object_name) config=self._get_config(None) low =0.0 #np.min(metrics) high = np.max([labelmetrics[i][metric_name] for i in range(len(labelmetrics))]) # print 'high',high,config['quality_scale'] if low == high: q_to_c = lambda quality: config['quality_scale'] else: q_to_c = lambda quality: config['quality_scale'] * (quality - low) / (high - low) recalculate_grasp=[] vis.figure() vis.mesh(object.mesh.trimesh ,style='surface') for i in range(len(labelgrasps)): if labelmetrics[i][metric_name]==0 : if not onlypositive: vis.figure() vis.mesh(object.mesh.trimesh ,style='surface') color = plt.get_cmap('hsv')(q_to_c(labelmetrics[i][metric_name]))[:-1] vis.graspwithapproachvectorusingcenter_point(labelgrasps[i] ,approaching_color=(1,0,0), grasp_axis_color=(1,0,0) ) for kk in range(len(candidate_labelgrasps[i] )): print 'candidate_labelgrasps[i][kk]',candidate_labelgrasps[i][kk] color = plt.get_cmap('hsv')(q_to_c(candidate_labelmetrics[i][candidate_labelgrasps[i][kk].id][metric_name]))[:-1] vis.graspwithapproachvectorusingcenter_point(candidate_labelgrasps[i][kk] ,approaching_color=color, grasp_axis_color=color ) recalculate_grasp.append(labelgrasps[i]) print 'maxtrics',i,labelmetrics[i][metric_name] # ,high vis.pose(RigidTransform(), alpha=0.1) vis.show(animate=False) elif labelmetrics[i][metric_name]==-1 : if not onlypositive: vis.shownormals(labelgrasps[i][1],labelgrasps[i][0],color=(0,0,1) ) else: if not onlynagative: vis.figure() vis.mesh(object.mesh.trimesh ,style='surface') color = plt.get_cmap('hsv')(q_to_c(labelmetrics[i][metric_name]))[:-1] if len(candidate_labelgrasps[i])>0: A=np.array(labelgrasps[i].rotated_full_axis[:,0] ) B=np.array(candidate_labelgrasps[i][0].rotated_full_axis[:,0] ) num = np.dot(A.T, B) print num if num<0.99: labelgrasps[i].approach_angle_= labelgrasps[i]._angle_aligned_with_table( -candidate_labelgrasps[i][0].rotated_full_axis[:,0]) vis.graspwithapproachvectorusingcenter_point(labelgrasps[i] ,approaching_color=color, grasp_axis_color=(0,0,1) ) print 'length',len(candidate_labelgrasps[i] ) for kk in range(len(candidate_labelgrasps[i] )): color = plt.get_cmap('hsv')(q_to_c(candidate_labelmetrics[i][candidate_labelgrasps[i][kk].id][metric_name]))[:-1] vis.graspwithapproachvectorusingcenter_point(candidate_labelgrasps[i][kk] ,approaching_color=(0,1,1), grasp_axis_color=color ) recalculate_grasp.append(labelgrasps[i]) print 'maxtrics',i,labelmetrics[i][metric_name] # ,high vis.pose(RigidTransform(), alpha=0.1) vis.show(animate=False)
break # Load grasp metrics grasp_metrics = dataset.grasp_metrics(obj.key, aligned_grasps, gripper=gripper.name) # Project grasp coordinates in image grasp_2d = aligned_grasp.project_camera(T_obj_camera, shifted_camera_intr) if VISUALISE_3D: vis.figure() T_obj_world = vis.mesh_stable_pose(obj.mesh.trimesh, stable_pose.T_obj_world, style='surface', dim=0.5) T_camera_world = T_obj_world * T_obj_camera.inverse() vis.gripper(gripper, aligned_grasp, T_obj_world, color=(0.3, 0.3, 0.3), T_camera_world=T_camera_world) vis.show() # Get translation image distances to grasp dx = cx - grasp_2d.center.x dy = cy - grasp_2d.center.y translation = np.array([dy, dx]) # Transform, crop and resize image binary_im_tf = binary_im.transform(translation, grasp_2d.angle) depth_im_tf_table = depth_im_table.transform(translation, grasp_2d.angle) binary_im_tf = binary_im_tf.crop(96, 96) depth_im_tf_table = depth_im_tf_table.crop(96, 96) depth_image = np.asarray(depth_im_tf_table.data) dep_image = Image.fromarray(depth_image).resize((32, 32), resample=Image.BILINEAR)
def save_samples(self, sample, grasps, collisions, T_obj_stp, grasp_metrics, stable_pose): self.tensor_datapoint['image_labels'] = self.cur_image_label T_stp_camera = sample.camera.object_to_camera_pose self.T_obj_camera = T_stp_camera * T_obj_stp.as_frames( 'obj', T_stp_camera.from_frame) depth_im_table = sample.renders[RenderMode.DEPTH_SCENE].image camera_pose = self.get_camera_pose(sample) final_camera_intr = sample.camera.camera_intr.crop( self.crop_height, self.crop_width, depth_im_table.center[0], depth_im_table.center[1]) self.tensor_datapoint['camera_intrs'] = np.r_[final_camera_intr.fx, final_camera_intr.fy, final_camera_intr.cx, final_camera_intr.cy] self.save_image(depth_im_table) for aligned_grasp, collision_free in zip(grasps, collisions): # Project grasp coordinates in image grasp_point = aligned_grasp.close_fingers(self.obj, vis=False, check_approach=False) if not grasp_point[0]: Warning("Could not find contact points") continue # Get grasp width in pixel from endpoints p_1 = np.dot(self.T_obj_camera.matrix, np.append(grasp_point[1][0].point, 1).T).T[:3] p_2 = np.dot(self.T_obj_camera.matrix, np.append(grasp_point[1][1].point, 1).T).T[:3] grasp_width_px = self.width_px(final_camera_intr, p_1, p_2) grasp_width = aligned_grasp.width_from_endpoints( grasp_point[1][0].point, grasp_point[1][1].point) if VISUALISE_3D: c = np.array((1, 0, 0)) vis.figure() T_obj_world = vis.mesh_stable_pose(self.obj.mesh.trimesh, stable_pose.T_obj_world, style='surface', dim=0.5, plot_table=True) points = np.array( (np.append(grasp_point[1][0].point, 1), np.append(grasp_point[1][1].point, 1))) vis.gripper(self.gripper, aligned_grasp, T_obj_world, color=(0.3, 0.3, 0.3), point=points, point_color=c) vis.show() grasp_2d = aligned_grasp.project_camera(self.T_obj_camera, final_camera_intr) grasp_center_x = self.crop_width // 2 - grasp_2d.center.x grasp_center_y = self.crop_height // 2 - grasp_2d.center.y hand_pose = self.get_hand_pose(grasp_center_x, grasp_center_y, grasp_2d, final_camera_intr, grasp_width, grasp_width_px) self.add_datapoint(hand_pose, collision_free, camera_pose, aligned_grasp, grasp_metrics) self.cur_image_label += 1
def antipodal_grasp_sampler(self): #of = ObjFile(OBJ_FILENAME) #sf = SdfFile(SDF_FILENAME) #mesh = of.read() #sdf = sf.read() #obj = GraspableObject3D(sdf, mesh) mass = 1.0 CONFIG['obj_rescaling_type'] = RescalingType.RELATIVE mesh_processor = MeshProcessor(OBJ_FILENAME, CONFIG['cache_dir']) mesh_processor.generate_graspable(CONFIG) mesh = mesh_processor.mesh sdf = mesh_processor.sdf obj = GraspableObject3D(sdf, mesh) gripper = RobotGripper.load(GRIPPER_NAME) ags = AntipodalGraspSampler(gripper, CONFIG) grasps = ags.sample_grasps(obj, num_grasps=100) print(len(grasps)) quality_config = GraspQualityConfigFactory.create_config(CONFIG['metrics']['robust_ferrari_canny']) #quality_config = GraspQualityConfigFactory.create_config(CONFIG['metrics']['force_closure']) quality_fn = GraspQualityFunctionFactory.create_quality_function(obj, quality_config) vis.figure() #vis.points(PointCloud(nparraypoints), scale=0.001, color=np.array([0.5,0.5,0.5])) #vis.plot3d(nparraypoints) metrics = [] vis.mesh(obj.mesh.trimesh, style='surface') for grasp in grasps: success, c = grasp.close_fingers(obj) if success: c1, c2 = c #fn_fc = quality_fn(grasp).quality true_fc = PointGraspMetrics3D.grasp_quality(grasp, obj, quality_config) true_fc = true_fc metrics.append(true_fc) #color = quality_fn(grasp).quality #if (true_fc > 1.0): # true_fc = 1.0 #color = (1.0-true_fc, true_fc, 0.0) low = np.min(metrics) high = np.max(metrics) print(low) print(high) if low == high: q_to_c = lambda quality: CONFIG['quality_scale'] else: q_to_c = lambda quality: CONFIG['quality_scale'] * (quality - low) / (high - low) print(len(metrics)) for grasp, metric in zip(grasps, metrics): color = plt.get_cmap('hsv')(q_to_c(metric))[:-1] print(color) vis.grasp(grasp, grasp_axis_color=color,endpoint_color=color) vis.show(False)
def read_grasps(self, object_name, stable_pose_id, max_grasps=10, visualize=False): # load gripper gripper = RobotGripper.load( GRIPPER_NAME, gripper_dir='/home/silvia/dex-net/data/grippers') # open Dex-Net API dexnet_handle = DexNet() dexnet_handle.open_database(self.database_path) dexnet_handle.open_dataset(self.dataset_name) # read the most robust grasp sorted_grasps, metrics = dexnet_handle.dataset.sorted_grasps( object_name, stable_pose_id=('pose_' + str(stable_pose_id)), metric='force_closure', gripper=gripper.name) if (len(sorted_grasps)) == 0: print('no grasps for this stable pose') if visualize: stable_pose = dexnet_handle.dataset.stable_pose( object_name, stable_pose_id=('pose_' + str(stable_pose_id))) obj = dexnet_handle.dataset.graspable(object_name) vis.figure() T_table_world = RigidTransform(from_frame='table', to_frame='world') T_obj_world = Visualizer3D.mesh_stable_pose( obj.mesh.trimesh, stable_pose.T_obj_world, T_table_world=T_table_world, color=(0.5, 0.5, 0.5), style='surface', plot_table=True, dim=0.15) vis.show(False) return None, None contact_points = map(get_contact_points, sorted_grasps) # ------------------------ VISUALIZATION CODE ------------------------ if visualize: low = np.min(metrics) high = np.max(metrics) if low == high: q_to_c = lambda quality: CONFIG['quality_scale'] else: q_to_c = lambda quality: CONFIG['quality_scale'] * ( quality - low) / (high - low) stable_pose = dexnet_handle.dataset.stable_pose( object_name, stable_pose_id=('pose_' + str(stable_pose_id))) obj = dexnet_handle.dataset.graspable(object_name) vis.figure() T_table_world = RigidTransform(from_frame='table', to_frame='world') T_obj_world = Visualizer3D.mesh_stable_pose( obj.mesh.trimesh, stable_pose.T_obj_world, T_table_world=T_table_world, color=(0.5, 0.5, 0.5), style='surface', plot_table=True, dim=0.15) for grasp, metric in zip(sorted_grasps, metrics): color = plt.get_cmap('hsv')((q_to_c(metric)))[:-1] vis.grasp(grasp, T_obj_world=stable_pose.T_obj_world, grasp_axis_color=color, endpoint_color=color) vis.show(False) # ------------------------ END VISUALIZATION CODE --------------------------- # ------------------------ START COLLISION CHECKING --------------------------- #stable_pose = dexnet_handle.dataset.stable_pose(object_name, stable_pose_id=('pose_'+str(stable_pose_id))) #graspable = dexnet_handle.dataset.graspable(object_name) #cc = GraspCollisionChecker(gripper).set_graspable_object(graspable, stable_pose.T_obj_world) stable_pose_matrix = self.get_stable_pose(object_name, stable_pose_id) # CLOSE DATABASE dexnet_handle.close_database() gripper_poses = [] for grasp in sorted_grasps[:max_grasps]: gripper_pose_matrix = np.eye(4, 4) center_world = np.matmul( stable_pose_matrix, [grasp.center[0], grasp.center[1], grasp.center[2], 1]) axis_world = np.matmul( stable_pose_matrix, [grasp.axis_[0], grasp.axis_[1], grasp.axis_[2], 1]) gripper_angle = math.atan2(axis_world[1], axis_world[0]) gripper_pose_matrix[:3, 3] = center_world[:3] gripper_pose_matrix[0, 0] = math.cos(gripper_angle) gripper_pose_matrix[0, 1] = -math.sin(gripper_angle) gripper_pose_matrix[1, 0] = math.sin(gripper_angle) gripper_pose_matrix[1, 1] = math.cos(gripper_angle) #if visualize: # vis.figure() # vis.gripper_on_object(gripper, grasp, obj, stable_pose=stable_pose.T_obj_world) # vis.show(False) gripper_poses.append(gripper_pose_matrix) return contact_points, gripper_poses
def save_samples(self, sample, grasps, collisions, T_obj_stp, grasp_metrics, stable_pose): self.tensor_datapoint['image_labels'] = self.cur_image_label T_stp_camera = sample.camera.object_to_camera_pose self.T_obj_camera = T_stp_camera * T_obj_stp.as_frames( 'obj', T_stp_camera.from_frame) depth_im_table = sample.renders[RenderMode.DEPTH_SCENE].image camera_pose = self.get_camera_pose(sample) shifted_camera_intr = sample.camera.camera_intr cx = depth_im_table.center[1] cy = depth_im_table.center[0] self.tensor_datapoint['camera_intrs'] = np.r_[shifted_camera_intr.fx, shifted_camera_intr.fy, shifted_camera_intr.cx, shifted_camera_intr.cy] for aligned_grasp, collision_free in zip(grasps, collisions): # Project grasp coordinates in image grasp_point = aligned_grasp.close_fingers(self.obj, vis=False, check_approach=False) if not grasp_point[0]: raise Warning("Could not find contact points") continue grasp_width = aligned_grasp.width_from_endpoints( grasp_point[1][0].point, grasp_point[1][1].point) if VISUALISE_3D: c = np.array((1, 0, 0)) vis.figure() T_obj_world = vis.mesh_stable_pose(self.obj.mesh.trimesh, stable_pose.T_obj_world, style='surface', dim=0.5, plot_table=True) points = np.array( (np.append(grasp_point[1][0].point, 1), np.append(grasp_point[1][1].point, 1))) vis.gripper(self.gripper, aligned_grasp, T_obj_world, color=(0.3, 0.3, 0.3), point=points, point_color=c) vis.show() grasp_2d = aligned_grasp.project_camera(self.T_obj_camera, shifted_camera_intr) depth_im_tf = self.prepare_images(depth_im_table, grasp_2d, cx, cy) hand_pose = self.get_hand_pose(grasp_2d, shifted_camera_intr, grasp_width) self.add_datapoint(depth_im_tf, hand_pose, collision_free, camera_pose, aligned_grasp, grasp_metrics) self.cur_image_label += 1
def antipodal_grasp_sampler(visual=False, debug=False): mass = 1.0 CONFIG['obj_rescaling_type'] = RescalingType.RELATIVE mesh_processor = MeshProcessor(OBJ_FILENAME, CONFIG['cache_dir']) mesh_processor.generate_graspable(CONFIG) mesh = mesh_processor.mesh sdf = mesh_processor.sdf stable_poses = mesh_processor.stable_poses obj = GraspableObject3D(sdf, mesh) stable_pose = stable_poses[0] if visual: vis.figure() T_table_world = RigidTransform(from_frame='table', to_frame='world') T_obj_world = Visualizer3D.mesh_stable_pose(obj.mesh.trimesh, stable_pose.T_obj_world, T_table_world=T_table_world, color=(0.5, 0.5, 0.5), style='surface', plot_table=True, dim=0.15) if debug: print(len(stable_poses)) #for stable_pose in stable_poses: # print(stable_pose.p) # print(stable_pose.r) # print(stable_pose.x0) # print(stable_pose.face) # print(stable_pose.id) # glass = 22 is standing straight if debug: print(stable_pose.r) print(stable_pose.T_obj_world) gripper = RobotGripper.load( GRIPPER_NAME, gripper_dir='/home/silvia/dex-net/data/grippers') ags = AntipodalGraspSampler(gripper, CONFIG) stable_pose.id = 0 grasps = ags.generate_grasps(obj, target_num_grasps=20, max_iter=5, stable_pose=stable_pose.r) quality_config = GraspQualityConfigFactory.create_config( CONFIG['metrics']['robust_ferrari_canny']) metrics = [] result = [] #grasps = map(lambda g : g.perpendicular_table(stable_pose), grasps) for grasp in grasps: c1, c2 = grasp.endpoints true_fc = PointGraspMetrics3D.grasp_quality(grasp, obj, quality_config) metrics.append(true_fc) result.append((c1, c2)) if debug: success, c = grasp.close_fingers(obj) if success: c1, c2 = c print("Grasp:") print(c1.point) print(c2.point) print(true_fc) low = np.min(metrics) high = np.max(metrics) if low == high: q_to_c = lambda quality: CONFIG['quality_scale'] else: q_to_c = lambda quality: CONFIG['quality_scale'] * (quality - low) / ( high - low) if visual: for grasp, metric in zip(grasps, metrics): #grasp2 = grasp.perpendicular_table(stable_pose) #c1, c2 = grasp.endpoints #axis = ParallelJawPtGrasp3D.axis_from_endpoints(c1, c2) #angle = np.dot(np.matmul(stable_pose.r, axis), [1,0,0]) #angle = math.tan(axis[1]/axis[0]) #angle = math.degrees(angle)%360 #print(angle) #print(angle/360.0) color = plt.get_cmap('hsv')(q_to_c(metric))[:-1] vis.grasp(grasp, T_obj_world=T_obj_world, grasp_axis_color=color, endpoint_color=color) #axis = np.array([[0,0,0], point]) #points = [(x[0], x[1], x[2]) for x in axis] #Visualizer3D.plot3d(points, color=(0,0,1), tube_radius=0.002) vis.show(False) pose_matrix = np.eye(4, 4) pose_matrix[:3, :3] = T_obj_world.rotation pose_matrix[:3, 3] = T_obj_world.translation return pose_matrix, result