def __init__(self, config): # store parameters self._config = config self._gripper_width = config['gripper_width'] self._crop_height = config['crop_height'] self._crop_width = config['crop_width'] self._sampling_config = config['sampling'] self._gqcnn_model_dir = config['gqcnn_model'] self._logging_dir = None if 'logging_dir' in config.keys(): self._logging_dir = config['logging_dir'] self._policy_dir = self._logging_dir if not os.path.exists(self._logging_dir): os.mkdir(self._logging_dir) sampler_type = self._sampling_config['type'] # init grasp sampler self._grasp_sampler = ImageGraspSamplerFactory.sampler( sampler_type, self._sampling_config, self._gripper_width) # init GQ-CNN self._gqcnn = GQCNN.load(self._gqcnn_model_dir) # open tensorflow session for gqcnn self._gqcnn.open_session()
def _setup(self): """ Setup for visualization """ # setup logger logging.getLogger().setLevel(logging.INFO) logging.info('Setting up for visualization.') #### read config params ### # dataset directory self.data_dir = self.cfg['dataset_dir'] # visualization params self.display_image_type = self.cfg['display_image_type'] self.font_size = self.cfg['font_size'] self.samples_per_object = self.cfg['samples_per_object'] # analysis params self.datapoint_type = self.cfg['datapoint_type'] self.image_mode = self.cfg['image_mode'] self.input_data_mode = self.cfg['data_format'] self.target_metric_name = self.cfg['metric_name'] self.metric_thresh = self.cfg['metric_thresh'] self.gripper_width_m = self.cfg['gripper_width_m'] # setup data filenames self._setup_data_filenames() # setup shuffled file indices self._compute_indices() # load gqcnn logging.info('Loading GQ-CNN') self.model_dir = self.cfg['model_dir'] self._gqcnn = GQCNN.load(self.model_dir) self._gqcnn.open_session()
def __init__(self, config): self.bridge = CvBridge() sensor = "realsense_r200" # Connect image topic, default use realsense img_topic = "/camera/rgb/image_rect_color" depth_topic = "/camera/depth_registered/sw_registered/image_rect" camera_info = "/camera/depth_registered/sw_registered/camera_info" if sensor == "kinect": img_topic = "/kinect2/qhd/image_color" depth_topic = "/kinect2/qhd/image_depth_rect" camera_info = "/kinect2/qhd/camera_info" self.image_sub = rospy.Subscriber(img_topic, Image, self.image_callback) self.depth_sub = rospy.Subscriber(depth_topic, Image, self.depth_callback) self.color_image = None self.depth_image = None self.camera_info = rospy.wait_for_message(camera_info, CameraInfo, timeout=2) self.bounding_box = BoundingBox() self.bounding_box.maxX = 4400 self.bounding_box.maxY = 3000 self.bounding_box.minX = -1 self.bounding_box.minY = -1 # store parameters self._config = config self._gripper_width = config['policy']['gripper_width'] self._crop_height = config['policy']['crop_height'] self._crop_width = config['policy']['crop_width'] self._sampling_config = config['policy']['sampling'] self._gqcnn_model_dir = config['policy']['gqcnn_model'] self._logging_dir = None if 'logging_dir' in config.keys(): self._logging_dir = config['logging_dir'] self._policy_dir = self._logging_dir if not os.path.exists(self._logging_dir): os.mkdir(self._logging_dir) sampler_type = self._sampling_config['type'] # init GQ-CNN self.gqcnn = GQCNN.load(self._gqcnn_model_dir) self.gqcnn_im_height = self.gqcnn.im_height self.gqcnn_im_width = self.gqcnn.im_width self.gqcnn_num_channels = self.gqcnn.num_channels self.gqcnn_pose_dim = self.gqcnn.pose_dim # open tensorflow session for gqcnn self.gqcnn.open_session()
def __init__(self, config): # store parameters self._config = config self._gripper_width = config['gripper_width'] self._crop_height = config['crop_height'] self._crop_width = config['crop_width'] self._sampling_config = config['sampling'] self._gqcnn_model_dir = config['gqcnn_model'] sampler_type = self._sampling_config['type'] # init grasp sampler self._grasp_sampler = ImageGraspSamplerFactory.sampler( sampler_type, self._sampling_config, self._gripper_width) # init GQ-CNN self._gqcnn = GQCNN.load(self._gqcnn_model_dir) # open tensorflow session for gqcnn self._gqcnn.open_session()
def predict(): modelpath = util.getEnv('gqcnn_train_stable_model', predicates=[os.path.exists], failIfNot=True) """ The images should be specified as an `N`x32x32x1 array and the poses should be specified as an `N`x1 array of depths, where `N` is the number of datapoints to predict. For an example, load a batch of images from `depth_ims_tf_table_00000.npz` and a batch of corresponding poses from column 2 of `hand_poses_00000.npz` from the Adv-Synth dataset. pred_p_success = output[:,1] """ gqcnn = GQCNN.load(modelpath) imgs = DatasetSlice(0).getObj(DatasetSlice.obj_depth_ims_tf_table) img = imgs[0] poses = DatasetSlice(0).getObj(DatasetSlice.obj_hand_poses) pose = poses[0] # prediction = gqcnn.predict(np.array([img]), np.array([pose[1]])) print(poses[:, 1].shape) # sys.exit() prediction = gqcnn.predict(imgs, poses[:, 1]) print('prediction is %s' % prediction)
gqcnn_config = train_config['gqcnn_config'] def get_elapsed_time(time_in_seconds): """ Helper function to get elapsed time """ if time_in_seconds < 60: return '%.1f seconds' % (time_in_seconds) elif time_in_seconds < 3600: return '%.1f minutes' % (time_in_seconds / 60) else: return '%.1f hours' % (time_in_seconds / 3600) ###Possible Use-Cases### # Training from Scratch start_time = time.time() gqcnn = GQCNN(gqcnn_config) sgdOptimizer = SGDOptimizer(gqcnn, train_config) with gqcnn.get_tf_graph().as_default(): sgdOptimizer.optimize() logging.info('Total Training Time:' + str(get_elapsed_time(time.time() - start_time))) # Prediction """ start_time = time.time() model_dir = '/home/user/Data/models/grasp_quality/model_ewlohgukns' gqcnn = GQCNN.load(model_dir) output = gqcnn.predict(images, poses) pred_p_success = output[:,1] gqcnn.close_session() logging.info('Total Prediction Time:' + str(get_elapsed_time(time.time() - start_time))) """
def _run_predictions(self): """ Run predictions to use for plotting """ logging.info('Running Predictions') self.train_class_results = {} self.val_class_results = {} for model_name in self.models.keys(): logging.info('Analyzing model %s' % (model_name)) # read in model config model_subdir = os.path.join(self.model_dir, model_name) model_config_filename = os.path.join(model_subdir, 'config.json') with open(model_config_filename) as data_file: model_config = json.load(data_file) model_type = self.models[model_name]['type'] model_tag = self.models[model_name]['tag'] split_type = self.models[model_name]['split_type'] # create output dir model_output_dir = os.path.join(self.output_dir, model_name) if not os.path.exists(model_output_dir): os.mkdir(model_output_dir) # load logging.info('Loading model %s' % (model_name)) if model_type == 'gqcnn': model = GQCNN.load(model_subdir) # load indices based on dataset-split-type if split_type == 'image_wise': train_indices_filename = os.path.join( model_subdir, 'train_indices_image_wise.pkl') val_indices_filename = os.path.join( model_subdir, 'val_indices_image_wise.pkl') elif split_type == 'object_wise': train_indices_filename = os.path.join( model_subdir, 'train_indices_object_wise.pkl') val_indices_filename = os.path.join( model_subdir, 'val_indices_object_wise.pkl') elif split_type == 'stable_pose_wise': train_indices_filename = os.path.join( model_subdir, 'train_indices_stable_pose_wise.pkl') val_indices_filename = os.path.join( model_subdir, 'val_indices_stable_pose_wise.pkl') model.open_session() # visualize filters if self.models[model_name]['vis_conv']: conv1_filters = model.filters num_filt = conv1_filters.shape[3] d = int(np.ceil(np.sqrt(num_filt))) plt.clf() for k in range(num_filt): filt = conv1_filters[:, :, 0, k] filt = sm.imresize(filt, 5.0, interp='bilinear', mode='F') plt.subplot(d, d, k + 1) plt.imshow(filt, cmap=plt.cm.gray) plt.axis('off') figname = os.path.join(model_output_dir, 'conv1_filters.pdf') plt.savefig(figname, dpi=dpi) exit(0) else: model = pkl.load(open(os.path.join(model_subdir, 'model.pkl'))) train_indices_filename = os.path.join(model_subdir, 'train_index_map.pkl') val_indices_filename = os.path.join(model_subdir, 'val_index_map.pkl') image_mean = np.load(os.path.join(model_subdir, 'mean.npy')) image_std = np.load(os.path.join(model_subdir, 'std.npy')) pose_mean = np.load(os.path.join(model_subdir, 'pose_mean.npy'))[2:3] pose_std = np.load(os.path.join(model_subdir, 'pose_std.npy'))[2:3] # read in training params model_training_dataset_dir = model_config['dataset_dir'] model_image_mode = model_config['image_mode'] model_target_metric = model_config['target_metric_name'] model_metric_thresh = model_config['metric_thresh'] model_input_data_mode = model_config['input_data_mode'] # read in training, val indices train_indices = pkl.load(open(train_indices_filename, 'r')) val_indices = pkl.load(open(val_indices_filename, 'r')) # get filenames filenames = [ os.path.join(model_training_dataset_dir, f) for f in os.listdir(model_training_dataset_dir) ] if model_image_mode == ImageMode.BINARY_TF: im_filenames = [ f for f in filenames if f.find(binary_im_tf_tensor_template) > -1 ] elif model_image_mode == ImageMode.DEPTH_TF: im_filenames = [ f for f in filenames if f.find(depth_im_tf_tensor_template) > -1 ] elif model_image_mode == ImageMode.DEPTH_TF_TABLE: im_filenames = [ f for f in filenames if f.find(depth_im_tf_table_tensor_template) > -1 ] else: raise ValueError('Model image mode %s not recognized' % (model_image_mode)) pose_filenames = [ f for f in filenames if f.find(hand_poses_template) > -1 ] metric_filenames = [ f for f in filenames if f.find(model_target_metric) > -1 ] # sort filenames for consistency im_filenames.sort(key=lambda x: int(x[-9:-4])) pose_filenames.sort(key=lambda x: int(x[-9:-4])) metric_filenames.sort(key=lambda x: int(x[-9:-4])) num_files = len(im_filenames) cur_file_num = 0 evaluation_time = 0 # aggregate training and validation true labels and predicted probabilities train_preds = [] train_labels = [] val_preds = [] val_labels = [] for im_filename, pose_filename, metric_filename in zip( im_filenames, pose_filenames, metric_filenames): if cur_file_num % self.cfg['out_rate'] == 0: logging.info('Reading file %d of %d' % (cur_file_num + 1, num_files)) # read data image_arr = np.load(im_filename)['arr_0'] pose_arr = np.load(pose_filename)['arr_0'] metric_arr = np.load(metric_filename)['arr_0'] labels_arr = 1 * (metric_arr > model_metric_thresh) num_datapoints = image_arr.shape[0] if model_type == 'gqcnn': # slice correct part of pose_arr corresponding to input_data_mode used for training model if model_input_data_mode == InputDataMode.TF_IMAGE: pose_arr = pose_arr[:, 2:3] elif model_input_data_mode == InputDataMode.TF_IMAGE_PERSPECTIVE: pose_arr = np.c_[pose_arr[:, 2:3], pose_arr[:, 4:6]] elif model_input_data_mode == InputDataMode.RAW_IMAGE: pose_arr = pose_arr[:, :4] elif model_input_data_mode == InputDataMode.RAW_IMAGE_PERSPECTIVE: pose_arr = pose_arr[:, :6] else: raise ValueError('Input data mode %s not supported' % (model_input_data_mode)) # predict pred_start = time.time() if model_type == 'gqcnn': pred_arr = model.predict(image_arr, pose_arr) else: pose_arr = (pose_arr - pose_mean) / pose_std if 'use_hog' in model_config.keys( ) and model_config['use_hog']: feature_arr = None for i in range(num_datapoints): image = image_arr[i, :, :, 0] feature_descriptor = hog( image, orientations=model_config[ 'hog_num_orientations'], pixels_per_cell=( model_config['hog_pixels_per_cell'], model_config['hog_pixels_per_cell']), cells_per_block=( model_config['hog_cells_per_block'], model_config['hog_cells_per_block'])) feature_dim = feature_descriptor.shape[0] if feature_arr is None: feature_arr = np.zeros( [num_datapoints, feature_dim + 1]) feature_arr[i, :] = np.r_[feature_descriptor, pose_arr[i]] else: feature_arr = np.c_[( (image_arr - image_mean) / image_std).reshape(num_datapoints, -1), (pose_arr - pose_mean) / pose_std] if model_type == 'rf': pred_arr = model.predict_proba(feature_arr) elif model_type == 'svm': pred_arr = model.decision_function(feature_arr) pred_arr = pred_arr / (2 * np.max(np.abs(pred_arr))) pred_arr = pred_arr - np.min(pred_arr) pred_arr = np.c_[1 - pred_arr, pred_arr] else: raise ValueError('Model type %s not supported' % (model_type)) pred_stop = time.time() evaluation_time += pred_stop - pred_start # break into training / val index_im_filename = im_filename new_train_indices = {} for key in train_indices.keys(): new_train_indices[os.path.join(model_training_dataset_dir, key)] = train_indices[key] train_indices = new_train_indices new_val_indices = {} for key in val_indices.keys(): new_val_indices[os.path.join(model_training_dataset_dir, key)] = val_indices[key] val_indices = new_val_indices # IPython.embed() train_preds.append(pred_arr[train_indices[index_im_filename]]) train_labels.append( labels_arr[train_indices[index_im_filename]]) val_preds.append(pred_arr[val_indices[index_im_filename]]) val_labels.append(labels_arr[val_indices[index_im_filename]]) cur_file_num += 1 # aggregate results train_class_result = ClassificationResult(copy.copy(train_preds), copy.copy(train_labels)) val_class_result = ClassificationResult(copy.copy(val_preds), copy.copy(val_labels)) self.train_class_results[model_tag] = train_class_result self.val_class_results[model_tag] = val_class_result train_class_result.save( os.path.join(model_output_dir, 'train_class_result.cres')) val_class_result.save( os.path.join(model_output_dir, 'val_class_result.cres')) if model_type == 'gqcnn': model.close_session() logging.info('Total evaluation time: %.3f sec' % (evaluation_time))
from autolab_core import YamlConfig """ Training config templates cfg/tools/training.yaml cfg/tools/analyze_gqcnn_performance.yaml cfg/tools/gqcnn_prediction_visualizer.yaml """ print('==============================') # Configuration for the training configfile = '/home/simon/sandbox/graspitmod/catkin_ws/src/gqcnn/custom/trainingConfig.yaml' train_config = YamlConfig( '/home/simon/sandbox/graspitmod/catkin_ws/src/gqcnn/custom/trainingConfig.yaml' ) gqcnn_config = train_config['gqcnn_config'] # refine existing... existing_set = os.environ.get('gqcnn_train_main_modeldir') if existing_set is not None and os.path.isdir(existing_set) and os.path.exists( existing_set): print('loading existing set: ' + existing_set) gqcnn = GQCNN.load(existing_set) else: print('making new set') gqcnn = GQCNN(gqcnn_config) SGDOptimizer = SGDOptimizer(gqcnn, train_config) with gqcnn.get_tf_graph().as_default(): SGDOptimizer.optimize()
# open train config train_config = YamlConfig(config_filename) train_config['seed'] = seed train_config['tensorboard_port'] = tensorboard_port gqcnn_params = train_config['gqcnn'] # create a unique output folder based on the date and time if save_datetime: # create output dir unique_name = time.strftime("%Y%m%d-%H%M%S") output_dir = os.path.join(output_dir, unique_name) utils.mkdir_safe(output_dir) # set visible devices if 'gpu_list' in train_config: gqcnn_utils.set_cuda_visible_devices(train_config['gpu_list']) # fine-tune the network start_time = time.time() gqcnn = GQCNN(gqcnn_params) optimizer = GQCNNOptimizer(gqcnn, dataset_dir, split_name, output_dir, train_config, name=name) optimizer.finetune(model_dir) logging.info('Total Fine-tuning Time:' + str(utils.get_elapsed_time(time.time() - start_time)))
type=str, default=None, help='dataset to test on') parser.add_argument('num_rots', type=int, default=None, help='number of rotations to evaluate') args = parser.parse_args() model_dir = args.model_dir dataset_dir = args.dataset_dir num_rots = args.num_rots plot = False # load model gqcnn = GQCNN.load(model_dir) # open dataset file_num = 6300 ims_filename = os.path.join(dataset_dir, 'depth_ims_tf_table_%05d.npz' % (file_num)) poses_filename = os.path.join(dataset_dir, 'hand_poses_%05d.npz' % (file_num)) metrics_filename = os.path.join( dataset_dir, 'robust_ferrari_canny_%05d.npz' % (file_num)) ims = np.load(ims_filename)['arr_0'] poses = np.load(poses_filename)['arr_0'] metrics = np.load(metrics_filename)['arr_0'] labels = 1 * (metrics > 0.002) batch_size = 16