def _setup(self):
""" Setup for visualization """
# setup logger
logging.getLogger().setLevel(logging.INFO)
logging.info('Setting up for visualization.')
#### read config params ###
# dataset directory
self.data_dir = self.cfg['dataset_dir']
# visualization params
self.display_image_type = self.cfg['display_image_type']
self.font_size = self.cfg['font_size']
self.samples_per_object = self.cfg['samples_per_object']
# analysis params
self.datapoint_type = self.cfg['datapoint_type']
self.image_mode = self.cfg['image_mode']
self.input_data_mode = self.cfg['data_format']
self.target_metric_name = self.cfg['metric_name']
self.metric_thresh = self.cfg['metric_thresh']
self.gripper_width_m = self.cfg['gripper_width_m']
# setup data filenames
self._setup_data_filenames()
# setup shuffled file indices
self._compute_indices()
# load gqcnn
logging.info('Loading GQ-CNN')
self.model_dir = self.cfg['model_dir']
self._gqcnn = GQCNN.load(self.model_dir)
self._gqcnn.open_session()
def __init__(self, config):
# store parameters
self._config = config
self._gripper_width = config['gripper_width']
self._crop_height = config['crop_height']
self._crop_width = config['crop_width']
self._sampling_config = config['sampling']
self._gqcnn_model_dir = config['gqcnn_model']
self._logging_dir = None
if 'logging_dir' in config.keys():
self._logging_dir = config['logging_dir']
self._policy_dir = self._logging_dir
if not os.path.exists(self._logging_dir):
os.mkdir(self._logging_dir)
sampler_type = self._sampling_config['type']
# init grasp sampler
self._grasp_sampler = ImageGraspSamplerFactory.sampler(
sampler_type, self._sampling_config, self._gripper_width)
# init GQ-CNN
self._gqcnn = GQCNN.load(self._gqcnn_model_dir)
# open tensorflow session for gqcnn
self._gqcnn.open_session()
def __init__(self, config):
self.bridge = CvBridge()
sensor = "realsense_r200"
# Connect image topic, default use realsense
img_topic = "/camera/rgb/image_rect_color"
depth_topic = "/camera/depth_registered/sw_registered/image_rect"
camera_info = "/camera/depth_registered/sw_registered/camera_info"
if sensor == "kinect":
img_topic = "/kinect2/qhd/image_color"
depth_topic = "/kinect2/qhd/image_depth_rect"
camera_info = "/kinect2/qhd/camera_info"
self.image_sub = rospy.Subscriber(img_topic, Image,
self.image_callback)
self.depth_sub = rospy.Subscriber(depth_topic, Image,
self.depth_callback)
self.color_image = None
self.depth_image = None
self.camera_info = rospy.wait_for_message(camera_info,
CameraInfo,
timeout=2)
self.bounding_box = BoundingBox()
self.bounding_box.maxX = 4400
self.bounding_box.maxY = 3000
self.bounding_box.minX = -1
self.bounding_box.minY = -1
# store parameters
self._config = config
self._gripper_width = config['policy']['gripper_width']
self._crop_height = config['policy']['crop_height']
self._crop_width = config['policy']['crop_width']
self._sampling_config = config['policy']['sampling']
self._gqcnn_model_dir = config['policy']['gqcnn_model']
self._logging_dir = None
if 'logging_dir' in config.keys():
self._logging_dir = config['logging_dir']
self._policy_dir = self._logging_dir
if not os.path.exists(self._logging_dir):
os.mkdir(self._logging_dir)
sampler_type = self._sampling_config['type']
# init GQ-CNN
self.gqcnn = GQCNN.load(self._gqcnn_model_dir)
self.gqcnn_im_height = self.gqcnn.im_height
self.gqcnn_im_width = self.gqcnn.im_width
self.gqcnn_num_channels = self.gqcnn.num_channels
self.gqcnn_pose_dim = self.gqcnn.pose_dim
# open tensorflow session for gqcnn
self.gqcnn.open_session()
def __init__(self, config):
# store parameters
self._config = config
self._gripper_width = config['gripper_width']
self._crop_height = config['crop_height']
self._crop_width = config['crop_width']
self._sampling_config = config['sampling']
self._gqcnn_model_dir = config['gqcnn_model']
sampler_type = self._sampling_config['type']
# init grasp sampler
self._grasp_sampler = ImageGraspSamplerFactory.sampler(
sampler_type, self._sampling_config, self._gripper_width)
# init GQ-CNN
self._gqcnn = GQCNN.load(self._gqcnn_model_dir)
# open tensorflow session for gqcnn
self._gqcnn.open_session()
def predict():
modelpath = util.getEnv('gqcnn_train_stable_model',
predicates=[os.path.exists],
failIfNot=True)
"""
The images should be specified as an `N`x32x32x1 array and the poses should be specified as an `N`x1 array of depths, where `N` is the number of datapoints to predict.
For an example, load a batch of images from `depth_ims_tf_table_00000.npz` and a batch of corresponding poses from column 2 of `hand_poses_00000.npz` from the Adv-Synth dataset.
pred_p_success = output[:,1]
"""
gqcnn = GQCNN.load(modelpath)
imgs = DatasetSlice(0).getObj(DatasetSlice.obj_depth_ims_tf_table)
img = imgs[0]
poses = DatasetSlice(0).getObj(DatasetSlice.obj_hand_poses)
pose = poses[0]
# prediction = gqcnn.predict(np.array([img]), np.array([pose[1]]))
print(poses[:, 1].shape)
# sys.exit()
prediction = gqcnn.predict(imgs, poses[:, 1])
print('prediction is %s' % prediction)
def _run_predictions(self):
""" Run predictions to use for plotting """
logging.info('Running Predictions')
self.train_class_results = {}
self.val_class_results = {}
for model_name in self.models.keys():
logging.info('Analyzing model %s' % (model_name))
# read in model config
model_subdir = os.path.join(self.model_dir, model_name)
model_config_filename = os.path.join(model_subdir, 'config.json')
with open(model_config_filename) as data_file:
model_config = json.load(data_file)
model_type = self.models[model_name]['type']
model_tag = self.models[model_name]['tag']
split_type = self.models[model_name]['split_type']
# create output dir
model_output_dir = os.path.join(self.output_dir, model_name)
if not os.path.exists(model_output_dir):
os.mkdir(model_output_dir)
# load
logging.info('Loading model %s' % (model_name))
if model_type == 'gqcnn':
model = GQCNN.load(model_subdir)
# load indices based on dataset-split-type
if split_type == 'image_wise':
train_indices_filename = os.path.join(
model_subdir, 'train_indices_image_wise.pkl')
val_indices_filename = os.path.join(
model_subdir, 'val_indices_image_wise.pkl')
elif split_type == 'object_wise':
train_indices_filename = os.path.join(
model_subdir, 'train_indices_object_wise.pkl')
val_indices_filename = os.path.join(
model_subdir, 'val_indices_object_wise.pkl')
elif split_type == 'stable_pose_wise':
train_indices_filename = os.path.join(
model_subdir, 'train_indices_stable_pose_wise.pkl')
val_indices_filename = os.path.join(
model_subdir, 'val_indices_stable_pose_wise.pkl')
model.open_session()
# visualize filters
if self.models[model_name]['vis_conv']:
conv1_filters = model.filters
num_filt = conv1_filters.shape[3]
d = int(np.ceil(np.sqrt(num_filt)))
plt.clf()
for k in range(num_filt):
filt = conv1_filters[:, :, 0, k]
filt = sm.imresize(filt,
5.0,
interp='bilinear',
mode='F')
plt.subplot(d, d, k + 1)
plt.imshow(filt, cmap=plt.cm.gray)
plt.axis('off')
figname = os.path.join(model_output_dir,
'conv1_filters.pdf')
plt.savefig(figname, dpi=dpi)
exit(0)
else:
model = pkl.load(open(os.path.join(model_subdir, 'model.pkl')))
train_indices_filename = os.path.join(model_subdir,
'train_index_map.pkl')
val_indices_filename = os.path.join(model_subdir,
'val_index_map.pkl')
image_mean = np.load(os.path.join(model_subdir, 'mean.npy'))
image_std = np.load(os.path.join(model_subdir, 'std.npy'))
pose_mean = np.load(os.path.join(model_subdir,
'pose_mean.npy'))[2:3]
pose_std = np.load(os.path.join(model_subdir,
'pose_std.npy'))[2:3]
# read in training params
model_training_dataset_dir = model_config['dataset_dir']
model_image_mode = model_config['image_mode']
model_target_metric = model_config['target_metric_name']
model_metric_thresh = model_config['metric_thresh']
model_input_data_mode = model_config['input_data_mode']
# read in training, val indices
train_indices = pkl.load(open(train_indices_filename, 'r'))
val_indices = pkl.load(open(val_indices_filename, 'r'))
# get filenames
filenames = [
os.path.join(model_training_dataset_dir, f)
for f in os.listdir(model_training_dataset_dir)
]
if model_image_mode == ImageMode.BINARY_TF:
im_filenames = [
f for f in filenames
if f.find(binary_im_tf_tensor_template) > -1
]
elif model_image_mode == ImageMode.DEPTH_TF:
im_filenames = [
f for f in filenames
if f.find(depth_im_tf_tensor_template) > -1
]
elif model_image_mode == ImageMode.DEPTH_TF_TABLE:
im_filenames = [
f for f in filenames
if f.find(depth_im_tf_table_tensor_template) > -1
]
else:
raise ValueError('Model image mode %s not recognized' %
(model_image_mode))
pose_filenames = [
f for f in filenames if f.find(hand_poses_template) > -1
]
metric_filenames = [
f for f in filenames if f.find(model_target_metric) > -1
]
# sort filenames for consistency
im_filenames.sort(key=lambda x: int(x[-9:-4]))
pose_filenames.sort(key=lambda x: int(x[-9:-4]))
metric_filenames.sort(key=lambda x: int(x[-9:-4]))
num_files = len(im_filenames)
cur_file_num = 0
evaluation_time = 0
# aggregate training and validation true labels and predicted probabilities
train_preds = []
train_labels = []
val_preds = []
val_labels = []
for im_filename, pose_filename, metric_filename in zip(
im_filenames, pose_filenames, metric_filenames):
if cur_file_num % self.cfg['out_rate'] == 0:
logging.info('Reading file %d of %d' %
(cur_file_num + 1, num_files))
# read data
image_arr = np.load(im_filename)['arr_0']
pose_arr = np.load(pose_filename)['arr_0']
metric_arr = np.load(metric_filename)['arr_0']
labels_arr = 1 * (metric_arr > model_metric_thresh)
num_datapoints = image_arr.shape[0]
if model_type == 'gqcnn':
# slice correct part of pose_arr corresponding to input_data_mode used for training model
if model_input_data_mode == InputDataMode.TF_IMAGE:
pose_arr = pose_arr[:, 2:3]
elif model_input_data_mode == InputDataMode.TF_IMAGE_PERSPECTIVE:
pose_arr = np.c_[pose_arr[:, 2:3], pose_arr[:, 4:6]]
elif model_input_data_mode == InputDataMode.RAW_IMAGE:
pose_arr = pose_arr[:, :4]
elif model_input_data_mode == InputDataMode.RAW_IMAGE_PERSPECTIVE:
pose_arr = pose_arr[:, :6]
else:
raise ValueError('Input data mode %s not supported' %
(model_input_data_mode))
# predict
pred_start = time.time()
if model_type == 'gqcnn':
pred_arr = model.predict(image_arr, pose_arr)
else:
pose_arr = (pose_arr - pose_mean) / pose_std
if 'use_hog' in model_config.keys(
) and model_config['use_hog']:
feature_arr = None
for i in range(num_datapoints):
image = image_arr[i, :, :, 0]
feature_descriptor = hog(
image,
orientations=model_config[
'hog_num_orientations'],
pixels_per_cell=(
model_config['hog_pixels_per_cell'],
model_config['hog_pixels_per_cell']),
cells_per_block=(
model_config['hog_cells_per_block'],
model_config['hog_cells_per_block']))
feature_dim = feature_descriptor.shape[0]
if feature_arr is None:
feature_arr = np.zeros(
[num_datapoints, feature_dim + 1])
feature_arr[i, :] = np.r_[feature_descriptor,
pose_arr[i]]
else:
feature_arr = np.c_[(
(image_arr - image_mean) /
image_std).reshape(num_datapoints, -1),
(pose_arr - pose_mean) / pose_std]
if model_type == 'rf':
pred_arr = model.predict_proba(feature_arr)
elif model_type == 'svm':
pred_arr = model.decision_function(feature_arr)
pred_arr = pred_arr / (2 * np.max(np.abs(pred_arr)))
pred_arr = pred_arr - np.min(pred_arr)
pred_arr = np.c_[1 - pred_arr, pred_arr]
else:
raise ValueError('Model type %s not supported' %
(model_type))
pred_stop = time.time()
evaluation_time += pred_stop - pred_start
# break into training / val
index_im_filename = im_filename
new_train_indices = {}
for key in train_indices.keys():
new_train_indices[os.path.join(model_training_dataset_dir,
key)] = train_indices[key]
train_indices = new_train_indices
new_val_indices = {}
for key in val_indices.keys():
new_val_indices[os.path.join(model_training_dataset_dir,
key)] = val_indices[key]
val_indices = new_val_indices
# IPython.embed()
train_preds.append(pred_arr[train_indices[index_im_filename]])
train_labels.append(
labels_arr[train_indices[index_im_filename]])
val_preds.append(pred_arr[val_indices[index_im_filename]])
val_labels.append(labels_arr[val_indices[index_im_filename]])
cur_file_num += 1
# aggregate results
train_class_result = ClassificationResult(copy.copy(train_preds),
copy.copy(train_labels))
val_class_result = ClassificationResult(copy.copy(val_preds),
copy.copy(val_labels))
self.train_class_results[model_tag] = train_class_result
self.val_class_results[model_tag] = val_class_result
train_class_result.save(
os.path.join(model_output_dir, 'train_class_result.cres'))
val_class_result.save(
os.path.join(model_output_dir, 'val_class_result.cres'))
if model_type == 'gqcnn':
model.close_session()
logging.info('Total evaluation time: %.3f sec' % (evaluation_time))
from autolab_core import YamlConfig
""" Training config templates
cfg/tools/training.yaml
cfg/tools/analyze_gqcnn_performance.yaml
cfg/tools/gqcnn_prediction_visualizer.yaml
"""
print('==============================')
# Configuration for the training
configfile = '/home/simon/sandbox/graspitmod/catkin_ws/src/gqcnn/custom/trainingConfig.yaml'
train_config = YamlConfig(
'/home/simon/sandbox/graspitmod/catkin_ws/src/gqcnn/custom/trainingConfig.yaml'
)
gqcnn_config = train_config['gqcnn_config']
# refine existing...
existing_set = os.environ.get('gqcnn_train_main_modeldir')
if existing_set is not None and os.path.isdir(existing_set) and os.path.exists(
existing_set):
print('loading existing set: ' + existing_set)
gqcnn = GQCNN.load(existing_set)
else:
print('making new set')
gqcnn = GQCNN(gqcnn_config)
SGDOptimizer = SGDOptimizer(gqcnn, train_config)
with gqcnn.get_tf_graph().as_default():
SGDOptimizer.optimize()
type=str,
default=None,
help='dataset to test on')
parser.add_argument('num_rots',
type=int,
default=None,
help='number of rotations to evaluate')
args = parser.parse_args()
model_dir = args.model_dir
dataset_dir = args.dataset_dir
num_rots = args.num_rots
plot = False
# load model
gqcnn = GQCNN.load(model_dir)
# open dataset
file_num = 6300
ims_filename = os.path.join(dataset_dir,
'depth_ims_tf_table_%05d.npz' % (file_num))
poses_filename = os.path.join(dataset_dir,
'hand_poses_%05d.npz' % (file_num))
metrics_filename = os.path.join(
dataset_dir, 'robust_ferrari_canny_%05d.npz' % (file_num))
ims = np.load(ims_filename)['arr_0']
poses = np.load(poses_filename)['arr_0']
metrics = np.load(metrics_filename)['arr_0']
labels = 1 * (metrics > 0.002)
batch_size = 16