def quality(self, state, actions, params): """Evaluate the quality of a set of actions according to a GQ-CNN. Parameters ---------- state : :obj:`RgbdImageState` State of the world described by an RGB-D image. actions: :obj:`object` Set of grasping actions to evaluate. params: dict Optional parameters for quality evaluation. Returns ------- :obj:`list` of float Real-valued grasp quality predictions for each action, between 0 and 1. """ # Form tensors. image_tensor, pose_tensor = self.grasps_to_tensors(actions, state) if params is not None and params["vis"]["tf_images"]: # Read vis params. k = params["vis"]["k"] d = utils.sqrt_ceil(k) # Display grasp transformed images. from visualization import Visualizer2D as vis2d vis2d.figure(size=(GeneralConstants.FIGSIZE, GeneralConstants.FIGSIZE)) for i, image_tf in enumerate(image_tensor[:k, ...]): depth = pose_tensor[i][0] vis2d.subplot(d, d, i + 1) vis2d.imshow(DepthImage(image_tf)) vis2d.title("Image %d: d=%.3f" % (i, depth)) vis2d.show() # Predict grasps. num_actions = len(actions) null_arr = -1 * np.ones(self._batch_size) predict_start = time() output_arr = np.zeros([num_actions, 2]) cur_i = 0 end_i = cur_i + min(self._batch_size, num_actions - cur_i) while cur_i < num_actions: output_arr[cur_i:end_i, :] = self.gqcnn( image_tensor[cur_i:end_i, :, :, 0], pose_tensor[cur_i:end_i, 0], null_arr)[0] cur_i = end_i end_i = cur_i + min(self._batch_size, num_actions - cur_i) q_values = output_arr[:, -1] self._logger.debug("Prediction took %.3f sec" % (time() - predict_start)) return q_values.tolist()
def quality(self, state, actions, params): """ Evaluate the quality of a set of actions according to a GQ-CNN. Parameters ---------- state : :obj:`RgbdImageState` state of the world described by an RGB-D image actions: :obj:`object` set of grasping actions to evaluate params: dict optional parameters for quality evaluation Returns ------- :obj:`list` of float real-valued grasp quality predictions for each action, between 0 and 1 """ # form tensors image_tensor, pose_tensor = self.grasps_to_tensors(actions, state) if params is not None and params['vis']['tf_images']: # read vis params k = params['vis']['k'] d = utils.sqrt_ceil(k) # display grasp transformed images from visualization import Visualizer2D as vis2d vis2d.figure(size=(FIGSIZE, FIGSIZE)) for i, image_tf in enumerate(image_tensor[:k, ...]): depth = pose_tensor[i][0] vis2d.subplot(d, d, i + 1) vis2d.imshow(GrayscaleImage(image_tf)) vis2d.title('Image %d: d=%.3f' % (i, depth)) vis2d.show() # predict grasps num_actions = len(actions) null_arr = -1 * np.ones(self._batch_size) predict_start = time() output_arr = np.zeros([num_actions, 2]) cur_i = 0 end_i = cur_i + min(self._batch_size, num_actions - cur_i) while cur_i < num_actions: output_arr[cur_i:end_i, :] = self.gqcnn( image_tensor[cur_i:end_i, :, :, 0], pose_tensor[cur_i:end_i, 0], null_arr)[0] cur_i = end_i end_i = cur_i + min(self._batch_size, num_actions - cur_i) q_values = output_arr[:, -1] logging.debug('Prediction took %.3f sec' % (time() - predict_start)) return q_values.tolist()
def quality(self, state, actions, params): """Evaluate the quality of a set of actions according to a GQ-CNN. Parameters ---------- state : :obj:`RgbdImageState` State of the world described by an RGB-D image. actions: :obj:`object` Set of grasping actions to evaluate. params: dict Optional parameters for quality evaluation. Returns ------- :obj:`list` of float Real-valued grasp quality predictions for each action, between 0 and 1. """ # Form tensors. tensor_start = time() image_tensor, pose_tensor = self.grasps_to_tensors(actions, state) self._logger.info("Image transformation took %.3f sec" % (time() - tensor_start)) if params is not None and params["vis"]["tf_images"]: # Read vis params. k = params["vis"]["k"] d = utils.sqrt_ceil(k) # Display grasp transformed images. from visualization import Visualizer2D as vis2d vis2d.figure(size=(GeneralConstants.FIGSIZE, GeneralConstants.FIGSIZE)) for i, image_tf in enumerate(image_tensor[:k, ...]): depth = pose_tensor[i][0] vis2d.subplot(d, d, i + 1) vis2d.imshow(DepthImage(image_tf)) vis2d.title("Image %d: d=%.3f" % (i, depth)) vis2d.show() # Predict grasps. predict_start = time() output_arr = self.gqcnn.predict(image_tensor, pose_tensor) q_values = output_arr[:, -1] self._logger.info("Inference took %.3f sec" % (time() - predict_start)) return q_values.tolist()
def quality(self, state, actions, params): """ Evaluate the quality of a set of actions according to a GQ-CNN. Parameters ---------- state : :obj:`RgbdImageState` state of the world described by an RGB-D image actions: :obj:`object` set of grasping actions to evaluate params: dict optional parameters for quality evaluation Returns ------- :obj:`list` of float real-valued grasp quality predictions for each action, between 0 and 1 """ # form tensors tensor_start = time() image_tensor, pose_tensor = self.grasps_to_tensors(actions, state) logging.info('Image transformation took %.3f sec' % (time() - tensor_start)) if params is not None and params['vis']['tf_images']: # read vis params k = params['vis']['k'] d = utils.sqrt_ceil(k) # display grasp transformed images from visualization import Visualizer2D as vis2d vis2d.figure(size=(FIGSIZE, FIGSIZE)) for i, image_tf in enumerate(image_tensor[:k, ...]): depth = pose_tensor[i][0] vis2d.subplot(d, d, i + 1) vis2d.imshow(DepthImage(image_tf)) vis2d.title('Image %d: d=%.3f' % (i, depth)) vis2d.show() # predict grasps predict_start = time() output_arr = self.gqcnn.predict(image_tensor, pose_tensor) q_values = output_arr[:, -1] logging.info('Inference took %.3f sec' % (time() - predict_start)) return q_values.tolist()
def analyze_classification_performance(model_dir, config, dataset_path=None): # read params predict_batch_size = config['batch_size'] randomize = config['randomize'] plotting_config = config['plotting'] figsize = plotting_config['figsize'] font_size = plotting_config['font_size'] legend_font_size = plotting_config['legend_font_size'] line_width = plotting_config['line_width'] colors = plotting_config['colors'] dpi = plotting_config['dpi'] style = '-' class_remapping = None if 'class_remapping' in config.keys(): class_remapping = config['class_remapping'] # read training config training_config_filename = os.path.join(model_dir, 'training_config.yaml') training_config = YamlConfig(training_config_filename) # read training params indices_filename = None if dataset_path is None: dataset_path = training_config['dataset'] indices_filename = os.path.join(model_dir, 'splits.npz') dataset_prefix, dataset_name = os.path.split(dataset_path) if dataset_name == '': _, dataset_name = os.path.split(dataset_prefix) x_names = training_config['x_names'] y_name = training_config['y_name'] batch_size = training_config['training']['batch_size'] iterator_config = training_config['data_iteration'] x_name = x_names[0] # set analysis dir analysis_dir = os.path.join(model_dir, 'analysis') if not os.path.exists(analysis_dir): os.mkdir(analysis_dir) # setup log file experiment_log_filename = os.path.join(analysis_dir, '%s_analysis.log' %(dataset_name)) if os.path.exists(experiment_log_filename): os.remove(experiment_log_filename) formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s') hdlr = logging.FileHandler(experiment_log_filename) hdlr.setFormatter(formatter) logging.getLogger().addHandler(hdlr) # setup plotting plt.figure(figsize=(figsize, figsize)) # read dataset dataset = TensorDataset.open(dataset_path) # read dataset splits if indices_filename is None: splits = {dataset_name: np.arange(dataset.num_datapoints)} else: splits = np.load(indices_filename)['arr_0'].tolist() # load cnn logging.info('Loading model %s' %(model_dir)) cnn = ClassificationCNN.open(model_dir) # save examples logging.info('Saving examples of each class') all_labels = np.arange(cnn.num_classes) label_counts = {} [label_counts.update({l:0}) for l in all_labels] for tensor_ind in range(dataset.num_tensors): tensor = dataset.tensor(y_name, tensor_ind) for label in tensor: label_counts[label] += 1 d = utils.sqrt_ceil(cnn.num_classes) plt.clf() for i, label in enumerate(all_labels): tensor_ind = 0 label_found = False while not label_found and tensor_ind < dataset.num_tensors: tensor = dataset.tensor(y_name, tensor_ind) ind = np.where(tensor.arr == label)[0] if ind.shape[0] > 0: ind = ind[0] + dataset.datapoints_per_tensor * (tensor_ind) label_found = True tensor_ind += 1 if not label_found: continue datapoint = dataset[ind] example_im = datapoint[x_name] plt.subplot(d,d,i+1) plt.imshow(example_im[:,:,:3].astype(np.uint8)) plt.title('Class %03d: %.3f%%' %(label, float(label_counts[label]) / dataset.num_datapoints), fontsize=3) plt.axis('off') plt.savefig(os.path.join(analysis_dir, '%s_classes.pdf' %(dataset_name))) # evaluate on dataset results = {} for split_name, indices in splits.iteritems(): logging.info('Evaluating performance on split: %s' %(split_name)) # predict if randomize: pred_probs, true_labels = cnn.evaluate_on_dataset(dataset, indices=indices, batch_size=predict_batch_size) pred_labels = np.argmax(pred_probs, axis=1) else: true_labels = [] pred_labels = [] pred_probs = [] for datapoint in dataset: im = ColorImage(datapoint['color_ims'].astype(np.uint8)[:,:,:3]) true_label = datapoint['stp_labels'] pred_prob = cnn.predict(im) pred_label = np.argmax(pred_prob, axis=1) true_labels.append(true_label) pred_labels.append(pred_label) pred_probs.append(pred_prob.ravel()) """ if class_remapping is not None: true_label = class_remapping[true_label] plt.figure() plt.imshow(im.raw_data) plt.title('T: %d, P: %d' %(true_label, pred_label)) plt.show() """ true_labels = np.array(true_labels) pred_labels = np.array(pred_labels) pred_probs = np.array(pred_probs) # apply optional class re-mapping if class_remapping is not None: new_true_labels = np.zeros(true_labels.shape) for orig_label, new_label in class_remapping.iteritems(): new_true_labels[true_labels==orig_label] = new_label true_labels = new_true_labels # compute classification results result = ClassificationResult([pred_probs], [true_labels]) results[split_name] = result # print stats logging.info('SPLIT: %s' %(split_name)) logging.info('Acc: %.3f' %(result.accuracy)) logging.info('AP: %.3f' %(result.ap_score)) logging.info('AUC: %.3f' %(result.auc_score)) # save confusion matrix confusion = result.confusion_matrix.data plt.clf() plt.imshow(confusion, cmap=plt.cm.gray, interpolation='none') plt.locator_params(nticks=cnn.num_classes) plt.xlabel('Predicted', fontsize=font_size) plt.ylabel('Actual', fontsize=font_size) plt.savefig(os.path.join(analysis_dir, '%s_confusion.pdf' %(split_name)), dpi=dpi) # save analysis result_filename = os.path.join(analysis_dir, '%s.cres' %(split_name)) result.save(result_filename) # plot colormap = plt.get_cmap('tab10') num_colors = 9 plt.clf() for i, split_name in enumerate(splits.keys()): result = results[split_name] precision, recall, taus = result.precision_recall_curve() color = colormap(float(colors[i%num_colors]) / num_colors) plt.plot(recall, precision, linewidth=line_width, color=color, linestyle=style, label=split_name) plt.xlabel('Recall', fontsize=font_size) plt.ylabel('Precision', fontsize=font_size) plt.title('Precision-Recall Curve', fontsize=font_size) handles, plt_labels = plt.gca().get_legend_handles_labels() plt.legend(handles, plt_labels, loc='best', fontsize=legend_font_size) plt.savefig(os.path.join(analysis_dir, '%s_precision_recall.pdf' %(dataset_name)), dpi=dpi) plt.clf() for i, split_name in enumerate(splits.keys()): result = results[split_name] fpr, tpr, taus = result.roc_curve() color = colormap(float(colors[i%num_colors]) / num_colors) plt.plot(fpr, tpr, linewidth=line_width, color=color, linestyle=style, label=split_name) plt.xlabel('FPR', fontsize=font_size) plt.ylabel('TPR', fontsize=font_size) plt.title('Receiver Operating Characteristic', fontsize=font_size) handles, plt_labels = plt.gca().get_legend_handles_labels() plt.legend(handles, plt_labels, loc='best', fontsize=legend_font_size) plt.savefig(os.path.join(analysis_dir, '%s_roc.pdf' %(dataset_name)), dpi=dpi)
def _run_prediction_single_model(self, model_dir, model_output_dir, dataset_config): """Analyze the performance of a single model.""" # Read in model config. model_config_filename = os.path.join(model_dir, GQCNNFilenames.SAVED_CFG) with open(model_config_filename) as data_file: model_config = json.load(data_file) # Load model. self.logger.info("Loading model %s" % (model_dir)) log_file = None for handler in self.logger.handlers: if isinstance(handler, logging.FileHandler): log_file = handler.baseFilename gqcnn = get_gqcnn_model(verbose=self.verbose).load( model_dir, verbose=self.verbose, log_file=log_file) gqcnn.open_session() gripper_mode = gqcnn.gripper_mode angular_bins = gqcnn.angular_bins # Read params from the config. if dataset_config is None: dataset_dir = model_config["dataset_dir"] split_name = model_config["split_name"] image_field_name = model_config["image_field_name"] pose_field_name = model_config["pose_field_name"] metric_name = model_config["target_metric_name"] metric_thresh = model_config["metric_thresh"] else: dataset_dir = dataset_config["dataset_dir"] split_name = dataset_config["split_name"] image_field_name = dataset_config["image_field_name"] pose_field_name = dataset_config["pose_field_name"] metric_name = dataset_config["target_metric_name"] metric_thresh = dataset_config["metric_thresh"] gripper_mode = dataset_config["gripper_mode"] self.logger.info("Loading dataset %s" % (dataset_dir)) dataset = TensorDataset.open(dataset_dir) train_indices, val_indices, _ = dataset.split(split_name) # Visualize conv filters. conv1_filters = gqcnn.filters num_filt = conv1_filters.shape[3] d = utils.sqrt_ceil(num_filt) vis2d.clf() for k in range(num_filt): filt = conv1_filters[:, :, 0, k] vis2d.subplot(d, d, k + 1) vis2d.imshow(DepthImage(filt)) figname = os.path.join(model_output_dir, "conv1_filters.pdf") vis2d.savefig(figname, dpi=self.dpi) # Aggregate training and validation true labels and predicted # probabilities. all_predictions = [] if angular_bins > 0: all_predictions_raw = [] all_labels = [] for i in range(dataset.num_tensors): # Log progress. if i % self.log_rate == 0: self.logger.info("Predicting tensor %d of %d" % (i + 1, dataset.num_tensors)) # Read in data. image_arr = dataset.tensor(image_field_name, i).arr pose_arr = read_pose_data( dataset.tensor(pose_field_name, i).arr, gripper_mode) metric_arr = dataset.tensor(metric_name, i).arr label_arr = 1 * (metric_arr > metric_thresh) label_arr = label_arr.astype(np.uint8) if angular_bins > 0: # Form mask to extract predictions from ground-truth angular # bins. raw_poses = dataset.tensor(pose_field_name, i).arr angles = raw_poses[:, 3] neg_ind = np.where(angles < 0) # TODO(vsatish): These should use the max angle instead. angles = np.abs(angles) % GeneralConstants.PI angles[neg_ind] *= -1 g_90 = np.where(angles > (GeneralConstants.PI / 2)) l_neg_90 = np.where(angles < (-1 * (GeneralConstants.PI / 2))) angles[g_90] -= GeneralConstants.PI angles[l_neg_90] += GeneralConstants.PI # TODO(vsatish): Fix this along with the others. angles *= -1 # Hack to fix reverse angle convention. angles += (GeneralConstants.PI / 2) pred_mask = np.zeros((raw_poses.shape[0], angular_bins * 2), dtype=bool) bin_width = GeneralConstants.PI / angular_bins for i in range(angles.shape[0]): pred_mask[i, int((angles[i] // bin_width) * 2)] = True pred_mask[i, int((angles[i] // bin_width) * 2 + 1)] = True # Predict with GQ-CNN. predictions = gqcnn.predict(image_arr, pose_arr) if angular_bins > 0: raw_predictions = np.array(predictions) predictions = predictions[pred_mask].reshape((-1, 2)) # Aggregate. all_predictions.extend(predictions[:, 1].tolist()) if angular_bins > 0: all_predictions_raw.extend(raw_predictions.tolist()) all_labels.extend(label_arr.tolist()) # Close session. gqcnn.close_session() # Create arrays. all_predictions = np.array(all_predictions) all_labels = np.array(all_labels) train_predictions = all_predictions[train_indices] val_predictions = all_predictions[val_indices] train_labels = all_labels[train_indices] val_labels = all_labels[val_indices] if angular_bins > 0: all_predictions_raw = np.array(all_predictions_raw) train_predictions_raw = all_predictions_raw[train_indices] val_predictions_raw = all_predictions_raw[val_indices] # Aggregate results. train_result = BinaryClassificationResult(train_predictions, train_labels) val_result = BinaryClassificationResult(val_predictions, val_labels) train_result.save(os.path.join(model_output_dir, "train_result.cres")) val_result.save(os.path.join(model_output_dir, "val_result.cres")) # Get stats, plot curves. self.logger.info("Model %s training error rate: %.3f" % (model_dir, train_result.error_rate)) self.logger.info("Model %s validation error rate: %.3f" % (model_dir, val_result.error_rate)) self.logger.info("Model %s training loss: %.3f" % (model_dir, train_result.cross_entropy_loss)) self.logger.info("Model %s validation loss: %.3f" % (model_dir, val_result.cross_entropy_loss)) # Save images. vis2d.figure() example_dir = os.path.join(model_output_dir, "examples") if not os.path.exists(example_dir): os.mkdir(example_dir) # Train. self.logger.info("Saving training examples") train_example_dir = os.path.join(example_dir, "train") if not os.path.exists(train_example_dir): os.mkdir(train_example_dir) # Train TP. true_positive_indices = train_result.true_positive_indices np.random.shuffle(true_positive_indices) true_positive_indices = true_positive_indices[:self.num_vis] for i, j in enumerate(true_positive_indices): k = train_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=train_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title( "Datapoint %d: Pred: %.3f Label: %.3f" % (k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(train_example_dir, "true_positive_%03d.png" % (i))) # Train FP. false_positive_indices = train_result.false_positive_indices np.random.shuffle(false_positive_indices) false_positive_indices = false_positive_indices[:self.num_vis] for i, j in enumerate(false_positive_indices): k = train_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=train_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title( "Datapoint %d: Pred: %.3f Label: %.3f" % (k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(train_example_dir, "false_positive_%03d.png" % (i))) # Train TN. true_negative_indices = train_result.true_negative_indices np.random.shuffle(true_negative_indices) true_negative_indices = true_negative_indices[:self.num_vis] for i, j in enumerate(true_negative_indices): k = train_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=train_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title( "Datapoint %d: Pred: %.3f Label: %.3f" % (k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(train_example_dir, "true_negative_%03d.png" % (i))) # Train TP. false_negative_indices = train_result.false_negative_indices np.random.shuffle(false_negative_indices) false_negative_indices = false_negative_indices[:self.num_vis] for i, j in enumerate(false_negative_indices): k = train_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=train_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title( "Datapoint %d: Pred: %.3f Label: %.3f" % (k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(train_example_dir, "false_negative_%03d.png" % (i))) # Val. self.logger.info("Saving validation examples") val_example_dir = os.path.join(example_dir, "val") if not os.path.exists(val_example_dir): os.mkdir(val_example_dir) # Val TP. true_positive_indices = val_result.true_positive_indices np.random.shuffle(true_positive_indices) true_positive_indices = true_positive_indices[:self.num_vis] for i, j in enumerate(true_positive_indices): k = val_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=val_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title("Datapoint %d: Pred: %.3f Label: %.3f" % (k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(val_example_dir, "true_positive_%03d.png" % (i))) # Val FP. false_positive_indices = val_result.false_positive_indices np.random.shuffle(false_positive_indices) false_positive_indices = false_positive_indices[:self.num_vis] for i, j in enumerate(false_positive_indices): k = val_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=val_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title("Datapoint %d: Pred: %.3f Label: %.3f" % (k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(val_example_dir, "false_positive_%03d.png" % (i))) # Val TN. true_negative_indices = val_result.true_negative_indices np.random.shuffle(true_negative_indices) true_negative_indices = true_negative_indices[:self.num_vis] for i, j in enumerate(true_negative_indices): k = val_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=val_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title("Datapoint %d: Pred: %.3f Label: %.3f" % (k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(val_example_dir, "true_negative_%03d.png" % (i))) # Val TP. false_negative_indices = val_result.false_negative_indices np.random.shuffle(false_negative_indices) false_negative_indices = false_negative_indices[:self.num_vis] for i, j in enumerate(false_negative_indices): k = val_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=val_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title("Datapoint %d: Pred: %.3f Label: %.3f" % (k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(val_example_dir, "false_negative_%03d.png" % (i))) # Save summary stats. train_summary_stats = { "error_rate": train_result.error_rate, "ap_score": train_result.ap_score, "auc_score": train_result.auc_score, "loss": train_result.cross_entropy_loss } train_stats_filename = os.path.join(model_output_dir, "train_stats.json") json.dump(train_summary_stats, open(train_stats_filename, "w"), indent=JSON_INDENT, sort_keys=True) val_summary_stats = { "error_rate": val_result.error_rate, "ap_score": val_result.ap_score, "auc_score": val_result.auc_score, "loss": val_result.cross_entropy_loss } val_stats_filename = os.path.join(model_output_dir, "val_stats.json") json.dump(val_summary_stats, open(val_stats_filename, "w"), indent=JSON_INDENT, sort_keys=True) return train_result, val_result
def action(self, state): """ Plans the grasp with the highest probability of success on the given RGB-D image. Attributes ---------- state : :obj:`RgbdImageState` image to plan grasps on Returns ------- :obj:`ParallelJawGrasp` grasp to execute """ # check valid input if not isinstance(state, RgbdImageState): raise ValueError('Must provide an RGB-D image state.') # parse state rgbd_im = state.rgbd_im camera_intr = state.camera_intr segmask = state.segmask # sample grasps grasps = self._grasp_sampler.sample( rgbd_im, camera_intr, self._num_seed_samples, segmask=segmask, visualize=self.config['vis']['grasp_sampling'], seed=self._seed) num_grasps = len(grasps) if num_grasps == 0: logging.warning('No valid grasps could be found') return None # form tensors image_tensor, pose_tensor = self.grasps_to_tensors(grasps, state) if self.config['vis']['tf_images']: # read vis params k = self.config['vis']['k'] d = utils.sqrt_ceil(k) # display grasp transformed images vis.figure(size=(FIGSIZE, FIGSIZE)) for i, image_tf in enumerate(image_tensor[:k, ...]): depth = pose_tensor[i][0] vis.subplot(d, d, i + 1) vis.imshow(DepthImage(image_tf)) vis.title('Image %d: d=%.3f' % (i, depth)) # display grasp transformed images vis.figure(size=(FIGSIZE, FIGSIZE)) for i in range(d): image_tf = image_tensor[i, ...] depth = pose_tensor[i][0] grasp = grasps[i] vis.subplot(d, 2, 2 * i + 1) vis.imshow(rgbd_im.depth) vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True) vis.title('Grasp %d: d=%.3f' % (i, depth)) vis.subplot(d, 2, 2 * i + 2) vis.imshow(DepthImage(image_tf)) vis.title('TF image %d: d=%.3f' % (i, depth)) vis.show() # iteratively refit and sample for j in range(self._num_iters): logging.debug('CEM iter %d' % (j)) # predict grasps predict_start = time() output_arr = self.gqcnn.predict(image_tensor, pose_tensor) q_values = output_arr[:, -1] logging.debug('Prediction took %.3f sec' % (time() - predict_start)) # sort grasps q_values_and_indices = zip(q_values, np.arange(num_grasps)) q_values_and_indices.sort(key=lambda x: x[0], reverse=True) if self.config['vis']['grasp_candidates']: # display each grasp on the original image, colored by predicted success vis.figure(size=(FIGSIZE, FIGSIZE)) vis.imshow(rgbd_im.depth) for grasp, q in zip(grasps, q_values): vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True, color=plt.cm.RdYlBu(q)) vis.title('Sampled grasps iter %d' % (j)) vis.show() if self.config['vis']['grasp_ranking']: # read vis params k = self.config['vis']['k'] d = utils.sqrt_ceil(k) # form camera intr for the thumbnail (to compute gripper width) scale_factor = float(self.gqcnn.im_width) / float( self._crop_width) scaled_camera_intr = camera_intr.resize(scale_factor) vis.figure(size=(FIGSIZE, FIGSIZE)) for i, p in enumerate(q_values_and_indices[:k]): # read stats for grasp q_value = p[0] ind = p[1] depth = pose_tensor[ind][0] image = DepthImage(image_tensor[ind, ...]) grasp = Grasp2D(Point(image.center), 0.0, depth, width=self._gripper_width, camera_intr=scaled_camera_intr) # plot vis.subplot(d, d, i + 1) vis.imshow(image) vis.grasp(grasp, scale=1.5) vis.title('K=%d: d=%.3f, q=%.3f' % (i, depth, q_value)) vis.show() # fit elite set num_refit = max(int(np.ceil(self._gmm_refit_p * num_grasps)), 1) elite_q_values = [i[0] for i in q_values_and_indices[:num_refit]] elite_grasp_indices = [ i[1] for i in q_values_and_indices[:num_refit] ] elite_grasps = [grasps[i] for i in elite_grasp_indices] elite_grasp_arr = np.array([g.feature_vec for g in elite_grasps]) if self.config['vis']['elite_grasps']: # display each grasp on the original image, colored by predicted success vis.figure(size=(FIGSIZE, FIGSIZE)) vis.imshow(rgbd_im.depth) for grasp, q in zip(elite_grasps, elite_q_values): vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True, color=plt.cm.RdYlBu(q)) vis.title('Elite grasps iter %d' % (j)) vis.show() # normalize elite set elite_grasp_mean = np.mean(elite_grasp_arr, axis=0) elite_grasp_std = np.std(elite_grasp_arr, axis=0) elite_grasp_std[elite_grasp_std == 0] = 1.0 elite_grasp_arr = (elite_grasp_arr - elite_grasp_mean) / elite_grasp_std # fit a GMM to the top samples num_components = max( int(np.ceil(self._gmm_component_frac * num_refit)), 1) uniform_weights = (1.0 / num_components) * np.ones(num_components) gmm = GaussianMixture(n_components=num_components, weights_init=uniform_weights, reg_covar=self._gmm_reg_covar) train_start = time() gmm.fit(elite_grasp_arr) train_duration = time() - train_start logging.debug('GMM fitting with %d components took %.3f sec' % (num_components, train_duration)) # sample the next grasps sample_start = time() grasp_vecs, _ = gmm.sample(n_samples=self._num_gmm_samples) grasp_vecs = elite_grasp_std * grasp_vecs + elite_grasp_mean sample_duration = time() - sample_start logging.debug('GMM sampling took %.3f sec' % (sample_duration)) # convert features to grasps grasps = [] for grasp_vec in grasp_vecs: grasps.append( Grasp2D.from_feature_vec(grasp_vec, width=self._gripper_width, camera_intr=camera_intr)) num_grasps = len(grasps) if num_grasps == 0: logging.warning('No valid grasps could be found') return None # form tensors image_tensor, pose_tensor = self.grasps_to_tensors(grasps, state) if self.config['vis']['tf_images']: # read vis params k = self.config['vis']['k'] d = utils.sqrt_ceil(k) # display grasp transformed images vis.figure(size=(FIGSIZE, FIGSIZE)) for i, image_tf in enumerate(image_tensor[:k, ...]): depth = pose_tensor[i][0] vis.subplot(d, d, i + 1) vis.imshow(DepthImage(image_tf)) vis.title('Image %d: d=%.3f' % (i, depth)) vis.show() # predict final set of grasps predict_start = time() output_arr = self.gqcnn.predict(image_tensor, pose_tensor) q_values = output_arr[:, -1] logging.debug('Final prediction took %.3f sec' % (time() - predict_start)) if self.config['vis']['grasp_candidates']: # display each grasp on the original image, colored by predicted success vis.figure(size=(FIGSIZE, FIGSIZE)) vis.imshow(rgbd_im.depth) for grasp, q in zip(grasps, q_values): vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True, color=plt.cm.RdYlBu(q)) vis.title('Final sampled grasps') vis.show() # select grasp index = self.select(grasps, q_values) grasp = grasps[index] q_value = q_values[index] image = DepthImage(image_tensor[index, ...]) pose = pose_tensor[index, ...] depth = pose[0] if self.config['vis']['grasp_plan']: scale_factor = float(self.gqcnn.im_width) / float(self._crop_width) scaled_camera_intr = camera_intr.resize(scale_factor) grasp = Grasp2D(Point(image.center), 0.0, pose[0], width=self._gripper_width, camera_intr=scaled_camera_intr) vis.figure() vis.imshow(image) vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True) vis.title('Best Grasp: d=%.3f, q=%.3f' % (depth, q_value)) vis.show() # return action return ParallelJawGrasp(grasp, q_value, image)
def action(self, state): """ Plans the grasp with the highest probability of success on the given RGB-D image. Attributes ---------- state : :obj:`RgbdImageState` image to plan grasps on Returns ------- :obj:`ParallelJawGrasp` grasp to execute """ # check valid input if not isinstance(state, RgbdImageState): raise ValueError('Must provide an RGB-D image state.') # parse state rgbd_im = state.rgbd_im camera_intr = state.camera_intr segmask = state.segmask # sample grasps grasps = self._grasp_sampler.sample( rgbd_im, camera_intr, self._num_grasp_samples, segmask=segmask, visualize=self.config['vis']['grasp_sampling'], seed=None) num_grasps = len(grasps) # form tensors image_tensor, pose_tensor = self.grasps_to_tensors(grasps, state) if self.config['vis']['tf_images']: # read vis params k = self.config['vis']['k'] d = utils.sqrt_ceil(k) # display grasp transformed images vis.figure(size=(FIGSIZE, FIGSIZE)) for i, image_tf in enumerate(image_tensor[:k, ...]): depth = pose_tensor[i][0] vis.subplot(d, d, i + 1) vis.imshow(DepthImage(image_tf)) vis.title('Image %d: d=%.3f' % (i, depth)) vis.show() # predict grasps predict_start = time() output_arr = self.gqcnn.predict(image_tensor, pose_tensor) q_values = output_arr[:, -1] logging.debug('Prediction took %.3f sec' % (time() - predict_start)) if self.config['vis']['grasp_candidates']: # display each grasp on the original image, colored by predicted success vis.figure(size=(FIGSIZE, FIGSIZE)) vis.imshow(rgbd_im.depth) for grasp, q in zip(grasps, q_values): vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True, color=plt.cm.RdYlBu(q)) vis.title('Sampled grasps') vis.show() if self.config['vis']['grasp_ranking']: # read vis params k = self.config['vis']['k'] d = utils.sqrt_ceil(k) # form camera intr for the thumbnail (to compute gripper width) scale_factor = float(self.gqcnn.im_width) / float(self._crop_width) scaled_camera_intr = camera_intr.resize(scale_factor) # sort grasps q_values_and_indices = zip(q_values, np.arange(num_grasps)) q_values_and_indices.sort(key=lambda x: x[0], reverse=True) vis.figure(size=(FIGSIZE, FIGSIZE)) for i, p in enumerate(q_values_and_indices[:k]): # read stats for grasp q_value = p[0] ind = p[1] depth = pose_tensor[ind][0] image = DepthImage(image_tensor[ind, ...]) grasp = Grasp2D(Point(image.center), 0.0, depth, width=self._gripper_width, camera_intr=scaled_camera_intr) # plot vis.subplot(d, d, i + 1) vis.imshow(image) vis.grasp(grasp, scale=1.5) vis.title('K=%d: d=%.3f, q=%.3f' % (i, depth, q_value)) vis.show() # select grasp index = self.select(grasps, q_values) grasp = grasps[index] q_value = q_values[index] image = DepthImage(image_tensor[index, ...]) pose = pose_tensor[index, ...] depth = pose[0] if self.config['vis']['grasp_plan']: scale_factor = float(self.gqcnn.im_width) / float(self._crop_width) scaled_camera_intr = camera_intr.resize(scale_factor) grasp = Grasp2D(Point(image.center), 0.0, pose[0], width=self._gripper_width, camera_intr=scaled_camera_intr) vis.figure() vis.imshow(image) vis.grasp(grasp, scale=1.5, show_center=False, show_axis=True) vis.title('Best Grasp: d=%.3f, q=%.3f' % (depth, q_value)) vis.show() # return action return ParallelJawGrasp(grasp, q_value, image)
def _run_prediction_single_model(self, model_dir, model_output_dir, dataset_config): """ Analyze the performance of a single model. """ # read in model config model_config_filename = os.path.join(model_dir, 'config.json') with open(model_config_filename) as data_file: model_config = json.load(data_file) # load model self.logger.info('Loading model %s' %(model_dir)) log_file = None for handler in self.logger.handlers: if isinstance(handler, logging.FileHandler): log_file = handler.baseFilename gqcnn = get_gqcnn_model(verbose=self.verbose).load(model_dir, verbose=self.verbose, log_file=log_file) gqcnn.open_session() gripper_mode = gqcnn.gripper_mode angular_bins = gqcnn.angular_bins # read params from the config if dataset_config is None: dataset_dir = model_config['dataset_dir'] split_name = model_config['split_name'] image_field_name = model_config['image_field_name'] pose_field_name = model_config['pose_field_name'] metric_name = model_config['target_metric_name'] metric_thresh = model_config['metric_thresh'] else: dataset_dir = dataset_config['dataset_dir'] split_name = dataset_config['split_name'] image_field_name = dataset_config['image_field_name'] pose_field_name = dataset_config['pose_field_name'] metric_name = dataset_config['target_metric_name'] metric_thresh = dataset_config['metric_thresh'] gripper_mode = dataset_config['gripper_mode'] self.logger.info('Loading dataset %s' %(dataset_dir)) dataset = TensorDataset.open(dataset_dir) train_indices, val_indices, _ = dataset.split(split_name) # visualize conv filters conv1_filters = gqcnn.filters num_filt = conv1_filters.shape[3] d = utils.sqrt_ceil(num_filt) vis2d.clf() for k in range(num_filt): filt = conv1_filters[:,:,0,k] vis2d.subplot(d,d,k+1) vis2d.imshow(DepthImage(filt)) figname = os.path.join(model_output_dir, 'conv1_filters.pdf') vis2d.savefig(figname, dpi=self.dpi) # aggregate training and validation true labels and predicted probabilities all_predictions = [] if angular_bins > 0: all_predictions_raw = [] all_labels = [] for i in range(dataset.num_tensors): # log progress if i % self.log_rate == 0: self.logger.info('Predicting tensor %d of %d' %(i+1, dataset.num_tensors)) # read in data image_arr = dataset.tensor(image_field_name, i).arr pose_arr = read_pose_data(dataset.tensor(pose_field_name, i).arr, gripper_mode) metric_arr = dataset.tensor(metric_name, i).arr label_arr = 1 * (metric_arr > metric_thresh) label_arr = label_arr.astype(np.uint8) if angular_bins > 0: # form mask to extract predictions from ground-truth angular bins raw_poses = dataset.tensor(pose_field_name, i).arr angles = raw_poses[:, 3] neg_ind = np.where(angles < 0) angles = np.abs(angles) % GeneralConstants.PI angles[neg_ind] *= -1 g_90 = np.where(angles > (GeneralConstants.PI / 2)) l_neg_90 = np.where(angles < (-1 * (GeneralConstants.PI / 2))) angles[g_90] -= GeneralConstants.PI angles[l_neg_90] += GeneralConstants.PI angles *= -1 # hack to fix reverse angle convention angles += (GeneralConstants.PI / 2) pred_mask = np.zeros((raw_poses.shape[0], angular_bins*2), dtype=bool) bin_width = GeneralConstants.PI / angular_bins for i in range(angles.shape[0]): pred_mask[i, int((angles[i] // bin_width)*2)] = True pred_mask[i, int((angles[i] // bin_width)*2 + 1)] = True # predict with GQ-CNN predictions = gqcnn.predict(image_arr, pose_arr) if angular_bins > 0: raw_predictions = np.array(predictions) predictions = predictions[pred_mask].reshape((-1, 2)) # aggregate all_predictions.extend(predictions[:,1].tolist()) if angular_bins > 0: all_predictions_raw.extend(raw_predictions.tolist()) all_labels.extend(label_arr.tolist()) # close session gqcnn.close_session() # create arrays all_predictions = np.array(all_predictions) all_labels = np.array(all_labels) train_predictions = all_predictions[train_indices] val_predictions = all_predictions[val_indices] train_labels = all_labels[train_indices] val_labels = all_labels[val_indices] if angular_bins > 0: all_predictions_raw = np.array(all_predictions_raw) train_predictions_raw = all_predictions_raw[train_indices] val_predictions_raw = all_predictions_raw[val_indices] # aggregate results train_result = BinaryClassificationResult(train_predictions, train_labels) val_result = BinaryClassificationResult(val_predictions, val_labels) train_result.save(os.path.join(model_output_dir, 'train_result.cres')) val_result.save(os.path.join(model_output_dir, 'val_result.cres')) # get stats, plot curves self.logger.info('Model %s training error rate: %.3f' %(model_dir, train_result.error_rate)) self.logger.info('Model %s validation error rate: %.3f' %(model_dir, val_result.error_rate)) self.logger.info('Model %s training loss: %.3f' %(model_dir, train_result.cross_entropy_loss)) self.logger.info('Model %s validation loss: %.3f' %(model_dir, val_result.cross_entropy_loss)) # save images vis2d.figure() example_dir = os.path.join(model_output_dir, 'examples') if not os.path.exists(example_dir): os.mkdir(example_dir) # train self.logger.info('Saving training examples') train_example_dir = os.path.join(example_dir, 'train') if not os.path.exists(train_example_dir): os.mkdir(train_example_dir) # train TP true_positive_indices = train_result.true_positive_indices np.random.shuffle(true_positive_indices) true_positive_indices = true_positive_indices[:self.num_vis] for i, j in enumerate(true_positive_indices): k = train_indices[j] datapoint = dataset.datapoint(k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=train_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' %(k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig(os.path.join(train_example_dir, 'true_positive_%03d.png' %(i))) # train FP false_positive_indices = train_result.false_positive_indices np.random.shuffle(false_positive_indices) false_positive_indices = false_positive_indices[:self.num_vis] for i, j in enumerate(false_positive_indices): k = train_indices[j] datapoint = dataset.datapoint(k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=train_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' %(k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig(os.path.join(train_example_dir, 'false_positive_%03d.png' %(i))) # train TN true_negative_indices = train_result.true_negative_indices np.random.shuffle(true_negative_indices) true_negative_indices = true_negative_indices[:self.num_vis] for i, j in enumerate(true_negative_indices): k = train_indices[j] datapoint = dataset.datapoint(k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=train_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' %(k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig(os.path.join(train_example_dir, 'true_negative_%03d.png' %(i))) # train TP false_negative_indices = train_result.false_negative_indices np.random.shuffle(false_negative_indices) false_negative_indices = false_negative_indices[:self.num_vis] for i, j in enumerate(false_negative_indices): k = train_indices[j] datapoint = dataset.datapoint(k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=train_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' %(k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig(os.path.join(train_example_dir, 'false_negative_%03d.png' %(i))) # val self.logger.info('Saving validation examples') val_example_dir = os.path.join(example_dir, 'val') if not os.path.exists(val_example_dir): os.mkdir(val_example_dir) # val TP true_positive_indices = val_result.true_positive_indices np.random.shuffle(true_positive_indices) true_positive_indices = true_positive_indices[:self.num_vis] for i, j in enumerate(true_positive_indices): k = val_indices[j] datapoint = dataset.datapoint(k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=val_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' %(k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig(os.path.join(val_example_dir, 'true_positive_%03d.png' %(i))) # val FP false_positive_indices = val_result.false_positive_indices np.random.shuffle(false_positive_indices) false_positive_indices = false_positive_indices[:self.num_vis] for i, j in enumerate(false_positive_indices): k = val_indices[j] datapoint = dataset.datapoint(k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=val_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' %(k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig(os.path.join(val_example_dir, 'false_positive_%03d.png' %(i))) # val TN true_negative_indices = val_result.true_negative_indices np.random.shuffle(true_negative_indices) true_negative_indices = true_negative_indices[:self.num_vis] for i, j in enumerate(true_negative_indices): k = val_indices[j] datapoint = dataset.datapoint(k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=val_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' %(k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig(os.path.join(val_example_dir, 'true_negative_%03d.png' %(i))) # val TP false_negative_indices = val_result.false_negative_indices np.random.shuffle(false_negative_indices) false_negative_indices = false_negative_indices[:self.num_vis] for i, j in enumerate(false_negative_indices): k = val_indices[j] datapoint = dataset.datapoint(k, field_names=[image_field_name, pose_field_name]) vis2d.clf() if angular_bins > 0: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode, angular_preds=val_predictions_raw[j]) else: self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' %(k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig(os.path.join(val_example_dir, 'false_negative_%03d.png' %(i))) # save summary stats train_summary_stats = { 'error_rate': train_result.error_rate, 'ap_score': train_result.ap_score, 'auc_score': train_result.auc_score, 'loss': train_result.cross_entropy_loss } train_stats_filename = os.path.join(model_output_dir, 'train_stats.json') json.dump(train_summary_stats, open(train_stats_filename, 'w'), indent=JSON_INDENT, sort_keys=True) val_summary_stats = { 'error_rate': val_result.error_rate, 'ap_score': val_result.ap_score, 'auc_score': val_result.auc_score, 'loss': val_result.cross_entropy_loss } val_stats_filename = os.path.join(model_output_dir, 'val_stats.json') json.dump(val_summary_stats, open(val_stats_filename, 'w'), indent=JSON_INDENT, sort_keys=True) return train_result, val_result
def _run_prediction_single_model(self, model_dir, model_output_dir, dataset_config): """ Analyze the performance of a single model. """ # read in model config model_config_filename = os.path.join(model_dir, 'config.json') with open(model_config_filename) as data_file: model_config = json.load(data_file) # load model logging.info('Loading model %s' % (model_dir)) gqcnn = GQCNN.load(model_dir) gqcnn.open_session() gripper_mode = gqcnn.gripper_mode # read params from the config if dataset_config is None: dataset_dir = model_config['dataset_dir'] split_name = model_config['split_name'] image_field_name = model_config['image_field_name'] pose_field_name = model_config['pose_field_name'] metric_name = model_config['target_metric_name'] metric_thresh = model_config['metric_thresh'] else: dataset_dir = dataset_config['dataset_dir'] split_name = dataset_config['split_name'] image_field_name = dataset_config['image_field_name'] pose_field_name = dataset_config['pose_field_name'] metric_name = dataset_config['target_metric_name'] metric_thresh = dataset_config['metric_thresh'] gripper_mode = dataset_config['gripper_mode'] logging.info('Loading dataset %s' % (dataset_dir)) dataset = TensorDataset.open(dataset_dir) train_indices, val_indices, _ = dataset.split(split_name) # visualize conv filters conv1_filters = gqcnn.filters num_filt = conv1_filters.shape[3] d = utils.sqrt_ceil(num_filt) vis2d.clf() for k in range(num_filt): filt = conv1_filters[:, :, 0, k] vis2d.subplot(d, d, k + 1) vis2d.imshow(DepthImage(filt)) figname = os.path.join(model_output_dir, 'conv1_filters.pdf') vis2d.savefig(figname, dpi=self.dpi) # aggregate training and validation true labels and predicted probabilities all_predictions = [] all_labels = [] for i in range(dataset.num_tensors): # log progress if i % self.log_rate == 0: logging.info('Predicting tensor %d of %d' % (i + 1, dataset.num_tensors)) # read in data image_arr = dataset.tensor(image_field_name, i).arr pose_arr = read_pose_data( dataset.tensor(pose_field_name, i).arr, gripper_mode) metric_arr = dataset.tensor(metric_name, i).arr label_arr = 1 * (metric_arr > metric_thresh) label_arr = label_arr.astype(np.uint8) # predict with GQ-CNN predictions = gqcnn.predict(image_arr, pose_arr) # aggregate all_predictions.extend(predictions[:, 1].tolist()) all_labels.extend(label_arr.tolist()) # close session gqcnn.close_session() # create arrays all_predictions = np.array(all_predictions) all_labels = np.array(all_labels) train_predictions = all_predictions[train_indices] val_predictions = all_predictions[val_indices] train_labels = all_labels[train_indices] val_labels = all_labels[val_indices] # aggregate results train_result = BinaryClassificationResult(train_predictions, train_labels) val_result = BinaryClassificationResult(val_predictions, val_labels) train_result.save(os.path.join(model_output_dir, 'train_result.cres')) val_result.save(os.path.join(model_output_dir, 'val_result.cres')) # get stats, plot curves logging.info('Model %s training error rate: %.3f' % (model_dir, train_result.error_rate)) logging.info('Model %s validation error rate: %.3f' % (model_dir, val_result.error_rate)) # save images vis2d.figure() example_dir = os.path.join(model_output_dir, 'examples') if not os.path.exists(example_dir): os.mkdir(example_dir) # train logging.info('Saving training examples') train_example_dir = os.path.join(example_dir, 'train') if not os.path.exists(train_example_dir): os.mkdir(train_example_dir) # train TP true_positive_indices = train_result.true_positive_indices np.random.shuffle(true_positive_indices) true_positive_indices = true_positive_indices[:self.num_vis] for i, j in enumerate(true_positive_indices): k = train_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title( 'Datapoint %d: Pred: %.3f Label: %.3f' % (k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(train_example_dir, 'true_positive_%03d.png' % (i))) # train FP false_positive_indices = train_result.false_positive_indices np.random.shuffle(false_positive_indices) false_positive_indices = false_positive_indices[:self.num_vis] for i, j in enumerate(false_positive_indices): k = train_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title( 'Datapoint %d: Pred: %.3f Label: %.3f' % (k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(train_example_dir, 'false_positive_%03d.png' % (i))) # train TN true_negative_indices = train_result.true_negative_indices np.random.shuffle(true_negative_indices) true_negative_indices = true_negative_indices[:self.num_vis] for i, j in enumerate(true_negative_indices): k = train_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title( 'Datapoint %d: Pred: %.3f Label: %.3f' % (k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(train_example_dir, 'true_negative_%03d.png' % (i))) # train TP false_negative_indices = train_result.false_negative_indices np.random.shuffle(false_negative_indices) false_negative_indices = false_negative_indices[:self.num_vis] for i, j in enumerate(false_negative_indices): k = train_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title( 'Datapoint %d: Pred: %.3f Label: %.3f' % (k, train_result.pred_probs[j], train_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(train_example_dir, 'false_negative_%03d.png' % (i))) # val logging.info('Saving validation examples') val_example_dir = os.path.join(example_dir, 'val') if not os.path.exists(val_example_dir): os.mkdir(val_example_dir) # val TP true_positive_indices = val_result.true_positive_indices np.random.shuffle(true_positive_indices) true_positive_indices = true_positive_indices[:self.num_vis] for i, j in enumerate(true_positive_indices): k = val_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' % (k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(val_example_dir, 'true_positive_%03d.png' % (i))) # val FP false_positive_indices = val_result.false_positive_indices np.random.shuffle(false_positive_indices) false_positive_indices = false_positive_indices[:self.num_vis] for i, j in enumerate(false_positive_indices): k = val_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' % (k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(val_example_dir, 'false_positive_%03d.png' % (i))) # val TN true_negative_indices = val_result.true_negative_indices np.random.shuffle(true_negative_indices) true_negative_indices = true_negative_indices[:self.num_vis] for i, j in enumerate(true_negative_indices): k = val_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' % (k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(val_example_dir, 'true_negative_%03d.png' % (i))) # val TP false_negative_indices = val_result.false_negative_indices np.random.shuffle(false_negative_indices) false_negative_indices = false_negative_indices[:self.num_vis] for i, j in enumerate(false_negative_indices): k = val_indices[j] datapoint = dataset.datapoint( k, field_names=[image_field_name, pose_field_name]) vis2d.clf() self._plot_grasp(datapoint, image_field_name, pose_field_name, gripper_mode) vis2d.title('Datapoint %d: Pred: %.3f Label: %.3f' % (k, val_result.pred_probs[j], val_result.labels[j]), fontsize=self.font_size) vis2d.savefig( os.path.join(val_example_dir, 'false_negative_%03d.png' % (i))) # save summary stats train_summary_stats = { 'error_rate': train_result.error_rate, 'ap_score': train_result.ap_score, 'auc_score': train_result.auc_score } train_stats_filename = os.path.join(model_output_dir, 'train_stats.json') json.dump(train_summary_stats, open(train_stats_filename, 'w'), indent=JSON_INDENT, sort_keys=True) val_summary_stats = { 'error_rate': val_result.error_rate, 'ap_score': val_result.ap_score, 'auc_score': val_result.auc_score } val_stats_filename = os.path.join(model_output_dir, 'val_stats.json') json.dump(val_summary_stats, open(val_stats_filename, 'w'), indent=JSON_INDENT, sort_keys=True) return train_result, val_result
rot_map = cv2.getRotationMatrix2D((f_center[1], f_center[0]), -theta, 1) new_convW = np.zeros(convW.shape) for i in range(num_in_channels): for j in range(num_out_channels): orig_filt = convW[:, :, i, j] rot_filt = cv2.warpAffine( orig_filt, rot_map, (width, height), flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REPLICATE) new_convW[:, :, i, j] = rot_filt if plot and layer == 'conv1_1': d = utils.sqrt_ceil(num_out_channels) plt.figure() for i in range(num_out_channels): plt.subplot(d, d, i + 1) plt.imshow(convW[:, :, 0, i], cmap=plt.cm.gray_r) plt.axis('off') plt.figure() for i in range(num_out_channels): plt.subplot(d, d, i + 1) plt.imshow(new_convW[:, :, 0, i], cmap=plt.cm.gray_r) plt.axis('off') plt.show() sess.run(convW_var.assign(new_convW)) #gqcnn._output_tensor = gqcnn._build_network(gqcnn._input_im_node, # gqcnn._input_pose_node)
if mesh_ext != '.obj': raise ValueError('Must provide mesh in Wavefront .OBJ format!') orig_mesh = ObjFile(mesh_filename).read() mesh = orig_mesh.subdivide(min_tri_length=0.01) mesh.compute_vertex_normals() stable_poses = mesh.stable_poses() if vis_normals: vis3d.figure() vis3d.mesh(mesh) vis3d.normals(NormalCloud(mesh.normals.T), PointCloud(mesh.vertices.T), subsample=10) vis3d.show() d = utils.sqrt_ceil(len(stable_poses)) vis.figure(size=(16, 16)) table_mesh = ObjFile('data/meshes/table.obj').read() table_mesh = table_mesh.subdivide() table_mesh.compute_vertex_normals() table_mat_props = MaterialProperties(color=(0, 255, 0), ambient=0.5, diffuse=1.0, specular=1, shininess=0) for k, stable_pose in enumerate(stable_poses): logging.info('Rendering stable pose %d' % (k)) # set resting pose