def run_affordance_net(net, image_name): im_file = img_folder + '/' + im_name im = cv2.imread(im_file) ori_height, ori_width, _ = im.shape # Detect all object classes and regress object bounds timer = Timer() timer.tic() if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2( net, im) else: 1 timer.toc() print('Detection took {:.3f}s for ' '{:d} object proposals').format(timer.total_time, rois_final.shape[0]) # Visualize detections for each class visualize_mask(im, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD)
def run_affordance_net_map(net, image_name, obj_name): im_file = img_folder + '/' + im_name im = cv2.imread(im_file) ori_height, ori_width, _ = im.shape if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2( net, im) else: 1 print "masks0 shape: ", masks.shape print "scores0 shape: ", scores.shape obj_iscontainer, obj_iscontainer_cfd = get_containability( im, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD)
def run_affordance_net(net, image_name): im_file = img_folder + '/' + im_name im = cv2.imread(im_file) ori_height, ori_width, _ = im.shape # Detect all object classes and regress object bounds timer = Timer() timer.tic() if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2( net, im) else: 1 timer.toc() # Visualize detections for each class color_cuur_mask, img_out = visualize_mask(im, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD) return color_cuur_mask, img_out
def run_affordance_net_classification(net, image_name, obj_name): im_file = img_folder + '/' + im_name im = cv2.imread(im_file) if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, im) else: 1 # print "rois_final: ", rois_final # print "rois_class_score: ", rois_class_score, rois_class_score.shape # print "rois_class_ind: ", rois_class_ind, rois_class_ind.shape assert rois_class_score.shape == rois_class_ind.shape largest_cfd_idx = np.argmax(rois_class_score) # print "largest_cfd_idx: ", largest_cfd_idx # obj_cfd = rois_class_score[largest_cfd_idx][0] obj_classification_idx = rois_class_ind[largest_cfd_idx][0] # Open container idx: bowl(1), cup(6), pan(3) if obj_classification_idx == 1 or obj_classification_idx == 3 or obj_classification_idx == 6: # print "Object is classified as an open container" obj_iscontainer = True else: obj_iscontainer = False print "obj_iscontainer: ", obj_iscontainer return obj_iscontainer
def run_affordance_net(net, image_name): im_file = img_folder + '/' + im_name im = cv2.imread(im_file) ori_height, ori_width, _ = im.shape # Detect all object classes and regress object bounds timer = Timer() timer.tic() if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, im) else: 1 timer.toc() # print ('Detection took {:.3f}s for ' # '{:d} object proposals').format(timer.total_time, rois_final.shape[0]) # print 'im_detect2 mask: ', masks.shape # print 'im_detect2 rois_final: ', rois_final # print 'im_detect2 rois_class_score: ', rois_class_score, rois_class_score.shape # print 'im_detect2 rois_class_ind: ', rois_class_ind # if rois_class_score.shape == (1, 1) : # if rois_class_score[0][0] == -1: # print im_name # print 'im_detect2 rois_class_ind: ', rois_class_ind # print 'im_detect2 rois_class_score: ', rois_class_score, rois_class_score.shape # print im_file, " has not detection!!!!!!!" # return None, None # Visualize detections for each class color_cuur_mask, img_out = visualize_mask(im, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD) return color_cuur_mask, img_out
def run_affordance_net_asus(net, im): tmp_g = im[:, :, 1] im[:, :, 1] = im[:, :, 2] im[:, :, 2] = tmp_g img = im.astype('uint8') # Detect all object classes and regress object bounds timer = Timer() timer.tic() if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2( net, im) # for some reason the kl model isnt working on landan's pc; we collect data from fu-jen's pc and transmit to landan if args.sim and os.path.exists( '/home/landan/data_shared/affordance/landan/support/im_detect2.npz' ): container = np.load( '/home/landan/data_shared/affordance/landan/support/im_detect2.npz' ) rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = container[ 'arr_0'], container['arr_1'], container['arr_2'], container[ 'arr_3'], container['arr_4'], container['arr_5'] else: 1 timer.toc() print('Detection took {:.3f}s for ' '{:d} object proposals').format(timer.total_time, rois_final.shape[0]) # Visualize detections for each class return visualize_mask_asus(im, rois_final, rois_class_score, rois_class_ind, masks, thresh=CONF_THRESHOLD)
def run_affordance_net_map_direct_crop(net, crop_img): ori_height, ori_width, _ = crop_img.shape if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, crop_img) else: 1 # print "rois_final: ", rois_final # print "scores shape: ", scores.shape # print "masks shape: ", masks.shape # print "boxes shape: ", boxes.shape # print ("rois_class_score: ", rois_class_score, rois_class_score.shape) # print ("rois_class_ind: ", rois_class_ind, rois_class_ind.shape) assert rois_class_score.shape == rois_class_ind.shape # Use the bounding box with the largest score as the classification result. largest_cfd_idx = np.argmax(rois_class_score) # print "largest_cfd_idx: ", largest_cfd_idx obj_cfd = rois_class_score[largest_cfd_idx][0] obj_classification_idx = rois_class_ind[largest_cfd_idx][0] # If there is no detection then the container cfd equals 0 if rois_class_score[0][0] == -1: obj_iscontainer = False obj_container_cfd = 0.0 # Nothing is detected bbox_score_list = [] else: # Find the bounding box in scores largest_cfd_idx_in_score = np.where(scores==obj_cfd) print ("largest_cfd_idx_in_score: ", largest_cfd_idx_in_score) # Make sure there is only one such box assert largest_cfd_idx_in_score[0].shape[0] == 1 assert largest_cfd_idx_in_score[1].shape[0] == 1 # Open container idx: bowl(1), cup(6), pan(3) if obj_classification_idx == 1 or obj_classification_idx == 3 or obj_classification_idx == 6: # print "Object is classified as an open container" obj_iscontainer = True else: obj_iscontainer = False bbox_score_list = scores[largest_cfd_idx_in_score[0][0],:] # print "bbox_score_list sum: ", np.sum(bbox_score_list) bbox_container_score_list = np.array([bbox_score_list[1], bbox_score_list[3], bbox_score_list[6]]) # print "bbox_container_score_list: ", bbox_container_score_list obj_container_cfd = np.max(bbox_container_score_list) color_curr_mask, img_out, list_bboxes = visualize_mask(crop_img, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD) # print "obj_iscontainer: ", obj_iscontainer # print "obj_container_cfd: ", obj_container_cfd return obj_iscontainer, obj_container_cfd, color_curr_mask, img_out, list_bboxes, bbox_score_list, rois_class_score, rois_class_ind
def demo(net, image_name): """Detect object classes in an image using pre-computed object proposals.""" #if cfg.TEST.CONTEXT: # txt_file = open('test/context/leannet/'+im_name.split('.')[0]+'.txt','w') #else: txt_file = open('test/context/faster/' + im_name.split('.')[0] + '.txt', 'w') # Load the demo image im_file = os.path.join('/content/py-faster-rcnn/data/demo_coco/', image_name) im = cv2.imread(im_file) height, width, channels = im.shape # Detect all object classes and regress object bounds timer = Timer() timer.tic() if cfg.TEST.PLACES: scores, boxes, top5 = im_detect2(net, im) else: scores, boxes = im_detect(net, im) top5 = [] timer.toc() print('Detection took {:.3f}s for ' '{:d} object proposals').format(timer.total_time, boxes.shape[0]) # Visualize detections for each class CONF_THRESH = 0.55 NMS_THRESH = 0.3 fig, ax = plt.subplots(figsize=(12, 12)) for cls_ind, cls in enumerate(CLASSES[1:]): cls_ind += 1 # because we skipped background cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)] cls_scores = scores[:, cls_ind] dets = np.hstack( (cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32) keep = nms(dets, NMS_THRESH) dets = dets[keep, :] vis_detections(cls_ind, im, ax, cls, dets, txt_file, top5, thresh=CONF_THRESH) ax.imshow(im[:, :, (2, 1, 0)], aspect='equal') #ax.set_title(('Probability of detection >= {:.1f}').format(CONF_THRESH),fontsize=14) plt.axis('off') plt.tight_layout() plt.draw() txt_file.close()
def run_affordance_net_map(net, image_name): im_file = img_folder + '/' + im_name im = cv2.imread(im_file) ori_height, ori_width, _ = im.shape if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, im) else: 1 # print "rois_final: ", rois_final # print "scores shape: ", scores.shape # print "masks shape: ", masks.shape # print "boxes shape: ", boxes.shape # print "rois_class_score: ", rois_class_score, rois_class_score.shape # print "rois_class_ind: ", rois_class_ind, rois_class_ind.shape assert rois_class_score.shape == rois_class_ind.shape # Use the bounding box with the largest score as the classification result. largest_cfd_idx = np.argmax(rois_class_score) # print "largest_cfd_idx: ", largest_cfd_idx obj_cfd = rois_class_score[largest_cfd_idx][0] obj_classification_idx = rois_class_ind[largest_cfd_idx][0] # Find the bounding box in scores largest_cfd_idx_in_score = np.where(scores==obj_cfd) # If there is no detection then the container cfd equals 0 if rois_class_score[0][0] == -1: obj_iscontainer = False obj_container_cfd = 0.0 else: # Open container idx: bowl(1), cup(6), pan(3) if obj_classification_idx == 1 or obj_classification_idx == 3 or obj_classification_idx == 6: # print "Object is classified as an open container" obj_iscontainer = True else: obj_iscontainer = False bbox_score_list = scores[largest_cfd_idx_in_score[0][0],:] # print "bbox_score_list sum: ", np.sum(bbox_score_list) bbox_container_score_list = np.array([bbox_score_list[1], bbox_score_list[3], bbox_score_list[6]]) # print "bbox_container_score_list: ", bbox_container_score_list obj_container_cfd = np.max(bbox_container_score_list) # print "obj_iscontainer: ", obj_iscontainer # print "obj_container_cfd: ", obj_container_cfd color_curr_mask, img_out = visualize_mask(im, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD) return obj_iscontainer, obj_container_cfd, color_curr_mask, img_out
def run_affordance_net_asus(net, im): # Detect all object classes and regress object bounds timer = Timer() timer.tic() if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, im) else: 1 timer.toc() print ('Detection took {:.3f}s for ' '{:d} object proposals').format(timer.total_time, rois_final.shape[0]) # Visualize detections for each class list_boxes, list_masks = visualize_mask_asus(im, rois_final, rois_class_score, rois_class_ind, masks, thresh=CONF_THRESHOLD) return list_boxes, list_masks
def detect(self, req): ######################################################################## # IMPORTANT: the pycaffe network needs to be loaded inside of the service handler. Otherwise, pycaffe calls will # be extremely slow. One possible reason is that threads that handle an incoming service request # cannot access the state instance of the main thread, so they have to (re)create / copy one of # their own. cfg.TEST.HAS_RPN = True # Use RPN for proposals caffe.set_mode_gpu() caffe.set_device(0) cfg.GPU_ID = 0 if not os.path.isfile(self.model_filepath): raise IOError("{:s} not found.".format(self.model_filepath)) net = caffe.Net(self.prototxt_filepath, self.model_filepath, caffe.TEST) ######################################################################## if self.bgr_img is not None and self.dep_img is not None: h, w, c = self.bgr_img.shape if h > 100 and w > 100: # Detect all object classes and regress object bounds timer = Timer() timer.tic() if cfg.TEST.MASK_REG: rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2( net, self.bgr_img) timer.toc() rospy.loginfo( 'Detection took {:.3f}s for {:d} object proposals'. format(timer.total_time, rois_final.shape[0])) list_boxes, list_masks = self.visualize_mask( self.bgr_img, rois_final, rois_class_score, rois_class_ind, masks) if list_boxes and list_masks: object_part_affordance_list = [] for i in range(len(list_boxes)): box = list_boxes[i] rospy.loginfo(box) mask = list_masks[i] object_part_affordance = ObjectPartAffordance() object_part_affordance.header.stamp = rospy.Time.now( ) object_part_affordance.header.frame_id = self.header.frame_id object_part_affordance.object_class = box[0] object_part_affordance.detection_score = box[1] object_part_affordance.bounding_box = box[2:] object_part_affordance.height = self.height object_part_affordance.width = self.width object_part_affordance.affordance_mask = mask.flatten( ).tolist() object_part_affordance_list.append( object_part_affordance) return DetectAffordancesResponse( success=True, object_part_affordance_list= object_part_affordance_list) return DetectAffordancesResponse(success=False)