def run_affordance_net(net, image_name):
im_file = img_folder + '/' + im_name
im = cv2.imread(im_file)
ori_height, ori_width, _ = im.shape
# Detect all object classes and regress object bounds
timer = Timer()
timer.tic()
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(
net, im)
else:
1
timer.toc()
print('Detection took {:.3f}s for '
'{:d} object proposals').format(timer.total_time,
rois_final.shape[0])
# Visualize detections for each class
visualize_mask(im,
rois_final,
rois_class_score,
rois_class_ind,
masks,
ori_height,
ori_width,
im_name,
thresh=CONF_THRESHOLD)
def run_affordance_net_map(net, image_name, obj_name):
im_file = img_folder + '/' + im_name
im = cv2.imread(im_file)
ori_height, ori_width, _ = im.shape
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(
net, im)
else:
1
print "masks0 shape: ", masks.shape
print "scores0 shape: ", scores.shape
obj_iscontainer, obj_iscontainer_cfd = get_containability(
im,
rois_final,
rois_class_score,
rois_class_ind,
masks,
ori_height,
ori_width,
im_name,
thresh=CONF_THRESHOLD)
def run_affordance_net(net, image_name):
im_file = img_folder + '/' + im_name
im = cv2.imread(im_file)
ori_height, ori_width, _ = im.shape
# Detect all object classes and regress object bounds
timer = Timer()
timer.tic()
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(
net, im)
else:
1
timer.toc()
# Visualize detections for each class
color_cuur_mask, img_out = visualize_mask(im,
rois_final,
rois_class_score,
rois_class_ind,
masks,
ori_height,
ori_width,
im_name,
thresh=CONF_THRESHOLD)
return color_cuur_mask, img_out
def run_affordance_net_classification(net, image_name, obj_name):
im_file = img_folder + '/' + im_name
im = cv2.imread(im_file)
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, im)
else:
1
# print "rois_final: ", rois_final
# print "rois_class_score: ", rois_class_score, rois_class_score.shape
# print "rois_class_ind: ", rois_class_ind, rois_class_ind.shape
assert rois_class_score.shape == rois_class_ind.shape
largest_cfd_idx = np.argmax(rois_class_score)
# print "largest_cfd_idx: ", largest_cfd_idx
# obj_cfd = rois_class_score[largest_cfd_idx][0]
obj_classification_idx = rois_class_ind[largest_cfd_idx][0]
# Open container idx: bowl(1), cup(6), pan(3)
if obj_classification_idx == 1 or obj_classification_idx == 3 or obj_classification_idx == 6:
# print "Object is classified as an open container"
obj_iscontainer = True
else:
obj_iscontainer = False
print "obj_iscontainer: ", obj_iscontainer
return obj_iscontainer
def run_affordance_net(net, image_name):
im_file = img_folder + '/' + im_name
im = cv2.imread(im_file)
ori_height, ori_width, _ = im.shape
# Detect all object classes and regress object bounds
timer = Timer()
timer.tic()
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, im)
else:
1
timer.toc()
# print ('Detection took {:.3f}s for '
# '{:d} object proposals').format(timer.total_time, rois_final.shape[0])
# print 'im_detect2 mask: ', masks.shape
# print 'im_detect2 rois_final: ', rois_final
# print 'im_detect2 rois_class_score: ', rois_class_score, rois_class_score.shape
# print 'im_detect2 rois_class_ind: ', rois_class_ind
# if rois_class_score.shape == (1, 1) :
# if rois_class_score[0][0] == -1:
# print im_name
# print 'im_detect2 rois_class_ind: ', rois_class_ind
# print 'im_detect2 rois_class_score: ', rois_class_score, rois_class_score.shape
# print im_file, " has not detection!!!!!!!"
# return None, None
# Visualize detections for each class
color_cuur_mask, img_out = visualize_mask(im, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD)
return color_cuur_mask, img_out
def run_affordance_net_asus(net, im):
tmp_g = im[:, :, 1]
im[:, :, 1] = im[:, :, 2]
im[:, :, 2] = tmp_g
img = im.astype('uint8')
# Detect all object classes and regress object bounds
timer = Timer()
timer.tic()
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(
net, im)
# for some reason the kl model isnt working on landan's pc; we collect data from fu-jen's pc and transmit to landan
if args.sim and os.path.exists(
'/home/landan/data_shared/affordance/landan/support/im_detect2.npz'
):
container = np.load(
'/home/landan/data_shared/affordance/landan/support/im_detect2.npz'
)
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = container[
'arr_0'], container['arr_1'], container['arr_2'], container[
'arr_3'], container['arr_4'], container['arr_5']
else:
1
timer.toc()
print('Detection took {:.3f}s for '
'{:d} object proposals').format(timer.total_time,
rois_final.shape[0])
# Visualize detections for each class
return visualize_mask_asus(im,
rois_final,
rois_class_score,
rois_class_ind,
masks,
thresh=CONF_THRESHOLD)
def run_affordance_net_map_direct_crop(net, crop_img):
ori_height, ori_width, _ = crop_img.shape
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, crop_img)
else:
1
# print "rois_final: ", rois_final
# print "scores shape: ", scores.shape
# print "masks shape: ", masks.shape
# print "boxes shape: ", boxes.shape
# print ("rois_class_score: ", rois_class_score, rois_class_score.shape)
# print ("rois_class_ind: ", rois_class_ind, rois_class_ind.shape)
assert rois_class_score.shape == rois_class_ind.shape
# Use the bounding box with the largest score as the classification result.
largest_cfd_idx = np.argmax(rois_class_score)
# print "largest_cfd_idx: ", largest_cfd_idx
obj_cfd = rois_class_score[largest_cfd_idx][0]
obj_classification_idx = rois_class_ind[largest_cfd_idx][0]
# If there is no detection then the container cfd equals 0
if rois_class_score[0][0] == -1:
obj_iscontainer = False
obj_container_cfd = 0.0
# Nothing is detected
bbox_score_list = []
else:
# Find the bounding box in scores
largest_cfd_idx_in_score = np.where(scores==obj_cfd)
print ("largest_cfd_idx_in_score: ", largest_cfd_idx_in_score)
# Make sure there is only one such box
assert largest_cfd_idx_in_score[0].shape[0] == 1
assert largest_cfd_idx_in_score[1].shape[0] == 1
# Open container idx: bowl(1), cup(6), pan(3)
if obj_classification_idx == 1 or obj_classification_idx == 3 or obj_classification_idx == 6:
# print "Object is classified as an open container"
obj_iscontainer = True
else:
obj_iscontainer = False
bbox_score_list = scores[largest_cfd_idx_in_score[0][0],:]
# print "bbox_score_list sum: ", np.sum(bbox_score_list)
bbox_container_score_list = np.array([bbox_score_list[1], bbox_score_list[3], bbox_score_list[6]])
# print "bbox_container_score_list: ", bbox_container_score_list
obj_container_cfd = np.max(bbox_container_score_list)
color_curr_mask, img_out, list_bboxes = visualize_mask(crop_img, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD)
# print "obj_iscontainer: ", obj_iscontainer
# print "obj_container_cfd: ", obj_container_cfd
return obj_iscontainer, obj_container_cfd, color_curr_mask, img_out, list_bboxes, bbox_score_list, rois_class_score, rois_class_ind
def demo(net, image_name):
"""Detect object classes in an image using pre-computed object proposals."""
#if cfg.TEST.CONTEXT:
# txt_file = open('test/context/leannet/'+im_name.split('.')[0]+'.txt','w')
#else:
txt_file = open('test/context/faster/' + im_name.split('.')[0] + '.txt',
'w')
# Load the demo image
im_file = os.path.join('/content/py-faster-rcnn/data/demo_coco/',
image_name)
im = cv2.imread(im_file)
height, width, channels = im.shape
# Detect all object classes and regress object bounds
timer = Timer()
timer.tic()
if cfg.TEST.PLACES:
scores, boxes, top5 = im_detect2(net, im)
else:
scores, boxes = im_detect(net, im)
top5 = []
timer.toc()
print('Detection took {:.3f}s for '
'{:d} object proposals').format(timer.total_time, boxes.shape[0])
# Visualize detections for each class
CONF_THRESH = 0.55
NMS_THRESH = 0.3
fig, ax = plt.subplots(figsize=(12, 12))
for cls_ind, cls in enumerate(CLASSES[1:]):
cls_ind += 1 # because we skipped background
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets, NMS_THRESH)
dets = dets[keep, :]
vis_detections(cls_ind,
im,
ax,
cls,
dets,
txt_file,
top5,
thresh=CONF_THRESH)
ax.imshow(im[:, :, (2, 1, 0)], aspect='equal')
#ax.set_title(('Probability of detection >= {:.1f}').format(CONF_THRESH),fontsize=14)
plt.axis('off')
plt.tight_layout()
plt.draw()
txt_file.close()
def run_affordance_net_map(net, image_name):
im_file = img_folder + '/' + im_name
im = cv2.imread(im_file)
ori_height, ori_width, _ = im.shape
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, im)
else:
1
# print "rois_final: ", rois_final
# print "scores shape: ", scores.shape
# print "masks shape: ", masks.shape
# print "boxes shape: ", boxes.shape
# print "rois_class_score: ", rois_class_score, rois_class_score.shape
# print "rois_class_ind: ", rois_class_ind, rois_class_ind.shape
assert rois_class_score.shape == rois_class_ind.shape
# Use the bounding box with the largest score as the classification result.
largest_cfd_idx = np.argmax(rois_class_score)
# print "largest_cfd_idx: ", largest_cfd_idx
obj_cfd = rois_class_score[largest_cfd_idx][0]
obj_classification_idx = rois_class_ind[largest_cfd_idx][0]
# Find the bounding box in scores
largest_cfd_idx_in_score = np.where(scores==obj_cfd)
# If there is no detection then the container cfd equals 0
if rois_class_score[0][0] == -1:
obj_iscontainer = False
obj_container_cfd = 0.0
else:
# Open container idx: bowl(1), cup(6), pan(3)
if obj_classification_idx == 1 or obj_classification_idx == 3 or obj_classification_idx == 6:
# print "Object is classified as an open container"
obj_iscontainer = True
else:
obj_iscontainer = False
bbox_score_list = scores[largest_cfd_idx_in_score[0][0],:]
# print "bbox_score_list sum: ", np.sum(bbox_score_list)
bbox_container_score_list = np.array([bbox_score_list[1], bbox_score_list[3], bbox_score_list[6]])
# print "bbox_container_score_list: ", bbox_container_score_list
obj_container_cfd = np.max(bbox_container_score_list)
# print "obj_iscontainer: ", obj_iscontainer
# print "obj_container_cfd: ", obj_container_cfd
color_curr_mask, img_out = visualize_mask(im, rois_final, rois_class_score, rois_class_ind, masks, ori_height, ori_width, im_name, thresh=CONF_THRESHOLD)
return obj_iscontainer, obj_container_cfd, color_curr_mask, img_out
def run_affordance_net_asus(net, im):
# Detect all object classes and regress object bounds
timer = Timer()
timer.tic()
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(net, im)
else:
1
timer.toc()
print ('Detection took {:.3f}s for '
'{:d} object proposals').format(timer.total_time, rois_final.shape[0])
# Visualize detections for each class
list_boxes, list_masks = visualize_mask_asus(im, rois_final, rois_class_score, rois_class_ind, masks,
thresh=CONF_THRESHOLD)
return list_boxes, list_masks
def detect(self, req):
########################################################################
# IMPORTANT: the pycaffe network needs to be loaded inside of the service handler. Otherwise, pycaffe calls will
# be extremely slow. One possible reason is that threads that handle an incoming service request
# cannot access the state instance of the main thread, so they have to (re)create / copy one of
# their own.
cfg.TEST.HAS_RPN = True # Use RPN for proposals
caffe.set_mode_gpu()
caffe.set_device(0)
cfg.GPU_ID = 0
if not os.path.isfile(self.model_filepath):
raise IOError("{:s} not found.".format(self.model_filepath))
net = caffe.Net(self.prototxt_filepath, self.model_filepath,
caffe.TEST)
########################################################################
if self.bgr_img is not None and self.dep_img is not None:
h, w, c = self.bgr_img.shape
if h > 100 and w > 100:
# Detect all object classes and regress object bounds
timer = Timer()
timer.tic()
if cfg.TEST.MASK_REG:
rois_final, rois_class_score, rois_class_ind, masks, scores, boxes = im_detect2(
net, self.bgr_img)
timer.toc()
rospy.loginfo(
'Detection took {:.3f}s for {:d} object proposals'.
format(timer.total_time, rois_final.shape[0]))
list_boxes, list_masks = self.visualize_mask(
self.bgr_img, rois_final, rois_class_score,
rois_class_ind, masks)
if list_boxes and list_masks:
object_part_affordance_list = []
for i in range(len(list_boxes)):
box = list_boxes[i]
rospy.loginfo(box)
mask = list_masks[i]
object_part_affordance = ObjectPartAffordance()
object_part_affordance.header.stamp = rospy.Time.now(
)
object_part_affordance.header.frame_id = self.header.frame_id
object_part_affordance.object_class = box[0]
object_part_affordance.detection_score = box[1]
object_part_affordance.bounding_box = box[2:]
object_part_affordance.height = self.height
object_part_affordance.width = self.width
object_part_affordance.affordance_mask = mask.flatten(
).tolist()
object_part_affordance_list.append(
object_part_affordance)
return DetectAffordancesResponse(
success=True,
object_part_affordance_list=
object_part_affordance_list)
return DetectAffordancesResponse(success=False)
