def get_mask(rgb, depth, T_w_k):
hsv = cv2.cvtColor(rgb, cv2.COLOR_BGR2HSV)
h = hsv[:,:,0]
s = hsv[:,:,1]
v = hsv[:,:,2]
red_mask = ((h<10) | (h>150)) & (s > 100) & (v > 100)
valid_mask = depth > 0
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3,:3].T) + T_w_k[:3,3][None,None,:]
z = xyz_w[:,:,2]
z0 = xyz_k[:,:,2]
# if DEBUG_PLOTS:
# cv2.imshow("z0",z0/z0.max())
# cv2.imshow("z",z/z.max())
# cv2.imshow("rgb", rgb)
# cv2.waitKey()
height_mask = xyz_w[:,:,2] > .7 # TODO pass in parameter
good_mask = red_mask & height_mask & valid_mask
return good_mask
def get_PC (self):
r, d = self.grabber.getRGBD()
x = clouds.depth_to_xyz(d, berkeley_pr2.f)
if Globals.pr2 is not None:
Globals.pr2.update_rave()
tfm = berkeley_pr2.get_kinect_transform(Globals.robot)
x = x.dot(tfm[:3,:3].T) + tfm[:3,3][None,None,:]
return ru.xyzrgb2pc(x, r, 'base_footprint')
def label_single_demo_points(seg_group, name):
if 'old_cloud_xyz' not in seg_group:
seg_group['old_cloud_xyz'] = seg_group['cloud_xyz'][:]
print name
rgb = seg_group['rgb'][:]
old_cloud_xyz = seg_group['old_cloud_xyz'][:]
depth = seg_group['depth'][:]
T_w_k = seg_group['T_w_k'][:]
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3, :3].T) + T_w_k[:3, 3][None, None, :]
old_cloud_kd = KDTree(old_cloud_xyz)
image = np.asarray(rgb)
windowName = name
ctr = 0
finished = False
labeled_points = []
cv2.namedWindow(windowName)
cv2.setMouseCallback(windowName, mark2, (image, labeled_points))
while(1):
cv2.imshow(windowName,image)
k = cv2.waitKey(20) & 0xFF
if k == ord('r'):
return "retry"
elif k == ord(' '):
break
elif k == ord('p'):
return "restart"
elif k == 27:
return "quit"
labeled_points = np.asarray(labeled_points)
if 'labeled_points' in seg_group:
del seg_group['labeled_points']
seg_group.create_dataset("labeled_points",shape=(len(labeled_points),3),data=labeled_points)
# resample the line
labeled_xyz = []
for i in range(labeled_points.shape[0]):
pt_i = labeled_points[i, :2]
x, y, z = xyz_k[pt_i[0], pt_i[1], :].dot(T_w_k[:3, :3].T) + T_w_k[:3, 3]
if depth[pt_i[0], pt_i[1]] > 0 and z > .7:
labeled_xyz.append([x, y, z])
labeled_xyz = np.asarray(labeled_xyz)
cloud_xyz = np.zeros((resample_size, 3))
rope_len = line_length(labeled_xyz)
for i, t in enumerate(np.linspace(0, rope_len, resample_size)):
cloud_xyz[i, :] = point_at(labeled_xyz, t)
plt.scatter(old_cloud_xyz[:, 0], old_cloud_xyz[:, 1], color='r')
plt.scatter(cloud_xyz[:, 0], cloud_xyz[:, 1], color='b')
plt.show(block=False)
k = cv2.waitKey(0) & 0xFF
plt.close()
if k == ord('r'):
return "retry"
if 'cloud_xyz' in seg_group:
del seg_group['cloud_xyz']
seg_group['cloud_xyz'] = cloud_xyz
def find_keypoint_transform_execution (kp, grabber, tfm, normal=False):
global server, app, found, done
found = False
done = False
app = None
if rospy.get_name() == '/unnamed':
rospy.init_node("find_keypoint_%s"%kp, anonymous=True, disable_signals=True)
pcpub = rospy.Publisher('keypoint_pcd', PointCloud2)
server = InteractiveMarkerServer("find_keypoint_%s"%kp)
menu_handler.insert("Done?", callback=doneCallback)
menu_handler.insert("Re-try?", callback=redoCallback)
make6DofMarker( False, normal )
#make6DofMarker( True )
server.applyChanges()
print "Open an RViz Window!"
#if yes_or_no("Do you want to open your own rviz window? Might be a better idea."):
outer_RVIZ = True
# else:
# outer_RVIZ = False
# app = myviz.QApplication( [""] )
# viz = myviz.MyViz()
# viz.resize( 1000, 1000 )
# viz.show()
p = rospy.Rate(10)
while not rospy.is_shutdown() and not done:
rgb, dep = grabber.getRGBD()
xyz = clouds.depth_to_xyz(dep, berkeley_pr2.f)
xyz = xyz.dot(tfm[:3,:3].T) + tfm[:3,3][None,None,:]
pcmsg = ru.xyzrgb2pc(xyz, rgb, 'base_footprint')
pcpub.publish(pcmsg)
if not outer_RVIZ:
app.processEvents()
p.sleep()
# Unhappy with data
if not found:
return None
marker = server.get("pose_marker")
pose = marker.pose
print "Pose of the marker for %s"%kp
print pose
server.erase("pose_marker")
#rospy.signal_shutdown('Finished finding transform')
return conversions.pose_to_hmat(pose)
def grabcut(rgb, depth, T_w_k):
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3, :3].T) + T_w_k[:3, 3][None, None, :]
valid_mask = depth > 0
import interactive_roi as ir
xys = ir.get_polyline(rgb, "rgb")
xy_corner1 = np.clip(np.array(xys).min(axis=0), [0, 0], [639, 479])
xy_corner2 = np.clip(np.array(xys).max(axis=0), [0, 0], [639, 479])
polymask = ir.mask_from_poly(xys)
#cv2.imshow("mask",mask)
xy_tl = np.array([xy_corner1, xy_corner2]).min(axis=0)
xy_br = np.array([xy_corner1, xy_corner2]).max(axis=0)
xl, yl = xy_tl
w, h = xy_br - xy_tl
mask = np.zeros((h, w), dtype='uint8')
mask[polymask[yl:yl + h, xl:xl + w] > 0] = cv2.GC_PR_FGD
print mask.shape
#mask[h//4:3*h//4, w//4:3*w//4] = cv2.GC_PR_FGD
tmp1 = np.zeros((1, 13 * 5))
tmp2 = np.zeros((1, 13 * 5))
cv2.grabCut(rgb[yl:yl + h, xl:xl + w, :],
mask, (0, 0, 0, 0),
tmp1,
tmp2,
10,
mode=cv2.GC_INIT_WITH_MASK)
mask = mask % 2
#mask = ndi.binary_erosion(mask, utils_images.disk(args.erode)).astype('uint8')
contours = cv2.findContours(mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)[0]
cv2.drawContours(rgb[yl:yl + h, xl:xl + w, :],
contours,
-1, (0, 255, 0),
thickness=2)
cv2.imshow('rgb', rgb)
print "press enter to continue"
cv2.waitKey()
zsel = xyz_w[yl:yl + h, xl:xl + w, 2]
mask = (mask % 2 == 1) & np.isfinite(zsel) # & (zsel - table_height > -1)
mask &= valid_mask[yl:yl + h, xl:xl + w]
xyz_sel = xyz_w[yl:yl + h, xl:xl + w, :][mask.astype('bool')]
return clouds.downsample(xyz_sel, .01)
def extract_red(rgb, depth, T_w_k):
"""
extract red points and downsample
"""
hsv = cv2.cvtColor(rgb, cv2.COLOR_BGR2HSV)
h = hsv[:,:,0]
s = hsv[:,:,1]
v = hsv[:,:,2]
red_mask = ((h<10) | (h>150)) & (s > 100) & (v > 100)
valid_mask = depth > 0
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3,:3].T) + T_w_k[:3,3][None,None,:]
z = xyz_w[:,:,2]
z0 = xyz_k[:,:,2]
if DEBUG_PLOTS:
cv2.imshow("z0",z0/z0.max())
cv2.imshow("z",z/z.max())
cv2.imshow("rgb", rgb)
cv2.waitKey()
height_mask = xyz_w[:,:,2] > .7 # XXXX pass in parameter
good_mask = red_mask & height_mask & valid_mask
if DEBUG_PLOTS:
cv2.imshow("red",red_mask.astype('uint8')*255)
cv2.imshow("above_table", height_mask.astype('uint8')*255)
cv2.imshow("red and above table", good_mask.astype('uint8')*255)
print "press enter to continue"
cv2.waitKey()
good_xyz = xyz_w[good_mask]
cloud = cloudprocpy.CloudXYZ()
good_xyz1 = np.zeros((len(good_xyz), 4), 'float32')
good_xyz1[:,:3] = good_xyz
cloud.from2dArray(good_xyz1)
cloud_ds = cloudprocpy.downsampleCloud(cloud, .01)
return cloud_ds.to2dArray()[:,:3]
def extract_red(rgb, depth, T_w_k):
"""
extract red points and downsample
"""
hsv = cv2.cvtColor(rgb, cv2.COLOR_BGR2HSV)
h = hsv[:, :, 0]
s = hsv[:, :, 1]
v = hsv[:, :, 2]
red_mask = ((h < 10) | (h > 150)) & (s > 100) & (v > 100)
valid_mask = depth > 0
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3, :3].T) + T_w_k[:3, 3][None, None, :]
z = xyz_w[:, :, 2]
z0 = xyz_k[:, :, 2]
if DEBUG_PLOTS:
cv2.imshow("z0", z0 / z0.max())
cv2.imshow("z", z / z.max())
cv2.imshow("rgb", rgb)
cv2.waitKey()
height_mask = xyz_w[:, :, 2] > .7 # XXXX pass in parameter
good_mask = red_mask & height_mask & valid_mask
if DEBUG_PLOTS:
cv2.imshow("red", red_mask.astype('uint8') * 255)
cv2.imshow("above_table", height_mask.astype('uint8') * 255)
cv2.imshow("red and above table", good_mask.astype('uint8') * 255)
print "press enter to continue"
cv2.waitKey()
good_xyz = xyz_w[good_mask]
cloud = cloudprocpy.CloudXYZ()
good_xyz1 = np.zeros((len(good_xyz), 4), 'float32')
good_xyz1[:, :3] = good_xyz
cloud.from2dArray(good_xyz1)
cloud_ds = cloudprocpy.downsampleCloud(cloud, .01)
return cloud_ds.to2dArray()[:, :3]
def extract_red(rgb, depth, T_w_k):
"""
extract red points and downsample
"""
hsv = cv2.cvtColor(rgb, cv2.COLOR_BGR2HSV)
h = hsv[:,:,0]
s = hsv[:,:,1]
v = hsv[:,:,2]
h_mask = (h<15) | (h>145)
s_mask = (s > 30 )
v_mask = (v > 100)
red_mask = h_mask & s_mask & v_mask
valid_mask = depth > 0
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3,:3].T) + T_w_k[:3,3][None,None,:]
z = xyz_w[:,:,2]
z0 = xyz_k[:,:,2]
height_mask = xyz_w[:,:,2] > .7 # TODO pass in parameter
good_mask = red_mask & height_mask & valid_mask
good_mask = skim.remove_small_objects(good_mask,min_size=64)
if DEBUG_PLOTS:
cv2.imshow("z0",z0/z0.max())
cv2.imshow("z",z/z.max())
cv2.imshow("hue", h_mask.astype('uint8')*255)
cv2.imshow("sat", s_mask.astype('uint8')*255)
cv2.imshow("val", v_mask.astype('uint8')*255)
cv2.imshow("final",good_mask.astype('uint8')*255)
cv2.imshow("rgb", rgb)
cv2.waitKey()
good_xyz = xyz_w[good_mask]
return clouds.downsample(good_xyz, .025)
def extract_red(rgb, depth, T_w_k):
"""
extract red points and downsample
"""
hsv = cv2.cvtColor(rgb, cv2.COLOR_BGR2HSV)
h = hsv[:,:,0]
s = hsv[:,:,1]
v = hsv[:,:,2]
red_mask = ((h<10) | (h>150)) & (s > 100) & (v > 100)
valid_mask = depth > 0
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3,:3].T) + T_w_k[:3,3][None,None,:]
z = xyz_w[:,:,2]
z0 = xyz_k[:,:,2]
if DEBUG_PLOTS:
cv2.imshow("z0",z0/z0.max())
cv2.imshow("z",z/z.max())
cv2.imshow("rgb", rgb)
cv2.waitKey()
height_mask = xyz_w[:,:,2] > .7 # TODO pass in parameter
good_mask = red_mask & height_mask & valid_mask
if DEBUG_PLOTS:
cv2.imshow("red",red_mask.astype('uint8')*255)
cv2.imshow("above_table", height_mask.astype('uint8')*255)
cv2.imshow("red and above table", good_mask.astype('uint8')*255)
print "press enter to continue"
cv2.waitKey()
good_xyz = xyz_w[good_mask]
return clouds.downsample(good_xyz, .025)
def grabcut(rgb, depth, T_w_k):
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3,:3].T) + T_w_k[:3,3][None,None,:]
valid_mask = depth > 0
import interactive_roi as ir
xys = ir.get_polyline(rgb, "rgb")
xy_corner1 = np.clip(np.array(xys).min(axis=0), [0,0], [639,479])
xy_corner2 = np.clip(np.array(xys).max(axis=0), [0,0], [639,479])
polymask = ir.mask_from_poly(xys)
#cv2.imshow("mask",mask)
xy_tl = np.array([xy_corner1, xy_corner2]).min(axis=0)
xy_br = np.array([xy_corner1, xy_corner2]).max(axis=0)
xl, yl = xy_tl
w, h = xy_br - xy_tl
mask = np.zeros((h,w),dtype='uint8')
mask[polymask[yl:yl+h, xl:xl+w] > 0] = cv2.GC_PR_FGD
print mask.shape
#mask[h//4:3*h//4, w//4:3*w//4] = cv2.GC_PR_FGD
tmp1 = np.zeros((1, 13 * 5))
tmp2 = np.zeros((1, 13 * 5))
cv2.grabCut(rgb[yl:yl+h, xl:xl+w, :],mask,(0,0,0,0),tmp1, tmp2,10,mode=cv2.GC_INIT_WITH_MASK)
mask = mask % 2
#mask = ndi.binary_erosion(mask, utils_images.disk(args.erode)).astype('uint8')
contours = cv2.findContours(mask,cv2.RETR_LIST,cv2.CHAIN_APPROX_NONE)[0]
cv2.drawContours(rgb[yl:yl+h, xl:xl+w, :],contours,-1,(0,255,0),thickness=2)
cv2.imshow('rgb', rgb)
print "press enter to continue"
cv2.waitKey()
zsel = xyz_w[yl:yl+h, xl:xl+w, 2]
mask = (mask%2==1) & np.isfinite(zsel)# & (zsel - table_height > -1)
mask &= valid_mask[yl:yl+h, xl:xl+w]
xyz_sel = xyz_w[yl:yl+h, xl:xl+w,:][mask.astype('bool')]
return clouds.downsample(xyz_sel, .01)
def extract_red(rgb, depth, T_w_k):
"""
extract red points and downsample
"""
hsv = cv2.cvtColor(rgb, cv2.COLOR_BGR2HSV)
h = hsv[:, :, 0]
s = hsv[:, :, 1]
v = hsv[:, :, 2]
red_mask = ((h < 10) | (h > 150)) & (s > 100) & (v > 100)
valid_mask = depth > 0
xyz_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
xyz_w = xyz_k.dot(T_w_k[:3, :3].T) + T_w_k[:3, 3][None, None, :]
z = xyz_w[:, :, 2]
z0 = xyz_k[:, :, 2]
if DEBUG_PLOTS:
cv2.imshow("z0", z0 / z0.max())
cv2.imshow("z", z / z.max())
cv2.imshow("rgb", rgb)
cv2.waitKey()
height_mask = xyz_w[:, :, 2] > .7 # TODO pass in parameter
good_mask = red_mask & height_mask & valid_mask
if DEBUG_PLOTS:
cv2.imshow("red", red_mask.astype('uint8') * 255)
cv2.imshow("above_table", height_mask.astype('uint8') * 255)
cv2.imshow("red and above table", good_mask.astype('uint8') * 255)
print "press enter to continue"
cv2.waitKey()
good_xyz = xyz_w[good_mask]
return clouds.downsample(good_xyz, .025)
rave_traj = [
r2r.convert(row)
for row in asarray(seg_info["joint_states"]["position"])
]
robot.SetActiveDOFs(r2r.rave_inds)
robot.SetActiveDOFValues(rave_traj[0])
handles = []
T_w_k = berkeley_pr2.get_kinect_transform(robot)
o = T_w_k[:3, 3]
x = T_w_k[:3, 0]
y = T_w_k[:3, 1]
z = T_w_k[:3, 2]
handles.append(env.drawarrow(o, o + .3 * x, .005, (1, 0, 0, 1)))
handles.append(env.drawarrow(o, o + .3 * y, .005, (0, 1, 0, 1)))
handles.append(env.drawarrow(o, o + .3 * z, .005, (0, 0, 1, 1)))
XYZ_k = clouds.depth_to_xyz(np.asarray(seg_info["depth"]), berkeley_pr2.f)
Twk = asarray(seg_info["T_w_k"])
XYZ_w = XYZ_k.dot(Twk[:3, :3].T) + Twk[:3, 3][None, None, :]
RGB = np.asarray(seg_info["rgb"])
handles.append(
env.plot3(XYZ_w.reshape(-1, 3), 2,
RGB.reshape(-1, 3)[:, ::-1] / 255.))
animate_traj.animate_traj(rave_traj, robot, pause=not args.nopause)
print "DONE"
trajoptpy.GetViewer(env).Idle()
robot.SetActiveDOFs(r2r.rave_inds)
robot.SetActiveDOFValues(rave_traj[0])
handles = []
T_w_k = berkeley_pr2.get_kinect_transform(robot)
o = T_w_k[:3,3]
x = T_w_k[:3,0]
y = T_w_k[:3,1]
z = T_w_k[:3,2]
handles.append(env.drawarrow(o, o+.3*x, .005,(1,0,0,1)))
handles.append(env.drawarrow(o, o+.3*y, .005,(0,1,0,1)))
handles.append(env.drawarrow(o, o+.3*z, .005,(0,0,1,1)))
XYZ_k = clouds.depth_to_xyz(np.asarray(seg_info["depth"]), berkeley_pr2.f)
Twk = asarray(seg_info["T_w_k"])
XYZ_w = XYZ_k.dot(Twk[:3,:3].T) + Twk[:3,3][None,None,:]
RGB = np.asarray(seg_info["rgb"])
handles.append(env.plot3(XYZ_w.reshape(-1,3), 2, RGB.reshape(-1,3)[:,::-1]/255.))
animate_traj.animate_traj(rave_traj, robot, pause = not args.nopause)
print "DONE"
trajoptpy.GetViewer(env).Idle()
env = openravepy.Environment()
viewer = trajoptpy.GetViewer(env)
if args.ropefile is None:
demo_rope_handle = env.plot3(demo_rope, 10, (0,1,1,1))
perturbed_rope_handle = env.drawlinestrip(perturbed_rope, 10, (1,0,1,1))
else:
perturbed_rope_handle = env.plot3(perturbed_rope, 16, (0,0,0,1))
try:
grabber = cloudprocpy.CloudGrabber()
grabber.startRGBD()
while True:
rgb, depth = grabber.getRGBD()
XYZ_k = clouds.depth_to_xyz(depth, berkeley_pr2.f)
Twk = berkeley_pr2.get_kinect_transform(pr2.robot)
XYZ_w = XYZ_k.dot(Twk[:3,:3].T) + Twk[:3,3][None,None,:]
cloud_handle = env.plot3(XYZ_w.reshape(-1,3), 2, rgb.reshape(-1,3)[:,::-1]/255.)
viewer.Step()
except:
import traceback
traceback.print_exc()
finally:
grabber.stop()
def getPC ():
global pc
r, d = grabber.getRGBD()
x = clouds.depth_to_xyz(d, berkeley_pr2.f)
pc = ru.xyzrgb2pc(x, r, 'camera_link_p')
import numpy as np
import itertools
import cv2
from rapprentice import ros2rave
import sensorsimpy, trajoptpy, openravepy
from rapprentice import berkeley_pr2
import scipy.optimize as opt
from rapprentice import clouds
DEBUG_PLOT = True
npzfile = np.load(args.fname)
depth_images = npzfile["depth_images"]
valid_masks = [(d > 0) & ((clouds.depth_to_xyz(d, 525)**2).sum(axis=2) < 1.5**2) for d in depth_images]
depth_images = depth_images/1000.
joint_positions = npzfile["joint_positions"]
joint_names = npzfile["names"] if "names" in npzfile else npzfile["joint_names"]
env = openravepy.Environment()
env.StopSimulation()
env.Load("robots/pr2-beta-static.zae")
robot = env.GetRobots()[0]
viewer = trajoptpy.GetViewer(env)
viewer.Step()
sensor = sensorsimpy.CreateFakeKinect(env)
r2r = ros2rave.RosToRave(robot, joint_names)
import numpy as np
import itertools
import cv2
from rapprentice import ros2rave
import sensorsimpy, trajoptpy, openravepy
from rapprentice import berkeley_pr2
import scipy.optimize as opt
from rapprentice import clouds
DEBUG_PLOT = True
npzfile = np.load(args.fname)
depth_images = npzfile["depth_images"]
valid_masks = [
(d > 0) & ((clouds.depth_to_xyz(d, 525)**2).sum(axis=2) < 1.5**2)
for d in depth_images
]
depth_images = depth_images / 1000.
joint_positions = npzfile["joint_positions"]
joint_names = npzfile["names"] if "names" in npzfile else npzfile["joint_names"]
env = openravepy.Environment()
env.StopSimulation()
env.Load("robots/pr2-beta-static.zae")
robot = env.GetRobots()[0]
viewer = trajoptpy.GetViewer(env)
viewer.Step()
sensor = sensorsimpy.CreateFakeKinect(env)