def add_object(self, obj_id, model_path, **kwargs):
"""See base class."""
# Color of the object model (the original color saved with the object model
# will be used if None).
surf_color = None
if 'surf_color' in kwargs:
surf_color = kwargs['surf_color']
# Load the object model.
model = inout.load_ply(model_path)
self.models[obj_id] = model
# Calculate the 3D bounding box of the model (will be used to set the near
# and far clipping plane).
bb = misc.calc_3d_bbox(
model['pts'][:, 0], model['pts'][:, 1], model['pts'][:, 2])
self.model_bbox_corners[obj_id] = np.array([
[bb[0], bb[1], bb[2]],
[bb[0], bb[1], bb[2] + bb[5]],
[bb[0], bb[1] + bb[4], bb[2]],
[bb[0], bb[1] + bb[4], bb[2] + bb[5]],
[bb[0] + bb[3], bb[1], bb[2]],
[bb[0] + bb[3], bb[1], bb[2] + bb[5]],
[bb[0] + bb[3], bb[1] + bb[4], bb[2]],
[bb[0] + bb[3], bb[1] + bb[4], bb[2] + bb[5]],
])
# Set texture/color of vertices.
self.model_textures[obj_id] = None
# Use the specified uniform surface color.
if surf_color is not None:
colors = np.tile(list(surf_color) + [1.0], [model['pts'].shape[0], 1])
# Set UV texture coordinates to dummy values.
texture_uv = np.zeros((model['pts'].shape[0], 2), np.float32)
# Use the model texture.
elif 'texture_file' in self.models[obj_id].keys():
model_texture_path = os.path.join(
os.path.dirname(model_path), self.models[obj_id]['texture_file'])
model_texture = inout.load_im(model_texture_path)
# Normalize the texture image.
if model_texture.max() > 1.0:
model_texture = model_texture.astype(np.float32) / 255.0
model_texture = np.flipud(model_texture)
self.model_textures[obj_id] = model_texture
# UV texture coordinates.
texture_uv = model['texture_uv']
# Set the per-vertex color to dummy values.
colors = np.zeros((model['pts'].shape[0], 3), np.float32)
# Use the original model color.
elif 'colors' in model.keys():
assert (model['pts'].shape[0] == model['colors'].shape[0])
colors = model['colors']
if colors.max() > 1.0:
colors /= 255.0 # Color values are expected in range [0, 1].
# Set UV texture coordinates to dummy values.
texture_uv = np.zeros((model['pts'].shape[0], 2), np.float32)
# Set the model color to gray.
else:
colors = np.ones((model['pts'].shape[0], 3), np.float32) * 0.5
# Set UV texture coordinates to dummy values.
texture_uv = np.zeros((model['pts'].shape[0], 2), np.float32)
# Set the vertex data.
if self.mode == 'depth':
vertices_type = [
('a_position', np.float32, 3),
('a_color', np.float32, colors.shape[1])
]
vertices = np.array(list(zip(model['pts'], colors)), vertices_type)
else:
if self.shading == 'flat':
vertices_type = [
('a_position', np.float32, 3),
('a_color', np.float32, colors.shape[1]),
('a_texcoord', np.float32, 2)
]
vertices = np.array(list(zip(model['pts'], colors, texture_uv)),
vertices_type)
elif self.shading == 'phong':
vertices_type = [
('a_position', np.float32, 3),
('a_normal', np.float32, 3),
('a_color', np.float32, colors.shape[1]),
('a_texcoord', np.float32, 2)
]
vertices = np.array(list(zip(model['pts'], model['normals'],
colors, texture_uv)), vertices_type)
else:
raise ValueError('Unknown shading type.')
# Create vertex and index buffer for the loaded object model.
self.vertex_buffers[obj_id] = vertices.view(gloo.VertexBuffer)
self.index_buffers[obj_id] = \
model['faces'].flatten().astype(np.uint32).view(gloo.IndexBuffer)
# Set shader for the selected shading.
if self.shading == 'flat':
rgb_fragment_code = _rgb_fragment_flat_code
elif self.shading == 'phong':
rgb_fragment_code = _rgb_fragment_phong_code
else:
raise ValueError('Unknown shading type.')
# Prepare the RGB OpenGL program.
rgb_program = gloo.Program(_rgb_vertex_code, rgb_fragment_code)
rgb_program.bind(self.vertex_buffers[obj_id])
if self.model_textures[obj_id] is not None:
rgb_program['u_use_texture'] = int(True)
rgb_program['u_texture'] = self.model_textures[obj_id]
else:
rgb_program['u_use_texture'] = int(False)
rgb_program['u_texture'] = np.zeros((1, 1, 4), np.float32)
self.rgb_programs[obj_id] = rgb_program
# Prepare the depth OpenGL program.
depth_program = gloo.Program(_depth_vertex_code,_depth_fragment_code)
depth_program.bind(self.vertex_buffers[obj_id])
self.depth_programs[obj_id] = depth_program
def subSampleImages(n, w, h, cenR, cenC):
# n=1
print n
start = time.time()
str = './RGB-D/rgb_noseg/color_%05d.png' % (n)
rgbImg = io.load_im(rgb_in_mpath.format(objId, n))
depImg = io.load_im(depth_in_mpath.format(objId, n))
labelImg = io.load_im(labelled_mpath.format(objId, n))
# rgbImg = cv2.imread(rgb_in_mpath.format(objID,n),cv2.IMREAD_UNCHANGED)
# labelImg = cv2.imread('./objCordLabel/Can/label_%05d.png'%(n),cv2.IMREAD_UNCHANGED)
# depImg = cv2.imread('./RGB-D/depth_noseg/depth_%05d.png'%(n),cv2.IMREAD_UNCHANGED)
# cv2.namedWindow('image', cv2.WINDOW_NORMAL)
# cv2.imshow('image', depImg)
# cv2.resizeWindow('image', depImg.shape[0], depImg.shape[1])
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# w = 40;
# h = 59;
# cenR= 262
# cenC= 373
width = 2 * w
height = 2 * h
(rows, cols, channels) = rgbImg.shape
# print(a)
ct = 0
for r in range(0, rows - height, 4):
for c in range(0, cols - width, 4):
patchCenR = r + h - 1
patchCenC = c + w - 1
ct = ct + 1
label = int(labelImg[r + h - 1, c + w - 1])
rgbSubImg = rgbImg[r:r + height - 1, c:c + width - 1]
depSubImg = depImg[r:r + height - 1, c:c + width - 1]
if ((patchCenR >= (cenR - h - 20)) and (patchCenR <=
(cenR + h + 20))
and (patchCenC >= (cenC - w - 20)) and (patchCenC <=
(cenC + w + 20))):
if (label > 0):
# rgbPatch = Image.fromarray(rgbSubImg, 'RGB')
# rgbPatch.save(labelledRGB_patch_path.format(objId,label,n,ct))
#
# depthPatch = Image.fromarray(depSubImg)
# depthPatch.save(labelledDepth_patch_path.format(objId,label,n, ct))
# trainInput.write('%03d_%04d_%05d.png %s\n' % (label, n, ct, label))
lblCnt[label - 1, 0] = lblCnt[label - 1, 0] + 1
else:
if (((r + 1) % 10 == 0) and ((c + 1) % 10 == 0)):
# rgbPatch = Image.fromarray(rgbSubImg, 'RGB')
# rgbPatch.save(labelledRGB_patch_path.format(objId,label,n, ct))
# #
# depthPatch = Image.fromarray(depSubImg)
# depthPatch.save(labelledDepth_patch_path.format(objId,label,n, ct))
trainInput.write('%03d_%04d_%05d.png %s\n' %
(label, n, ct, label))
lblCnt[64, 0] = lblCnt[64, 0] + 1
else:
if ((r % 160 == 0) and (c % 160 == 0)):
# rgbPatch = Image.fromarray(rgbSubImg, 'RGB')
# rgbPatch.save(labelledRGB_patch_path.format(objId,label,n, ct))
#
# depthPatch = Image.fromarray(depSubImg)
# depthPatch.save(labelledDepth_patch_path.format(objId,label,n, ct))
# trainInput.write('%03d_%04d_%05d.png %s\n' % (label, n, ct, label))
lblCnt[64, 0] = lblCnt[64, 0] + 1
end = time.time()
print(end - start)
return lblCnt
'R':
gt['cam_R_m2c'],
't':
gt['cam_t_m2c'],
'text_info': [{
'name': '',
'val': '{}:{}'.format(gt['obj_id'], gt_id),
'fmt': ''
}]
})
# Load the color and depth images and prepare images for rendering.
rgb = None
if p['vis_rgb']:
if 'rgb' in dp_split['im_modalities']:
rgb = inout.load_im(dp_split['rgb_tpath'].format(
scene_id=scene_id, im_id=im_id))[:, :, :3]
elif 'gray' in dp_split['im_modalities']:
gray = inout.load_im(dp_split['gray_tpath'].format(
scene_id=scene_id, im_id=im_id))
rgb = np.dstack([gray, gray, gray])
else:
raise ValueError('RGB nor gray images are available.')
depth = None
if p['vis_depth_diff'] or (p['vis_rgb']
and p['vis_rgb_resolve_visib']):
depth = inout.load_depth(dp_split['depth_tpath'].format(
scene_id=scene_id, im_id=im_id))
depth *= scene_camera[im_id]['depth_scale'] # Convert to [mm].
# Path to the output RGB visualization.
rotY180 = np.array([-1, 0, 0, 0, 1, 0, 0, 0, -1]).reshape([3, 3])
rotZ180 = np.array([-1, 0, 0, 0, -1, 0, 0, 0, 1]).reshape([3, 3])
K = np.zeros((3, 3))
K[0, 0] = 572.41140
K[0, 2] = 325.2611
K[1, 1] = 573.57043
K[1, 2] = 242.04899
K[2, 2] = 1
with open(scene_gt_mpath.format(scene_id), 'r') as f:
gts = yaml.load(f, Loader=yaml.CLoader)
rgbImg = io.load_im(rgb_in_mpath.format(objId, start))
depImg = io.load_im(depth_in_mpath.format(objId, start))
[r, t] = getRotTrans(start, objId)
# print r
# print t
r = r.dot(rotY180)
r = r.dot(rotZ180)
modelCen_h = np.array([0, 0, 0, 1]).reshape(4, 1)
P = K.dot(np.hstack((r, t)))
pts_im = P.dot(modelCen_h)
pts_im /= pts_im[2, :]
cenC0 = int(pts_im[0, 0])
cenR0 = int(pts_im[1, 0])
dep0 = depImg[cenR0, cenC0]
def pixelOverwrite(p1, p2):
if p1[0] < p2[0]:
return p1
else:
return p2
def mergeMasks(a, b):
return list(map(pixelOverwrite, a, b))
for im_id in range(0, numImages):
rgb = io.load_im(rgb_in_mpath.format(centre_obj_id, im_id))
depth = io.load_im(rgb_in_mpath.format(centre_obj_id, im_id))
depthMask = np.ones([480, 640]) * 100000
gt_info_im = gt_info[im_id]
numObjs = len(gt_info_im)
objList = np.arange(0, numObjs)
print objList
start = time.time()
depthMaskedImage = list(map(createMask, objList))
mergedDepthMaskedImage = [zip(depthMask.flatten(), np.zeros(480 * 640))
] + depthMaskedImage
maskedImage = reduce(mergeMasks, depthMaskedImage)
misc.ensure_dir(
os.path.dirname(
out_rgb_tpath.format(out_path=out_path, obj_id=obj_id, im_id=0)))
misc.ensure_dir(
os.path.dirname(
out_depth_tpath.format(out_path=out_path, obj_id=obj_id, im_id=0)))
# Load model.
model_path = dp_model['model_tpath'].format(obj_id=obj_id)
model = inout.load_ply(model_path)
# Load model texture.
if 'texture_file' in model:
model_texture_path =\
os.path.join(os.path.dirname(model_path), model['texture_file'])
model_texture = inout.load_im(model_texture_path)
else:
model_texture = None
scene_camera = {}
scene_gt = {}
im_id = 0
for radius in radii:
# Sample viewpoints.
view_sampler_mode = 'hinterstoisser' # 'hinterstoisser' or 'fibonacci'.
views, views_level = view_sampler.sample_views(
min_n_views, radius, dp_split_test['azimuth_range'],
dp_split_test['elev_range'], view_sampler_mode)
misc.log('Sampled views: ' + str(len(views)))
# out_views_vis_path = out_views_vis_tpath.format(
scene_info_mpath = base_path + 'train/{:02d}/info.yml'
scene_gt_mpath = base_path + 'train/{:02d}/gt.yml'
w, h = 640, 480
with open(scene_gt_mpath.format(scene_id), 'r') as f:
gt_info = yaml.load(f, Loader=yaml.CLoader)
model = io.load_ply(model_mpath.format(objId))
numImages = len(gt_info)
for im_id in range(0, 1):
print rgb_in_mpath.format(objId, im_id)
maskData = np.zeros((h, w), dtype=np.uint8)
rgb = io.load_im(rgb_in_mpath.format(objId, im_id))
[r, t] = getRotTrans(gt_info, im_id, objId)
# print r
# print t
m_rgb = renderer.render(model, (w, h),
cam_mat,
r,
t,
shading='phong',
mode='rgb')
# time.sleep(1);
r1, g1, b1 = 0, 0, 0 # Original value
r2, g2, b2 = 255, 255, 255 # Value that we want to replace it with
red, green, blue = m_rgb[:, :, 0], m_rgb[:, :, 1], m_rgb[:, :, 2]
minZ = min(bb3D[:, 2])
maxZ = max(bb3D[:, 2])
#Find the dimension of bin along each dimension
unitX = (maxX - minX) / bin_dim
unitY = (maxY - minY) / bin_dim
unitZ = (maxZ - minZ) / bin_dim
numImages = len(gts)
for im_id in range(0, numImages):
print im_id
objLabel = np.zeros([h, w, 3], dtype=np.uint8)
labelImg = np.zeros([h, w], dtype=np.uint8)
seg = io.load_im(seg_in_mpath.format(objId, im_id))
depth = io.load_im(depth_in_mpath.format(objId, im_id))
[r, t] = getRotTrans(im_id, objId)
t = t / 1000
print r
print t
objC = np.array([])
numPt = 0
for col in range(0, 640, 1):
for row in range(0, 480, 1):
if (seg[row, col] > 0):
d = depth[row, col]
if (d != 0):
numPt = numPt + 1