def captureDepth(model, rotations, imageWidth=224, imageHeight=224, cameraZTranslation=2.5, lightIntensity=2.0, depthBegin=1, depthEnd=5):
# Construct an offline scene
scene = pyrender.Scene()
# add parts
for part in model.parts:
partMesh = pyrender.Mesh.from_trimesh(part.mesh, smooth=False)
scene.add(partMesh)
# add camera
renderCamera = pyrender.PerspectiveCamera(
yfov=np.pi / 3.0, aspectRatio=imageWidth/imageHeight)
cameraNode = pyrender.Node(camera=renderCamera, matrix=np.eye(4))
cameraNode.translation[2] = cameraZTranslation
scene.add_node(cameraNode)
# add light
light = pyrender.DirectionalLight(
color=[1.0, 1.0, 1.0], intensity=lightIntensity)
lightNode = pyrender.Node(light=light, matrix=np.eye(4))
scene.add_node(lightNode)
# initialize offscreen renderer
offscreenRenderer = pyrender.OffscreenRenderer(
viewport_width=imageWidth, viewport_height=imageHeight, point_size=1.0)
# render
result = []
for rotation in rotations:
result.append(renderSceneWithRotation(offscreenRenderer,
scene, rotation, depthBegin, depthEnd))
return result
def render_depth_map_mesh(
self,
K,
R,
t,
height,
width,
znear=0.05,
zfar=1500,
):
scene = pyrender.Scene()
mesh = pyrender.Mesh(
primitives=[
pyrender.Primitive(
positions=self.verts,
normals=self.normals,
color_0=self.colors,
indices=self.faces,
mode=pyrender.GLTF.TRIANGLES,
)
],
is_visible=True,
)
mesh_node = pyrender.Node(mesh=mesh, matrix=np.eye(4))
scene.add_node(mesh_node)
cam = pyrender.IntrinsicsCamera(
fx=K[0, 0],
fy=K[1, 1],
cx=K[0, 2],
cy=K[1, 2],
znear=znear,
zfar=zfar,
)
T = np.eye(4)
T[:3, :3] = R.T
T[:3, 3] = (-R.T @ t.reshape(3, 1)).ravel()
cv2gl = np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0],
[0, 0, 0, 1]])
T = T @ cv2gl
cam_node = pyrender.Node(camera=cam, matrix=T)
scene.add_node(cam_node)
light = pyrender.DirectionalLight(color=np.ones(3), intensity=3)
light_node = pyrender.Node(light=light, matrix=np.eye(4))
scene.add_node(light_node, parent_node=cam_node)
render = pyrender.OffscreenRenderer(self.width, self.height)
color, depth = render.render(scene)
# if self.vis:
# depth[depth <= 0] = np.NaN
# depth = co.plt.image_colorcode(depth)
# imwrite(dm_path.with_suffix(".jpg"), depth)
return depth
def visualize_prediction(target_r, target_t, pred_r, pred_t, model_id):
"""
Function to visualize prediction of object pose
"""
#rotation = np.eye(3)
# Get random translation
translation = np.random.uniform(-0.07, 0.07, 3)
translation[2] += 1
translation[2] *= -1
# Build transformation matrix
mat = np.eye(4)
mat[:3, :3] = rotation
mat[:3, 3] = translation
# Create light object
light = pyrender.PointLight(color=[1.0, 1.0, 1.0],
intensity=np.random.normal(10, 5))
# Create a scene
scene = pyrender.Scene()
# Create camera node object
nc = pyrender.Node(camera=camera, matrix=np.eye(4))
# Create object node object
no = pyrender.Node(mesh=model, matrix=mat)
# Create light node object
nl = pyrender.Node(light=light, matrix=np.eye(4))
# Add camera to scene
scene.add_node(nc)
# Add object to scene
scene.add_node(no, parent_node=nc)
# Add light to scene
scene.add_node(nl, parent_node=nc)
# Create object renderer
render = pyrender.OffscreenRenderer(image_shape[0], image_shape[1])
# Render images
color, depth = render.render(scene)
# Convert color
color = cv2.cvtColor(color, cv2.COLOR_BGR2RGB)
if show_image:
# Show image
cv2.imshow("image", color)
cv2.waitKey(0)
def _init_light(self):
"""
Set up light
"""
# Load light from config file
light = self.conf.sensor.lights
origin = np.array(light.origin)
xyz = []
if light.polar:
# Apply polar coordinates
thetas = light.xrtheta.thetas
rs = light.xrtheta.rs
xs = light.xrtheta.xs
for i in range(len(thetas)):
theta = np.pi / 180 * thetas[i]
xyz.append(
[xs[i], rs[i] * np.cos(theta), rs[i] * np.sin(theta)])
else:
# Apply cartesian coordinates
xyz = np.array(light.xyz.coords)
colors = np.array(light.colors)
intensities = light.intensities
# Save light nodes
self.light_nodes = []
self.light_poses0 = []
for i in range(len(colors)):
color = colors[i]
light_pose_0 = euler2matrix(angles=[0, 0, 0],
translation=xyz[i] + origin)
light = pyrender.PointLight(color=color, intensity=intensities[i])
light_node = pyrender.Node(light=light, matrix=light_pose_0)
self.scene.add_node(light_node)
self.light_nodes.append(light_node)
self.light_poses0.append(light_pose_0)
self.current_light_nodes.append(light_node)
# Add extra light node into scene_depth
light_node_depth = pyrender.Node(light=light, matrix=light_pose_0)
self.scene_depth.add_node(light_node_depth)
def add_light(self, light_color=np.ones(3)):
# light = pyrender.DirectionalLight(color=np.ones(3), intensity=1.0)
# light_pose = np.eye(4)
# self.scene.add(light, pose=light_pose)
thetas = np.pi * np.array([1.0 / 6.0, 1.0 / 6.0, 1.0 / 6.0])
phis = np.pi * np.array([0.0, 2.0 / 3.0, 4.0 / 3.0])
for phi, theta in zip(phis, thetas):
xp = np.sin(theta) * np.cos(phi)
yp = np.sin(theta) * np.sin(phi)
zp = np.cos(theta)
z = np.array([xp, yp, zp])
z = z / np.linalg.norm(z)
x = np.array([-z[1], z[0], 0.0])
if np.linalg.norm(x) == 0:
x = np.array([1.0, 0.0, 0.0])
x = x / np.linalg.norm(x)
y = np.cross(z, x)
matrix = np.eye(4)
matrix[:3,:3] = np.c_[x,y,z]
node = pyrender.Node(
light=pyrender.DirectionalLight(color=light_color, intensity=1.0),
matrix=matrix
)
self.scene.add_node(node)
def _create_node_from_points(points,
name=None,
pose=None,
color=None,
material=None,
radius=None,
n_divs=20):
points = np.asanyarray(points)
if points.ndim == 1:
points = np.array([points])
if pose is None:
pose = np.eye(4)
else:
pose = pose.matrix
# Create vertex colors if needed
if color is not None:
color = np.asanyarray(color, dtype=np.float)
if color.ndim == 1 or len(color) != len(points):
color = np.repeat(color[np.newaxis, :], len(points), axis=0)
if radius is not None:
poses = None
mesh = trimesh.creation.uv_sphere(radius, [n_divs, n_divs])
if color is not None:
mesh.visual.vertex_colors = color[0]
poses = np.tile(np.eye(4), (len(points), 1)).reshape(len(points), 4, 4)
poses[:, :3, 3::4] = points[:, :, None]
m = pyrender.Mesh.from_trimesh(mesh, material=material, poses=poses)
else:
m = pyrender.Mesh.from_points(points, colors=color)
return pyrender.Node(mesh=m, name=name, matrix=pose)
def rnder_one_scene(args, mesh_pth, obj_pose, camera_pose):
try:
fuze_trimesh = trimesh.load(mesh_pth)
mesh = pyrender.Mesh.from_trimesh(fuze_trimesh)
except Exception:
print("Error loadding from {}".format(mesh_pth))
return
scene = pyrender.Scene(ambient_light=[0.9, 0.9, 0.9])
nm = pyrender.Node(mesh=mesh, matrix=obj_pose)
scene.add_node(nm)
h, w = args.h, args.w
if type(args.K) == list:
K = np.array(args.K).reshape(3, 3)
else:
K = args.K
camera = pyrender.IntrinsicsCamera(K[0][0], K[1][1], K[0][2], K[1][2])
scene.add(camera, pose=camera_pose)
light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=1.0)
# light = pyrender.light.Light(color=[1.0, 1.0, 1.0], intensity=5)
scene.add(light)
r = pyrender.OffscreenRenderer(w, h)
color, depth = r.render(scene)
return color, depth
def _init_gel(self):
"""
Add gel surface in the scene
"""
# Create gel surface (flat/curve surface based on config file)
gel_trimesh = self._generate_gel_trimesh()
mesh_gel = pyrender.Mesh.from_trimesh(gel_trimesh, smooth=False)
self.gel_pose0 = np.eye(4)
self.gel_node = pyrender.Node(mesh=mesh_gel, matrix=self.gel_pose0)
self.scene.add_node(self.gel_node)
# Add extra gel node into scene_depth
self.gel_node_depth = pyrender.Node(mesh=mesh_gel,
matrix=self.gel_pose0)
self.scene_depth.add_node(self.gel_node_depth)
def _save_trimesh(env, scene, rend):
"""Save the cloth as an obj so we can render it.
"""
cloth = [[p.x, p.y, p.z] for p in env.cloth.pts]
cloth = np.array(cloth)
assert cloth.shape == (625,3), cloth.shape
wh = 25
faces = []
for r in range(wh-1):
for c in range(wh-1):
pp = r*wh + c
faces.append( [pp, pp+wh, pp+1] )
faces.append( [pp+1, pp+wh, pp+wh+1] )
faces = np.array(faces)
assert faces.shape == (1152,3), faces.shape
# Save as obj so that we can render in Blender more nicely. :-)
tm = trimesh.Trimesh(vertices=cloth, faces=faces)
num = len([x for x in os.listdir('examples') if x[-4:] == '.obj'])
tm.export('examples/trimesh_cloth_{}.obj'.format(str(num).zfill(3)))
# Create mesh+node, add node, save images, then remove mesh+node.
mesh = pyrender.Mesh.from_trimesh(tm, smooth=True)
mesh_node = pyrender.Node(mesh=mesh, matrix=np.eye(4))
scene.add_node(mesh_node)
_save_render_images(scene, rend)
scene.remove_node(mesh_node)
def show_visualisation(self, cfg=None, use_collision=False):
"""
Initialisieren der Visualisierung mit initalen Joint-Werten
wurde aus urdfpy kopiert und sinnvoll angepasst
:param cfg: configuration
:param use_collision: collision mesh or visual
:return: scene
"""
#für collision == True wird die collisoion visualisierung genommen
if use_collision:
fk = self.robot.collision_trimesh_fk(cfg=cfg)
else:
fk = self.robot.visual_trimesh_fk(cfg=cfg)
#visualisierung:
self.scene = pyrender.Scene()
#hier werden die aktuellen meshes ausgelesen und visualisiert
for tm in fk:
pose = fk[tm]
mesh = pyrender.Mesh.from_trimesh(tm, smooth=False)
nm = pyrender.Node(mesh=mesh,
matrix=np.eye(4)) #node wird gegettet
self.meshes.append(nm)
#visualisierung:
self.scene.add_node(nm)
def render(obj_path, camera_mat, return_depth=False, im_size=128):
fuze_trimesh = trimesh.load(obj_path)
if type(fuze_trimesh) == trimesh.base.Trimesh:
m = pyrender.Mesh.from_trimesh(fuze_trimesh)
scene = pyrender.Scene()
scene.add_node(pyrender.Node(mesh=m))
else:
assert type(
fuze_trimesh) == trimesh.scene.scene.Scene, "Unrognized file"
scene = pyrender.Scene.from_trimesh_scene(fuze_trimesh)
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
s = np.sqrt(2) / 2
scene.add(camera, pose=camera_mat)
light = pyrender.SpotLight(color=np.ones(3),
intensity=4.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_mat)
light = pyrender.SpotLight(color=np.ones(3),
intensity=6.0,
innerConeAngle=0.2 * np.pi)
light_pose = np.array(
[[0, 1, 0, 0], [0, 0, 1, 1], [1, 0, 0, 0], [0, 0, 0, 1]],
dtype=np.float32)
scene.add(light, pose=light_pose)
r = pyrender.OffscreenRenderer(im_size, im_size)
color, depth = r.render(scene)
r.delete()
if return_depth:
return color, depth
return color
def _load_object(self, path, scale):
if (path, scale) in self._cache:
return self._cache[(path, scale)]
obj = sample.Object(path)
obj.rescale(scale)
tmesh = obj.mesh
tmesh_mean = np.mean(tmesh.vertices, 0)
tmesh.vertices -= np.expand_dims(tmesh_mean, 0)
lbs = np.min(tmesh.vertices, 0)
ubs = np.max(tmesh.vertices, 0)
object_distance = np.max(ubs - lbs) * 5
mesh = pyrender.Mesh.from_trimesh(tmesh)
context = {
'tmesh': copy.deepcopy(tmesh),
'distance': object_distance,
'node': pyrender.Node(mesh=mesh),
'mesh_mean': np.expand_dims(tmesh_mean, 0),
}
self._cache[(path, scale)] = context
return self._cache[(path, scale)]
def _create_raymond_lights(self):
thetas = np.pi * np.array([1.0 / 6.0, 1.0 / 6.0, 1.0 / 6.0])
phis = np.pi * np.array([0.0, 2.0 / 3.0, 4.0 / 3.0])
nodes = []
for phi, theta in zip(phis, thetas):
xp = np.sin(theta) * np.cos(phi)
yp = np.sin(theta) * np.sin(phi)
zp = np.cos(theta)
z = np.array([xp, yp, zp])
z = z / np.linalg.norm(z)
x = np.array([-z[1], z[0], 0.0])
if np.linalg.norm(x) == 0:
x = np.array([1.0, 0.0, 0.0])
x = x / np.linalg.norm(x)
y = np.cross(z, x)
matrix = np.eye(4)
matrix[:3, :3] = np.c_[x, y, z]
nodes.append(
pyrender.Node(light=pyrender.DirectionalLight(color=np.ones(3),
intensity=1.0),
matrix=matrix))
return nodes
def update_graph(self, scene_graph, materials_only):
"""Update scene graph.
This function rebuild scene completely each time when something changed.
TODO: update changed nodes instead.
Arguments:
scene_graph {SceneGraph} -- scene description
materials_only {bool} -- update only shape materials
"""
for uid, node in self._bullet_nodes.items():
self.remove_node(node)
self._bullet_nodes = {}
self._seg_node_map = {}
for uid, body in scene_graph.nodes.items():
node = pyr.Node(uid)
self.add_node(node)
self._bullet_nodes[uid] = node
for shape in body.shapes:
if shape.material is None:
material = None
else:
material = pyr.MetallicRoughnessMaterial(
baseColorFactor=shape.material.diffuse_color,
metallicFactor=0.2,
roughnessFactor=0.8,
alphaMode='BLEND')
texture = shape.material.diffuse_texture
if texture is not None:
if texture.bitmap is not None:
image = Image.fromarray(texture.bitmap)
else:
image = Image.open(
os.path.abspath(texture.filename))
texture = pyr.Texture(source=image,
source_channels=image.mode)
material.baseColorTexture = texture
if shape.mesh is None:
mesh = primitive_mesh(shape)
elif shape.mesh.data is None:
mesh = trimesh.load(os.path.abspath(shape.mesh.filename))
else:
data = shape.mesh.data
mesh = trimesh.Trimesh(
vertices=data.vertices,
vertex_normals=data.normals,
faces=data.faces,
visual=trimesh.visual.TextureVisuals(uv=data.uvs))
mesh = pyr.Mesh.from_trimesh(mesh, material=material)
mesh_node = self.add(mesh,
pose=shape.pose.matrix.T,
parent_node=node)
self._seg_node_map[mesh_node] = mask_to_rgb(
body.body, body.link)
def draw_circle(scene, center, radius, color, thickness):
circle_trimesh = trimesh.creation.annulus(radius, radius + thickness, 0.1)
circle_trimesh.visual.face_colors = np.repeat(
np.array(color)[np.newaxis, :], circle_trimesh.faces.shape[0], axis=0)
circle_mesh = pyrender.Mesh.from_trimesh(circle_trimesh, smooth=False)
circle_node = pyrender.Node(mesh=circle_mesh,
translation=np.array([center[0], center[1],
2]))
scene.add_node(circle_node)
def draw_box(scene, center, extends, color):
box_trimesh = trimesh.creation.box(extents=extends)
box_trimesh.visual.face_colors = np.repeat(np.array(color)[np.newaxis, :],
box_trimesh.faces.shape[0],
axis=0)
box_mesh = pyrender.Mesh.from_trimesh(box_trimesh, smooth=False)
box_node = pyrender.Node(mesh=box_mesh,
translation=np.array(
[center[0], center[1], center[2]]))
scene.add_node(box_node)
def _create_object_node(obj: meshutils.Object3D):
smooth = True
# Turn smooth shading off if vertex normals are unreliable.
if obj.are_normals_corrupt():
smooth = False
mesh = pyrender.Mesh.from_trimesh(obj.meshes, smooth=smooth)
node = pyrender.Node(mesh=mesh)
return node
def add_obj(self, mesh, matrix=np.eye(4), color='lightblue'):
tri = mesh.tri_mesh
if isinstance(color, (str)):
color = trimesh.visual.color.hex_to_rgba(cnames[color])
color[-1] = 200
tri.visual.face_colors = color
tri.visual.vertex_colors = color
render_mesh = pyrender.Mesh.from_trimesh(tri)
n = pyrender.Node(mesh=render_mesh, matrix=matrix)
self.add_node(n)
def test2():
model_path = 'data/models/basic/cow.obj'
# load the cow model
tm = trimesh.load(model_path)
tm.visual.vertex_colors = np.random.uniform(
size=tm.visual.vertex_colors.shape)
tm.visual.face_colors = np.random.uniform(size=tm.visual.face_colors.shape)
mesh = pyrender.Mesh.from_trimesh(tm, smooth=False)
light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=2.0)
cam = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.414)
nm = pyrender.Node(mesh=mesh, matrix=np.eye(4))
nl = pyrender.Node(light=light, matrix=np.eye(4))
nc = pyrender.Node(camera=cam, matrix=np.eye(4))
scene = pyrender.Scene(ambient_light=[1.0, 1.0, 1.0], bg_color=gray)
scene.add_node(nm)
scene.add_node(nl, parent_node=nm)
scene.add_node(nc, parent_node=nm)
pyrender.Viewer(scene, use_raymond_lighting=True)
def __init__(self):
"""Construct a Scene."""
super().__init__()
self._bullet_nodes = {}
self._seg_node_map = {}
self.bg_color = (0.7, 0.7, 0.8)
self.ambient_light = (0.2, 0.2, 0.2)
self._camera_node = pyr.Node(camera=pyr.PerspectiveCamera(
np.deg2rad(60.0)),
translation=(0.0, -2.0, 3.0),
rotation=(-0.472, 0.0, 0.0, 0.882))
self.add_node(self._camera_node)
self._light_node = pyr.Node(light=pyr.DirectionalLight(color=(0.8, 0.8,
0.8),
intensity=5.0),
translation=(-0.8, -0.2, 2.0),
rotation=(-0.438, 0.342, -0.511, 0.655))
self.add_node(self._light_node)
def color(self, new_val):
self.tmesh.visual.vertex_colors = [
new_val for i in range(0, self.tmesh.vertices.shape[0])
]
self.mesh = pyrender.Mesh.from_trimesh(self.tmesh)
# Re add itself to the scene I guess...
self.scene.remove_node(self.node)
self.node = None
self.node = pyrender.Node(mesh=self.mesh, matrix=np.eye(4))
self.scene.add_node(self.node)
def actor_to_node(self, actor, flags):
shapes = [
s for s in actor.get_atached_shapes()
if PyPhysxViewer.has_shape_any_of_flags(s, flags)
]
if len(shapes) == 0:
return None
all_nodes: List[pyrender.Node] = []
for s in shapes:
all_nodes += self.shape_to_nodes(s)
return pyrender.Node(children=all_nodes)
def __init__(self, object_name_or_mesh, K, camera_name, mesh_scale=1.0):
"""
object_name_or_mesh: either object name string (for objects),
or {'vertices': ..., 'faces': ...} (for hand mesh)
K: 3x3 intrinsics matrix
mesh_scale: scale factor applied to the mesh (1.0 for hand, 1e-3 for object)
"""
self.K = K
self.camera_name = camera_name
if camera_name == 'kinect2_middle':
self.flip_fn = lambda x: cv2.flip(cv2.flip(x, 0), 1)
self.out_imsize = (960, 540)
elif camera_name == 'kinect2_left':
self.flip_fn = lambda x: cv2.flip(cv2.transpose(x), 1)
self.out_imsize = (540, 960)
elif camera_name == 'kinect2_right':
self.flip_fn = lambda x: cv2.flip(cv2.transpose(x), 0)
self.out_imsize = (540, 960)
else:
raise NotImplementedError
# mesh
if isinstance(object_name_or_mesh, str):
filename = osp.join('data', 'object_models',
'{:s}.ply'.format(object_name_or_mesh))
mesh_t = trimesh.load_mesh(filename)
elif isinstance(object_name_or_mesh, dict):
mesh_t = trimesh.Trimesh(vertices=object_name_or_mesh['vertices'],
faces=object_name_or_mesh['faces'])
else:
raise NotImplementedError
mesh_t.apply_transform(np.diag([mesh_scale, mesh_scale, mesh_scale,
1]))
self.oX = mesh_t.vertices
mesh = pyrender.Mesh.from_trimesh(mesh_t)
self.scene = pyrender.Scene()
self.scene.add(mesh, pose=np.eye(4))
# camera
camera = pyrender.IntrinsicsCamera(K[0, 0],
K[1, 1],
K[0, 2],
K[1, 2],
znear=0.1,
zfar=2.0)
self.camera_node = pyrender.Node(camera=camera, matrix=np.eye(4))
self.scene.add_node(self.camera_node)
self.cTopengl = np.eye(4)
self.cTopengl[:3, :3] = txe.euler2mat(np.pi, 0, 0)
# renderer object
self.renderer = pyrender.OffscreenRenderer(960, 540)
def _add_table_node(self):
"""
Adds table mesh and sets pose
"""
if self._viewer:
return
table_mesh = trimesh.creation.box(self._table_dims)
mesh = pyrender.Mesh.from_trimesh(table_mesh)
table_node = pyrender.Node(mesh=mesh, name='table')
self._scene.add_node(table_node)
self._scene.set_pose(table_node, self._table_pose)
def __init__(self,
render_scene=None,
viewport_size=None,
render_flags=None,
viewer_flags=None,
registered_keys=None,
run_in_thread=True,
video_filename=None,
**kwargs):
"""
Use render_scene to specify camera or lighting or additional geometries if required.
Additional viewer flags:
- axes_scale - size of coordinate axes
"""
if render_scene is None:
render_scene = pyrender.Scene()
render_scene.bg_color = np.array([0.75] * 3)
if render_scene.main_camera_node is None:
cam = pyrender.PerspectiveCamera(yfov=np.deg2rad(60),
aspectRatio=1.414,
znear=0.005)
cam_pose = np.eye(4)
cam_pose[:3, 3] = RoboticTrackball.spherical_to_cartesian(
3., 0., np.deg2rad(45.), target=np.zeros(3))
cam_pose[:3, :3] = RoboticTrackball.look_at_rotation(
eye=cam_pose[:3, 3], target=np.zeros(3), up=[0, 0, 1])
nc = pyrender.Node(camera=cam, matrix=cam_pose)
render_scene.add_node(nc)
render_scene.main_camera_node = nc
_viewer_flags = {
'view_center': np.zeros(3),
'window_title': 'PyPhysX Scene Viewer',
'show_world_axis': True,
'show_mesh_axes': False,
'axes_scale': 0.5,
'use_raymond_lighting': True,
'plane_grid_spacing': 1.,
'plane_grid_num_of_lines': 10,
'spheres_count': [8, 8],
}
if viewer_flags is not None:
_viewer_flags.update(viewer_flags)
super().__init__(render_scene, viewport_size, render_flags,
_viewer_flags, registered_keys, run_in_thread,
**kwargs)
self.nodes_and_actors = []
fps = self.viewer_flags['refresh_rate']
self.video_writer = imageio.get_writer(
video_filename, fps=fps) if video_filename is not None else None
def addObjectFromDict(self, dict, nodeName, pose=np.eye(4)):
assert 'vertices' in dict.keys(), 'Vertices not present in the mesh dict...'
assert 'faces' in dict.keys(), 'Faces not present in the mesh dict...'
# fuze_trimesh = trimesh.load({}, None, None, dict)
# mesh = pyrender.Mesh.from_trimesh(fuze_trimesh)
if 'vertex_colors' in dict.keys():
triMesh = trimesh.Trimesh(vertices=dict['vertices'], faces=dict['faces'], vertex_colors=dict['vertex_colors'])
else:
triMesh = trimesh.Trimesh(vertices=dict['vertices'], faces=dict['faces'])
mesh = pyrender.Mesh.from_trimesh(triMesh)
self.nodesDict[nodeName] = pyrender.Node(mesh=mesh, matrix=pose)
self.scene.add_node(self.nodesDict[nodeName])
def load_obj_model(self, fname):
self.obj_nodes = []
data = trimesh.load(fname, force='scene')
scenetmp = pyrender.Scene.from_trimesh_scene(data)
self.base_M = self.create_base_correct_matrix(scene=scenetmp,
mesh_rotate=[0, 0, 0])
for x in scenetmp.meshes:
self.obj_nodes.extend([pyrender.Node(mesh=x, matrix=np.eye(4))])
for j in range(len(self.obj_nodes)):
self.scene.add_node(self.obj_nodes[j])
def render(obj, cam_pos):
# create scene
scene = pyrender.Scene()
# MESHES
obj_trimesh = trimesh.load(obj)
obj_trimesh.vertices = scaleVertices(obj_trimesh.vertices,
feature_range=(-1, 1))
mesh = pyrender.Mesh.from_trimesh(obj_trimesh)
nm = pyrender.Node(mesh=mesh, matrix=np.eye(4))
scene.add_node(nm)
# CAMERA
cam = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0)
nc = pyrender.Node(camera=cam, matrix=cam_pos)
scene.add_node(nc)
# RENDER
flags = RenderFlags.DEPTH_ONLY
r = pyrender.OffscreenRenderer(400, 400)
depth = r.render(scene, flags=flags)
return (depth)
def __init__(self, options):
"""Constructor."""
# RGB frames path
self.rgb_path = os.path.join('datasets', 'ycb-video', options.video_id,
'rgb')
# Estimates path
self.estimates_path = os.path.join('results', options.algorithm, 'nrt',
options.mask_set, 'validation',
options.object, options.video_id,
'object-tracking_estimate.txt')
# Mesh path
object_mesh_path = os.path.join('models', 'YCB_models', 'models',
options.object, 'textured')
if options.mesh_type == 'low-quality':
object_mesh_path += '_simple'
object_mesh_path += '.obj'
# Mesh
trimesh_mesh = trimesh.load(object_mesh_path)
mesh = pyrender.Mesh.from_trimesh(trimesh_mesh)
# Scene
self.scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0])
# Camera
fx = 1066.8
fy = 1067.5
cx = 312.99
cy = 241.31
width = 640
height = 480
camera_transform = Quaternion(axis=[1.0, 0.0, 0.0],
angle=numpy.pi).transformation_matrix
self.camera = pyrender.IntrinsicsCamera(fx=fx, fy=fy, cx=cx, cy=cy)
self.scene.add(self.camera, pose=camera_transform)
# Light
self.light = pyrender.PointLight(intensity=20.0)
self.scene.add(self.light)
# Object node
self.mesh_node = pyrender.Node(mesh=mesh, matrix=numpy.eye(4))
self.scene.add_node(self.mesh_node)
# Renderer
self.renderer = pyrender.OffscreenRenderer(width, height)
def _init_camera(self):
"""
Set up camera
"""
self.camera_nodes = []
self.camera_zero_poses = []
conf_cam = self.conf.sensor.camera
self.nb_cam = len(conf_cam)
for i in range(self.nb_cam):
cami = conf_cam[i]
camera = pyrender.PerspectiveCamera(
yfov=np.deg2rad(cami.yfov),
znear=cami.znear,
)
camera_zero_pose = euler2matrix(
angles=np.deg2rad(cami.orientation),
translation=cami.position,
)
self.camera_zero_poses.append(camera_zero_pose)
# Add camera node into scene
camera_node = pyrender.Node(camera=camera, matrix=camera_zero_pose)
self.scene.add_node(camera_node)
self.camera_nodes.append(camera_node)
# Add extra camera node into scene_depth
self.camera_node_depth = pyrender.Node(camera=camera,
matrix=camera_zero_pose)
self.scene_depth.add_node(self.camera_node_depth)
# Add corresponding light for rendering the camera
self.cam_light_ids.append(list(cami.lightIDList))
