def get_image_depth_and_mask(scene: pyrender.Scene,
scene_setup_loader: DiceScene.SceneSetupLoader,
width: int, height: int,
keep_nodes_in_scene: bool):
"""Renders an image ggiven a scene and seetup, along with the depth and segmentation mask labelling each die."""
r = pyrender.OffscreenRenderer(width, height)
color_bg, depth_bg = r.render(scene)
depth_nodes = []
for node in scene_setup_loader.dice_nodes:
scene.add_node(node)
color_node, depth_node = r.render(scene)
depth_nodes.append(depth_node)
scene.remove_node(node)
scene_setup_loader.add_loaded_to_scene(scene)
color_final, depth_final = r.render(scene)
if not keep_nodes_in_scene:
scene_setup_loader.remove_nodes_from_scene(scene)
#Initialize labels of pixels to -1 (for background)
labels_mask = np.ones((height, width), dtype=np.int8) * -1
for index, depth_for_node in enumerate(depth_nodes):
depth_not_background = np.not_equal(depth_bg, depth_for_node)
depth_at_foreground = np.equal(depth_final, depth_for_node)
depth_at_dice = np.logical_and(depth_not_background,
depth_at_foreground)
labels_mask[depth_at_dice] = index
return color_final, depth_final, labels_mask
def build_scene(num_cubes, color_candidates):
# Generate positions of each cube
cube_position_array, barycenter = generate_block_positions(num_cubes)
assert len(cube_position_array) == num_cubes
# Place cubes
scene = Scene(bg_color=np.array([0.0, 0.0, 0.0]),
ambient_light=np.array([0.3, 0.3, 0.3, 1.0]))
cube_nodes = []
for position in cube_position_array:
mesh = trimesh.creation.box(extents=cube_size * np.ones(3))
mesh = Mesh.from_trimesh(mesh, smooth=False)
node = Node(mesh=mesh,
translation=np.array(([
position[0] - barycenter[0],
position[1] - barycenter[1],
position[2] - barycenter[2],
])))
scene.add_node(node)
cube_nodes.append(node)
update_cube_color_and_position(cube_nodes, color_candidates)
# Place a light
light = DirectionalLight(color=np.ones(3), intensity=15.0)
quaternion_yaw = pyrender.quaternion.from_yaw(math.pi / 4)
quaternion_pitch = pyrender.quaternion.from_pitch(-math.pi / 5)
quaternion = pyrender.quaternion.multiply(quaternion_pitch, quaternion_yaw)
quaternion = quaternion / np.linalg.norm(quaternion)
node = Node(light=light,
rotation=quaternion,
translation=np.array([1, 1, 1]))
scene.add_node(node)
return scene, cube_nodes
def build_scene(max_num_cubes, color_candidates, probabilities):
num_cubes = np.random.choice(np.arange(1, max_num_cubes + 1),
size=1,
p=probabilities)[0]
# Generate positions of each cube
cube_position_array, barycenter = generate_block_positions(num_cubes)
assert len(cube_position_array) == num_cubes
# Place cubes
scene = Scene(bg_color=np.array([0.0, 0.0, 0.0]),
ambient_light=np.array([0.3, 0.3, 0.3, 1.0]))
cube_nodes = []
for position in cube_position_array:
mesh = trimesh.creation.box(extents=cube_size * np.ones(3))
mesh = Mesh.from_trimesh(mesh, smooth=False)
node = Node(mesh=mesh,
translation=np.array(([
position[0] - barycenter[0],
position[1] - barycenter[1],
position[2] - barycenter[2],
])))
scene.add_node(node)
cube_nodes.append(node)
# Generate positions of each cube
cube_position_array, barycenter = generate_block_positions(num_cubes)
for position, node in zip(cube_position_array, cube_nodes):
color = np.array(random.choice(color_candidates))
vertex_colors = np.broadcast_to(
color, node.mesh.primitives[0].positions.shape)
node.mesh.primitives[0].color_0 = vertex_colors
node.translation = np.array(([
position[0] - barycenter[0],
position[1] - barycenter[1],
position[2] - barycenter[2],
]))
# Place a light
light = DirectionalLight(color=np.ones(3), intensity=15.0)
quaternion_yaw = pyrender.quaternion.from_yaw(math.pi / 4)
quaternion_pitch = pyrender.quaternion.from_pitch(-math.pi / 5)
quaternion = pyrender.quaternion.multiply(quaternion_pitch, quaternion_yaw)
quaternion = quaternion / np.linalg.norm(quaternion)
node = Node(light=light,
rotation=quaternion,
translation=np.array([1, 1, 1]))
scene.add_node(node)
return scene, cube_nodes
def _reset_scene(self, scale_factor=1.0):
"""Resets the scene.
Parameters
----------
scale_factor : float
optional scale factor to apply to the image dimensions
"""
# delete scene
if self._scene is not None:
self._scene.clear()
del self._scene
# create scene
scene = Scene()
# setup camera
camera = IntrinsicsCamera(
self.camera.intrinsics.fx,
self.camera.intrinsics.fy,
self.camera.intrinsics.cx,
self.camera.intrinsics.cy,
)
pose_m = self.camera.pose.matrix.copy()
pose_m[:, 1:3] *= -1.0
scene.add(camera, pose=pose_m, name=self.camera.frame)
scene.main_camera_node = next(
iter(scene.get_nodes(name=self.camera.frame))
)
material = MetallicRoughnessMaterial(
baseColorFactor=np.array([1, 1, 1, 1.0]),
metallicFactor=0.2,
roughnessFactor=0.8,
)
# add workspace objects
for obj_key in self.state.workspace_keys:
obj_state = self.state[obj_key]
obj_mesh = Mesh.from_trimesh(obj_state.mesh, material=material)
T_obj_world = obj_state.pose.matrix
scene.add(obj_mesh, pose=T_obj_world, name=obj_key)
# add scene objects
for obj_key in self.state.obj_keys:
obj_state = self.state[obj_key]
obj_mesh = Mesh.from_trimesh(obj_state.mesh, material=material)
T_obj_world = obj_state.pose.matrix
scene.add(obj_mesh, pose=T_obj_world, name=obj_key)
# add light (for color rendering)
light = DirectionalLight(color=np.ones(3), intensity=1.0)
scene.add(light, pose=np.eye(4))
ray_light_nodes = self._create_raymond_lights()
[scene.add_node(rln) for rln in ray_light_nodes]
self._scene = scene
# Scene creation
# ==============================================================================
scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02, 1.0]))
# ==============================================================================
# Adding objects to the scene
# ==============================================================================
# ------------------------------------------------------------------------------
# By manually creating nodes
# ------------------------------------------------------------------------------
fuze_node = Node(mesh=fuze_mesh,
translation=np.array(
[0.1, 0.15, -np.min(fuze_trimesh.vertices[:, 2])]))
scene.add_node(fuze_node)
boxv_node = Node(mesh=boxv_mesh, translation=np.array([-0.1, 0.10, 0.05]))
scene.add_node(boxv_node)
boxf_node = Node(mesh=boxf_mesh, translation=np.array([-0.1, -0.10, 0.05]))
scene.add_node(boxf_node)
# ------------------------------------------------------------------------------
# By using the add() utility function
# ------------------------------------------------------------------------------
drill_node = scene.add(drill_mesh, pose=drill_pose)
bottle_node = scene.add(bottle_mesh, pose=bottle_pose)
wood_node = scene.add(wood_mesh)
direc_l_node = scene.add(direc_l, pose=cam_pose)
spot_l_node = scene.add(spot_l, pose=cam_pose)
# ==============================================================================
def build_scene(floor_textures, wall_textures, fix_light_position=False):
scene = Scene(bg_color=np.array([153 / 255, 226 / 255, 249 / 255]),
ambient_light=np.array([0.5, 0.5, 0.5, 1.0]))
floor_trimesh = trimesh.load("{}/floor.obj".format(object_directory))
mesh = Mesh.from_trimesh(floor_trimesh, smooth=False)
node = Node(mesh=mesh,
rotation=pyrender.quaternion.from_pitch(-math.pi / 2),
translation=np.array([0, 0, 0]))
texture_path = random.choice(floor_textures)
set_random_texture(node, texture_path)
scene.add_node(node)
texture_path = random.choice(wall_textures)
wall_trimesh = trimesh.load("{}/wall.obj".format(object_directory))
mesh = Mesh.from_trimesh(wall_trimesh, smooth=False)
node = Node(mesh=mesh, translation=np.array([0, 1.15, -3.5]))
set_random_texture(node, texture_path)
scene.add_node(node)
mesh = Mesh.from_trimesh(wall_trimesh, smooth=False)
node = Node(mesh=mesh,
rotation=pyrender.quaternion.from_yaw(math.pi),
translation=np.array([0, 1.15, 3.5]))
set_random_texture(node, texture_path)
scene.add_node(node)
mesh = Mesh.from_trimesh(wall_trimesh, smooth=False)
node = Node(mesh=mesh,
rotation=pyrender.quaternion.from_yaw(-math.pi / 2),
translation=np.array([3.5, 1.15, 0]))
set_random_texture(node, texture_path)
scene.add_node(node)
mesh = Mesh.from_trimesh(wall_trimesh, smooth=False)
node = Node(mesh=mesh,
rotation=pyrender.quaternion.from_yaw(math.pi / 2),
translation=np.array([-3.5, 1.15, 0]))
set_random_texture(node, texture_path)
scene.add_node(node)
light = DirectionalLight(color=np.ones(3), intensity=10)
if fix_light_position == True:
translation = np.array([1, 1, 1])
else:
xz = np.random.uniform(-1, 1, size=2)
translation = np.array([xz[0], 1, xz[1]])
yaw, pitch = compute_yaw_and_pitch(translation)
node = Node(light=light,
rotation=genearte_camera_quaternion(yaw, pitch),
translation=translation)
scene.add_node(node)
return scene
def add_loaded_to_scene(self, scene: pyrender.Scene):
"""Add to scene all dice nodes in loaded internal list."""
for node in self.dice_nodes:
scene.add_node(node)
def test_scenes():
# Basics
s = Scene()
assert np.allclose(s.bg_color, np.ones(4))
assert np.allclose(s.ambient_light, np.zeros(3))
assert len(s.nodes) == 0
assert s.name is None
s.name = 'asdf'
s.bg_color = None
s.ambient_light = None
assert np.allclose(s.bg_color, np.ones(4))
assert np.allclose(s.ambient_light, np.zeros(3))
assert s.nodes == set()
assert s.cameras == set()
assert s.lights == set()
assert s.point_lights == set()
assert s.spot_lights == set()
assert s.directional_lights == set()
assert s.meshes == set()
assert s.camera_nodes == set()
assert s.light_nodes == set()
assert s.point_light_nodes == set()
assert s.spot_light_nodes == set()
assert s.directional_light_nodes == set()
assert s.mesh_nodes == set()
assert s.main_camera_node is None
assert np.all(s.bounds == 0)
assert np.all(s.centroid == 0)
assert np.all(s.extents == 0)
assert np.all(s.scale == 0)
# From trimesh scene
tms = trimesh.load('tests/data/WaterBottle.glb')
s = Scene.from_trimesh_scene(tms)
assert len(s.meshes) == 1
assert len(s.mesh_nodes) == 1
# Test bg color formatting
s = Scene(bg_color=[0, 1.0, 0])
assert np.allclose(s.bg_color, np.array([0.0, 1.0, 0.0, 1.0]))
# Test constructor for nodes
n1 = Node()
n2 = Node()
n3 = Node()
nodes = [n1, n2, n3]
s = Scene(nodes=nodes)
n1.children.append(n2)
s = Scene(nodes=nodes)
n3.children.append(n2)
with pytest.raises(ValueError):
s = Scene(nodes=nodes)
n3.children = []
n2.children.append(n3)
n3.children.append(n2)
with pytest.raises(ValueError):
s = Scene(nodes=nodes)
# Test node accessors
n1 = Node()
n2 = Node()
n3 = Node()
nodes = [n1, n2]
s = Scene(nodes=nodes)
assert s.has_node(n1)
assert s.has_node(n2)
assert not s.has_node(n3)
# Test node poses
for n in nodes:
assert np.allclose(s.get_pose(n), np.eye(4))
with pytest.raises(ValueError):
s.get_pose(n3)
with pytest.raises(ValueError):
s.set_pose(n3, np.eye(4))
tf = np.eye(4)
tf[:3, 3] = np.ones(3)
s.set_pose(n1, tf)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), np.eye(4))
nodes = [n1, n2, n3]
tf2 = np.eye(4)
tf2[:3, :3] = np.diag([-1, -1, 1])
n1.children.append(n2)
n1.matrix = tf
n2.matrix = tf2
s = Scene(nodes=nodes)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), tf.dot(tf2))
assert np.allclose(s.get_pose(n3), np.eye(4))
n1 = Node()
n2 = Node()
n3 = Node()
n1.children.append(n2)
s = Scene()
s.add_node(n1)
with pytest.raises(ValueError):
s.add_node(n2)
s.set_pose(n1, tf)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), tf)
s.set_pose(n2, tf2)
assert np.allclose(s.get_pose(n2), tf.dot(tf2))
# Test node removal
n1 = Node()
n2 = Node()
n3 = Node()
n1.children.append(n2)
n2.children.append(n3)
s = Scene(nodes=[n1, n2, n3])
s.remove_node(n2)
assert len(s.nodes) == 1
assert n1 in s.nodes
assert len(n1.children) == 0
assert len(n2.children) == 1
s.add_node(n2, parent_node=n1)
assert len(n1.children) == 1
n1.matrix = tf
n3.matrix = tf2
assert np.allclose(s.get_pose(n3), tf.dot(tf2))
# Now test ADD function
s = Scene()
m = Mesh([], name='m')
cp = PerspectiveCamera(yfov=2.0)
co = OrthographicCamera(xmag=1.0, ymag=1.0)
dl = DirectionalLight()
pl = PointLight()
sl = SpotLight()
n1 = s.add(m, name='mn')
assert n1.mesh == m
assert len(s.nodes) == 1
assert len(s.mesh_nodes) == 1
assert n1 in s.mesh_nodes
assert len(s.meshes) == 1
assert m in s.meshes
assert len(s.get_nodes(node=n2)) == 0
n2 = s.add(m, pose=tf)
assert len(s.nodes) == len(s.mesh_nodes) == 2
assert len(s.meshes) == 1
assert len(s.get_nodes(node=n1)) == 1
assert len(s.get_nodes(node=n1, name='mn')) == 1
assert len(s.get_nodes(name='mn')) == 1
assert len(s.get_nodes(obj=m)) == 2
assert len(s.get_nodes(obj=m, obj_name='m')) == 2
assert len(s.get_nodes(obj=co)) == 0
nsl = s.add(sl, name='sln')
npl = s.add(pl, parent_name='sln')
assert nsl.children[0] == npl
ndl = s.add(dl, parent_node=npl)
assert npl.children[0] == ndl
nco = s.add(co)
ncp = s.add(cp)
assert len(s.light_nodes) == len(s.lights) == 3
assert len(s.point_light_nodes) == len(s.point_lights) == 1
assert npl in s.point_light_nodes
assert len(s.spot_light_nodes) == len(s.spot_lights) == 1
assert nsl in s.spot_light_nodes
assert len(s.directional_light_nodes) == len(s.directional_lights) == 1
assert ndl in s.directional_light_nodes
assert len(s.cameras) == len(s.camera_nodes) == 2
assert s.main_camera_node == nco
s.main_camera_node = ncp
s.remove_node(ncp)
assert len(s.cameras) == len(s.camera_nodes) == 1
assert s.main_camera_node == nco
s.remove_node(n2)
assert len(s.meshes) == 1
s.remove_node(n1)
assert len(s.meshes) == 0
s.remove_node(nsl)
assert len(s.lights) == 0
s.remove_node(nco)
assert s.main_camera_node is None
s.add_node(n1)
s.clear()
assert len(s.nodes) == 0
# Trigger final errors
with pytest.raises(ValueError):
s.main_camera_node = None
with pytest.raises(ValueError):
s.main_camera_node = ncp
with pytest.raises(ValueError):
s.add(m, parent_node=n1)
with pytest.raises(ValueError):
s.add(m, name='asdf')
s.add(m, name='asdf')
s.add(m, parent_name='asdf')
with pytest.raises(ValueError):
s.add(m, parent_name='asfd')
with pytest.raises(TypeError):
s.add(None)
s.clear()
# Test bounds
m1 = Mesh.from_trimesh(trimesh.creation.box())
m2 = Mesh.from_trimesh(trimesh.creation.box())
m3 = Mesh.from_trimesh(trimesh.creation.box())
n1 = Node(mesh=m1)
n2 = Node(mesh=m2, translation=[1.0, 0.0, 0.0])
n3 = Node(mesh=m3, translation=[0.5, 0.0, 1.0])
s.add_node(n1)
s.add_node(n2)
s.add_node(n3)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [1.5, 0.5, 1.5]])
s.clear()
s.add_node(n1)
s.add_node(n2, parent_node=n1)
s.add_node(n3, parent_node=n2)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [2.0, 0.5, 1.5]])
tf = np.eye(4)
tf[:3, 3] = np.ones(3)
s.set_pose(n3, tf)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [2.5, 1.5, 1.5]])
s.remove_node(n2)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]])
s.clear()
assert np.allclose(s.bounds, 0.0)
class PybulletPhysicsEngine(PhysicsEngine):
"""Wrapper for pybullet physics engine that is tied to a single ID"""
def __init__(self, urdf_cache_dir, debug=False):
PhysicsEngine.__init__(self)
self._physics_client = None
self._debug = debug
self._urdf_cache_dir = urdf_cache_dir
if not os.path.isabs(self._urdf_cache_dir):
self._urdf_cache_dir = os.path.join(
os.getcwd(), self._urdf_cache_dir
)
if not os.path.exists(os.path.join(self._urdf_cache_dir, "plane")):
os.makedirs(os.path.join(self._urdf_cache_dir, "plane"))
shutil.copy(
pkg_resources.resource_filename(
"sd_maskrcnn", "data/plane/plane.urdf"
),
os.path.join(self._urdf_cache_dir, "plane", "plane.urdf"),
)
shutil.copy(
pkg_resources.resource_filename(
"sd_maskrcnn", "data/plane/plane_convex_piece_0.obj"
),
os.path.join(
self._urdf_cache_dir, "plane", "plane_convex_piece_0.obj"
),
)
def add(self, obj, static=False):
# create URDF
urdf_filename = os.path.join(
self._urdf_cache_dir,
KEY_SEP_TOKEN.join(obj.key.split(KEY_SEP_TOKEN)[:-1]),
"{}.urdf".format(
KEY_SEP_TOKEN.join(obj.key.split(KEY_SEP_TOKEN)[:-1])
),
)
urdf_dir = os.path.dirname(urdf_filename)
if not os.path.exists(urdf_filename):
try:
os.makedirs(urdf_dir)
except:
self._logger.warning(
"Failed to create dir %s. The object may have been created simultaneously by another process"
% (urdf_dir)
)
self._logger.info(
"Exporting URDF for object {}".format(
KEY_SEP_TOKEN.join(obj.key.split(KEY_SEP_TOKEN)[:-1])
)
)
# Fix center of mass (for rendering) and density and export
geometry = obj.mesh.copy()
geometry.apply_translation(-obj.mesh.center_mass)
trimesh.exchange.export.export_urdf(geometry, urdf_dir)
com = obj.mesh.center_mass
pose = self._convert_pose(obj.pose, com)
obj_t = pose.translation
obj_q_wxyz = pose.quaternion
obj_q_xyzw = np.roll(obj_q_wxyz, -1)
try:
obj_id = pybullet.loadURDF(
urdf_filename,
obj_t,
obj_q_xyzw,
useFixedBase=static,
physicsClientId=self._physics_client,
)
except:
raise Exception("Failed to load %s" % (urdf_filename))
if self._debug:
self._add_to_scene(obj)
self._key_to_id[obj.key] = obj_id
self._key_to_com[obj.key] = com
def get_velocity(self, key):
obj_id = self._key_to_id[key]
return pybullet.getBaseVelocity(
obj_id, physicsClientId=self._physics_client
)
def get_pose(self, key):
obj_id = self._key_to_id[key]
obj_t, obj_q_xyzw = pybullet.getBasePositionAndOrientation(
obj_id, physicsClientId=self._physics_client
)
obj_q_wxyz = np.roll(obj_q_xyzw, 1)
pose = RigidTransform(
rotation=obj_q_wxyz,
translation=obj_t,
from_frame="obj",
to_frame="world",
)
pose = self._deconvert_pose(pose, self._key_to_com[key])
return pose
def remove(self, key):
obj_id = self._key_to_id[key]
pybullet.removeBody(obj_id, physicsClientId=self._physics_client)
self._key_to_id.pop(key)
self._key_to_com.pop(key)
if self._debug:
self._remove_from_scene(key)
def step(self):
pybullet.stepSimulation(physicsClientId=self._physics_client)
if self._debug:
time.sleep(0.04)
self._update_scene()
def reset(self):
if self._physics_client is not None:
self.stop()
self.start()
def start(self):
if self._physics_client is None:
self._physics_client = pybullet.connect(pybullet.DIRECT)
pybullet.setGravity(
0, 0, -GRAVITY_ACCEL, physicsClientId=self._physics_client
)
self._key_to_id = {}
self._key_to_com = {}
if self._debug:
self._create_scene()
self._viewer = Viewer(
self._scene, use_raymond_lighting=True, run_in_thread=True
)
def stop(self):
if self._physics_client is not None:
pybullet.disconnect(self._physics_client)
self._physics_client = None
if self._debug:
self._scene = None
self._viewer.close_external()
while self._viewer.is_active:
pass
def __del__(self):
self.stop()
del self
def _convert_pose(self, pose, com):
new_pose = pose.copy()
new_pose.translation = pose.rotation.dot(com) + pose.translation
return new_pose
def _deconvert_pose(self, pose, com):
new_pose = pose.copy()
new_pose.translation = pose.rotation.dot(-com) + pose.translation
return new_pose
def _create_scene(self):
self._scene = Scene()
camera = PerspectiveCamera(
yfov=0.833, znear=0.05, zfar=3.0, aspectRatio=1.0
)
cn = Node()
cn.camera = camera
pose_m = np.array(
[
[0.0, 1.0, 0.0, 0.0],
[1.0, 0.0, 0.0, 0.0],
[0.0, 0.0, -1.0, 0.88],
[0.0, 0.0, 0.0, 1.0],
]
)
pose_m[:, 1:3] *= -1.0
cn.matrix = pose_m
self._scene.add_node(cn)
self._scene.main_camera_node = cn
def _add_to_scene(self, obj):
self._viewer.render_lock.acquire()
n = Node(
mesh=Mesh.from_trimesh(obj.mesh),
matrix=obj.pose.matrix,
name=obj.key,
)
self._scene.add_node(n)
self._viewer.render_lock.release()
def _remove_from_scene(self, key):
self._viewer.render_lock.acquire()
if self._scene.get_nodes(name=key):
self._scene.remove_node(
next(iter(self._scene.get_nodes(name=key)))
)
self._viewer.render_lock.release()
def _update_scene(self):
self._viewer.render_lock.acquire()
for key in self._key_to_id.keys():
obj_pose = self.get_pose(key).matrix
if self._scene.get_nodes(name=key):
next(iter(self._scene.get_nodes(name=key))).matrix = obj_pose
self._viewer.render_lock.release()
while True:
r += delta_r
frame += 1
t += 50
if t % 1000 == 0:
delta_r *= -1
s = parametric_surface.doughnut(R, r, [50, 20])
doughnut_trimesh = trimesh.Trimesh(
vertices=s.flat_vertices,
faces=s.flat_triangular_mesh_indices,
)
# for facet in doughnut_trimesh.facets:
# doughnut_trimesh.visual.face_colors[facet] = trimesh.visual.random_color()
mesh = pyrender.Mesh.from_trimesh(doughnut_trimesh, smooth=False)
mesh_node = Node(mesh=mesh, translation=np.array([0.0, 0.0, 0.0]))
scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02, 1.0]),
bg_color=[0.0, 0.0, 0.0])
cam_node = scene.add(cam, pose=cam_pose)
scene.add_node(mesh_node)
# v = Viewer(scene)
color, depth = off_screen_renderer.render(scene)
cv2.imshow('f', color)
cv2.waitKey(1)
end_time = time.time()
print(frame / (end_time - start_time))
off_screen_renderer.delete()
def test_offscreen_renderer(tmpdir):
# Fuze trimesh
fuze_trimesh = trimesh.load('examples/models/fuze.obj')
fuze_mesh = Mesh.from_trimesh(fuze_trimesh)
# Drill trimesh
drill_trimesh = trimesh.load('examples/models/drill.obj')
drill_mesh = Mesh.from_trimesh(drill_trimesh)
drill_pose = np.eye(4)
drill_pose[0, 3] = 0.1
drill_pose[2, 3] = -np.min(drill_trimesh.vertices[:, 2])
# Wood trimesh
wood_trimesh = trimesh.load('examples/models/wood.obj')
wood_mesh = Mesh.from_trimesh(wood_trimesh)
# Water bottle trimesh
bottle_gltf = trimesh.load('examples/models/WaterBottle.glb')
bottle_trimesh = bottle_gltf.geometry[list(bottle_gltf.geometry.keys())[0]]
bottle_mesh = Mesh.from_trimesh(bottle_trimesh)
bottle_pose = np.array([
[1.0, 0.0, 0.0, 0.1],
[0.0, 0.0, -1.0, -0.16],
[0.0, 1.0, 0.0, 0.13],
[0.0, 0.0, 0.0, 1.0],
])
boxv_trimesh = trimesh.creation.box(extents=0.1 * np.ones(3))
boxv_vertex_colors = np.random.uniform(size=(boxv_trimesh.vertices.shape))
boxv_trimesh.visual.vertex_colors = boxv_vertex_colors
boxv_mesh = Mesh.from_trimesh(boxv_trimesh, smooth=False)
boxf_trimesh = trimesh.creation.box(extents=0.1 * np.ones(3))
boxf_face_colors = np.random.uniform(size=boxf_trimesh.faces.shape)
boxf_trimesh.visual.face_colors = boxf_face_colors
# Instanced
poses = np.tile(np.eye(4), (2, 1, 1))
poses[0, :3, 3] = np.array([-0.1, -0.10, 0.05])
poses[1, :3, 3] = np.array([-0.15, -0.10, 0.05])
boxf_mesh = Mesh.from_trimesh(boxf_trimesh, poses=poses, smooth=False)
points = trimesh.creation.icosphere(radius=0.05).vertices
point_colors = np.random.uniform(size=points.shape)
points_mesh = Mesh.from_points(points, colors=point_colors)
direc_l = DirectionalLight(color=np.ones(3), intensity=1.0)
spot_l = SpotLight(color=np.ones(3),
intensity=10.0,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6)
cam = PerspectiveCamera(yfov=(np.pi / 3.0))
cam_pose = np.array([[0.0, -np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.5], [1.0, 0.0, 0.0, 0.0],
[0.0, np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.4], [0.0, 0.0, 0.0, 1.0]])
scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02]))
fuze_node = Node(mesh=fuze_mesh,
translation=np.array(
[0.1, 0.15, -np.min(fuze_trimesh.vertices[:, 2])]))
scene.add_node(fuze_node)
boxv_node = Node(mesh=boxv_mesh, translation=np.array([-0.1, 0.10, 0.05]))
scene.add_node(boxv_node)
boxf_node = Node(mesh=boxf_mesh)
scene.add_node(boxf_node)
_ = scene.add(drill_mesh, pose=drill_pose)
_ = scene.add(bottle_mesh, pose=bottle_pose)
_ = scene.add(wood_mesh)
_ = scene.add(direc_l, pose=cam_pose)
_ = scene.add(spot_l, pose=cam_pose)
_ = scene.add(points_mesh)
_ = scene.add(cam, pose=cam_pose)
r = OffscreenRenderer(viewport_width=640, viewport_height=480)
color, depth = r.render(scene)
assert color.shape == (480, 640, 3)
assert depth.shape == (480, 640)
assert np.max(depth.data) > 0.05
assert np.count_nonzero(depth.data) > (0.2 * depth.size)
r.delete()
def __init__(self, scene: pyrender.Scene, parent_node=None, **kwargs):
super().__init__(**kwargs)
self._scene = scene
self._enabled = True
scene.add_node(self, parent_node=parent_node)
