def shape_to_mesh(self, shape: Shape):
""" Convert pyphysx shape into the pyrender Mesh """
clr_string = shape.get_user_data().get(
'color', None) if shape.get_user_data() is not None else None
clr = gl_color_from_matplotlib(color=clr_string, return_rgba=True)
basic_trimesh = self._trimesh_from_basic_shape(shape, clr)
if basic_trimesh is not None:
return Mesh.from_trimesh(basic_trimesh)
elif shape.get_geometry_type() == GeometryType.CONVEXMESH:
data = shape.get_shape_data() # N x 9 - i.e. 3 triangles
primitive = Primitive(data.reshape(-1, 3),
normals=None,
color_0=clr,
mode=GLTF.TRIANGLES,
poses=None)
return Mesh(primitives=[primitive])
elif shape.get_geometry_type() == GeometryType.PLANE:
clr = [0.8] * 3 + [
0.1
] if clr_string is None else gl_color_from_matplotlib(
clr_string, return_rgba=True)
primitive = Primitive(positions=self._grid_lines(),
normals=None,
color_0=clr,
mode=GLTF.LINES,
poses=None)
return Mesh(primitives=[primitive])
else:
raise NotImplementedError('Not supported type of the geometry.')
def _reset_scene(self, scale_factor=1.0):
"""Resets the scene.
Parameters
----------
scale_factor : float
optional scale factor to apply to the image dimensions
"""
# delete scene
if self._scene is not None:
self._scene.clear()
del self._scene
# create scene
scene = Scene()
# setup camera
camera = IntrinsicsCamera(
self.camera.intrinsics.fx,
self.camera.intrinsics.fy,
self.camera.intrinsics.cx,
self.camera.intrinsics.cy,
)
pose_m = self.camera.pose.matrix.copy()
pose_m[:, 1:3] *= -1.0
scene.add(camera, pose=pose_m, name=self.camera.frame)
scene.main_camera_node = next(
iter(scene.get_nodes(name=self.camera.frame))
)
material = MetallicRoughnessMaterial(
baseColorFactor=np.array([1, 1, 1, 1.0]),
metallicFactor=0.2,
roughnessFactor=0.8,
)
# add workspace objects
for obj_key in self.state.workspace_keys:
obj_state = self.state[obj_key]
obj_mesh = Mesh.from_trimesh(obj_state.mesh, material=material)
T_obj_world = obj_state.pose.matrix
scene.add(obj_mesh, pose=T_obj_world, name=obj_key)
# add scene objects
for obj_key in self.state.obj_keys:
obj_state = self.state[obj_key]
obj_mesh = Mesh.from_trimesh(obj_state.mesh, material=material)
T_obj_world = obj_state.pose.matrix
scene.add(obj_mesh, pose=T_obj_world, name=obj_key)
# add light (for color rendering)
light = DirectionalLight(color=np.ones(3), intensity=1.0)
scene.add(light, pose=np.eye(4))
ray_light_nodes = self._create_raymond_lights()
[scene.add_node(rln) for rln in ray_light_nodes]
self._scene = scene
def build_scene(floor_textures, wall_textures, fix_light_position=False):
scene = Scene(bg_color=np.array([153 / 255, 226 / 255, 249 / 255]),
ambient_light=np.array([0.5, 0.5, 0.5, 1.0]))
floor_trimesh = trimesh.load("{}/floor.obj".format(object_directory))
mesh = Mesh.from_trimesh(floor_trimesh, smooth=False)
node = Node(mesh=mesh,
rotation=pyrender.quaternion.from_pitch(-math.pi / 2),
translation=np.array([0, 0, 0]))
texture_path = random.choice(floor_textures)
set_random_texture(node, texture_path)
scene.add_node(node)
texture_path = random.choice(wall_textures)
wall_trimesh = trimesh.load("{}/wall.obj".format(object_directory))
mesh = Mesh.from_trimesh(wall_trimesh, smooth=False)
node = Node(mesh=mesh, translation=np.array([0, 1.15, -3.5]))
set_random_texture(node, texture_path)
scene.add_node(node)
mesh = Mesh.from_trimesh(wall_trimesh, smooth=False)
node = Node(mesh=mesh,
rotation=pyrender.quaternion.from_yaw(math.pi),
translation=np.array([0, 1.15, 3.5]))
set_random_texture(node, texture_path)
scene.add_node(node)
mesh = Mesh.from_trimesh(wall_trimesh, smooth=False)
node = Node(mesh=mesh,
rotation=pyrender.quaternion.from_yaw(-math.pi / 2),
translation=np.array([3.5, 1.15, 0]))
set_random_texture(node, texture_path)
scene.add_node(node)
mesh = Mesh.from_trimesh(wall_trimesh, smooth=False)
node = Node(mesh=mesh,
rotation=pyrender.quaternion.from_yaw(math.pi / 2),
translation=np.array([-3.5, 1.15, 0]))
set_random_texture(node, texture_path)
scene.add_node(node)
light = DirectionalLight(color=np.ones(3), intensity=10)
if fix_light_position == True:
translation = np.array([1, 1, 1])
else:
xz = np.random.uniform(-1, 1, size=2)
translation = np.array([xz[0], 1, xz[1]])
yaw, pitch = compute_yaw_and_pitch(translation)
node = Node(light=light,
rotation=genearte_camera_quaternion(yaw, pitch),
translation=translation)
scene.add_node(node)
return scene
def traverse_static_dataset(root_dir):
for dir_name in os.listdir(root_dir):
parent_dir = os.path.join(root_dir, dir_name)
if not os.path.isdir(parent_dir): continue
write_log("Visualizing human model: %s" % dir_name)
meshes = {}
# Read origin mesh first
write_log("Loading origin mesh ...")
origin_mesh = read_mesh_from_dir(parent_dir, load_texture=True)
if origin_mesh is None:
write_log("Fatal: No origin mesh!")
continue
# Extract common material
write_log("Extracting common material ...")
common_material = extract_material(origin_mesh)
if common_material is None:
write_log("Fatal: Invalid material extracted")
continue
# Convert origin mesh to pyrender mesh
meshes['model'] = Mesh.from_trimesh(origin_mesh, material=common_material)
# Force to release memory
del origin_mesh
all_components = ['top', 'bottom', 'shoes', 'naked']
for component in all_components:
write_log("Loading component %s mesh ..." % component)
component_dir = os.path.join(parent_dir, component)
component_mesh = read_mesh_from_dir(component_dir, load_texture=False)
if component_mesh is None:
write_log("No component %s mesh" % component)
continue
# Convert to pyrender mesh
material = common_material if component != 'naked' else None
mesh = Mesh.from_trimesh(component_mesh, material=material)
meshes[component] = mesh
# Force to release memory
del component_mesh
if len(meshes) <= 1:
write_log("Warning: Only origin mesh available")
# Return vis image path
vis_path = {}
for key in g_vis_list.keys():
vis_path[key] = os.path.join(parent_dir, key+'_vis.png')
yield meshes, vis_path
def test_duck():
bm = trimesh.load('tests/data/Duck.glb').dump()[0]
x = Mesh.from_trimesh(bm)
assert x.primitives[0].material.baseColorTexture is not None
pixel = x.primitives[0].material.baseColorTexture.source[100, 100]
yellowish = np.array([1.0, 0.7411765, 0.0, 1.0])
assert np.allclose(pixel, yellowish)
def _add_to_scene(self, obj):
self._viewer.render_lock.acquire()
n = Node(mesh=Mesh.from_trimesh(obj.mesh),
matrix=obj.pose.matrix,
name=obj.key)
self._scene.add_node(n)
self._viewer.render_lock.release()
def create_2d_point_mesh(width, height, pts_2d, point_size_px):
# create a triangle mesh
def _rot_mat(angle_rad):
return np.array(
[[np.cos(angle_rad), -np.sin(angle_rad)],
[np.sin(angle_rad), np.cos(angle_rad)]],
dtype=np.float32)
if isinstance(pts_2d, list):
pts_2d = np.array(pts_2d, dtype=np.float32)
vertices = np.zeros((3, 3), dtype=np.float32)
point_size_ndc = point_size_px / height
vec = np.array([0, point_size_ndc])
vertices[0, 0:2] = vec
vertices[1, 0:2] = _rot_mat(np.radians(120)) @ vec
vertices[2, 0:2] = _rot_mat(np.radians(-120)) @ vec
mat = MetallicRoughnessMaterial(doubleSided=True,
baseColorFactor=[1.0, 0, 0, 1])
instances = [np.eye(4) for i in range(pts_2d.shape[0])]
for i in range(pts_2d.shape[0]):
instances[i][0, 3] = (2 * pts_2d[i][0] - width) / height
instances[i][1, 3] = pts_2d[i][1] / height * 2 - 1
instances[i][2, 3] = -1
prim = Primitive(positions=vertices,
mode=GLTF.TRIANGLES,
material=mat,
poses=instances)
return Mesh(primitives=[prim])
def place_dice(scene,
mnist_images,
discrete_position=False,
rotate_dice=False):
dice_trimesh = trimesh.load("objects/dice.obj")
mesh = Mesh.from_trimesh(dice_trimesh, smooth=False)
node = Node(mesh=mesh,
scale=np.array([0.75, 0.75, 0.75]),
translation=np.array([0, 0.75, 0]))
texture_image = generate_mnist_texture(mnist_images)
primitive = node.mesh.primitives[0]
primitive.material.baseColorTexture.source = texture_image
primitive.material.baseColorTexture.sampler.minFilter = GL_LINEAR_MIPMAP_LINEAR
directions = [-1.0, 0.0, 1.0]
available_positions = []
for z in directions:
for x in directions:
available_positions.append((x, z))
xz = np.array(random.choice(available_positions))
if discrete_position == False:
xz += np.random.uniform(-0.25, 0.25, size=xz.shape)
if rotate_dice:
yaw = np.random.uniform(0, math.pi * 2, size=1)[0]
rotation = pyrender.quaternion.from_yaw(yaw)
parent = Node(children=[node],
rotation=rotation,
translation=np.array([xz[0], 0, xz[1]]))
else:
parent = Node(children=[node], translation=np.array([xz[0], 0, xz[1]]))
scene.add_node(parent)
def build_scene(num_cubes, color_candidates):
# Generate positions of each cube
cube_position_array, barycenter = generate_block_positions(num_cubes)
assert len(cube_position_array) == num_cubes
# Place cubes
scene = Scene(bg_color=np.array([0.0, 0.0, 0.0]),
ambient_light=np.array([0.3, 0.3, 0.3, 1.0]))
cube_nodes = []
for position in cube_position_array:
mesh = trimesh.creation.box(extents=cube_size * np.ones(3))
mesh = Mesh.from_trimesh(mesh, smooth=False)
node = Node(mesh=mesh,
translation=np.array(([
position[0] - barycenter[0],
position[1] - barycenter[1],
position[2] - barycenter[2],
])))
scene.add_node(node)
cube_nodes.append(node)
update_cube_color_and_position(cube_nodes, color_candidates)
# Place a light
light = DirectionalLight(color=np.ones(3), intensity=15.0)
quaternion_yaw = pyrender.quaternion.from_yaw(math.pi / 4)
quaternion_pitch = pyrender.quaternion.from_pitch(-math.pi / 5)
quaternion = pyrender.quaternion.multiply(quaternion_pitch, quaternion_yaw)
quaternion = quaternion / np.linalg.norm(quaternion)
node = Node(light=light,
rotation=quaternion,
translation=np.array([1, 1, 1]))
scene.add_node(node)
return scene, cube_nodes
def make_normals_node(trimesh, node, normal_direction):
normals_mesh = Mesh.from_trimesh(trimesh, smooth=False)
normals_mesh.primitives[0].material = SpecularGlossinessMaterial(
diffuseFactor=list(np.asarray(normal_direction) / 2 + 0.5) + [1.])
return Node(mesh=normals_mesh,
rotation=node.rotation,
translation=node.translation)
def _build_scene(self, path, size, light, y_fov):
self.scene = Scene(bg_color=[0, 0, 0, 0])
self.light = self.scene.add(DirectionalLight([1.0, 1.0, 1.0], light))
self.camera = self.scene.add(
PerspectiveCamera(y_fov, aspectRatio=np.divide(*size)))
self.mesh = self.scene.add(
Mesh.from_trimesh(trimesh.load(path), smooth=True))
self.world_origin = self.mesh.mesh.centroid
def color_object(path):
mesh = load_obj(path)
colors = compute_vertices_colors(mesh.vertices)
mesh.visual = mesh.visual.to_color()
mesh.visual.vertex_colors = colors
mesh = Mesh.from_trimesh(mesh, smooth=False)
mesh.primitives[0].material.metallicFactor = 0.0
mesh.primitives[0].material.roughnessFactor = 1.0
mesh.primitives[0].material.alphaMode = 'OPAQUE'
return mesh
def add_point_cloud_mesh_to_scene(point_cloud, scene, pose, pts_colors):
'''
pts_colors: np array, Nx3
point_cloud: NX3, np array
'''
points = point_cloud
mn = Mesh.from_points(points, colors=pts_colors)
pc_node = scene.add(mn, pose=pose)
return pc_node
def _reset_scene(self, scale_factor=1.0):
""" Resets the scene.
Parameters
----------
scale_factor : float
optional scale factor to apply to the image dimensions
"""
# delete scene
if self._scene is not None:
self._scene.clear()
del self._scene
# create scene
scene = Scene()
# setup camera
camera = IntrinsicsCamera(self.camera.intrinsics.fx, self.camera.intrinsics.fy,
self.camera.intrinsics.cx, self.camera.intrinsics.cy)
pose_m = self.camera.pose.matrix.copy()
pose_m[:,1:3] *= -1.0
scene.add(camera, pose=pose_m, name=self.camera.frame)
scene.main_camera_node = next(iter(scene.get_nodes(name=self.camera.frame)))
# add workspace objects
for obj_key in self.state.workspace_keys:
obj_state = self.state[obj_key]
obj_mesh = Mesh.from_trimesh(obj_state.mesh)
T_obj_world = obj_state.pose.matrix
scene.add(obj_mesh, pose=T_obj_world, name=obj_key)
# add scene objects
for obj_key in self.state.obj_keys:
obj_state = self.state[obj_key]
obj_mesh = Mesh.from_trimesh(obj_state.mesh)
T_obj_world = obj_state.pose.matrix
scene.add(obj_mesh, pose=T_obj_world, name=obj_key)
self._scene = scene
def build_scene(max_num_cubes, color_candidates, probabilities):
num_cubes = np.random.choice(np.arange(1, max_num_cubes + 1),
size=1,
p=probabilities)[0]
# Generate positions of each cube
cube_position_array, barycenter = generate_block_positions(num_cubes)
assert len(cube_position_array) == num_cubes
# Place cubes
scene = Scene(bg_color=np.array([0.0, 0.0, 0.0]),
ambient_light=np.array([0.3, 0.3, 0.3, 1.0]))
cube_nodes = []
for position in cube_position_array:
mesh = trimesh.creation.box(extents=cube_size * np.ones(3))
mesh = Mesh.from_trimesh(mesh, smooth=False)
node = Node(mesh=mesh,
translation=np.array(([
position[0] - barycenter[0],
position[1] - barycenter[1],
position[2] - barycenter[2],
])))
scene.add_node(node)
cube_nodes.append(node)
# Generate positions of each cube
cube_position_array, barycenter = generate_block_positions(num_cubes)
for position, node in zip(cube_position_array, cube_nodes):
color = np.array(random.choice(color_candidates))
vertex_colors = np.broadcast_to(
color, node.mesh.primitives[0].positions.shape)
node.mesh.primitives[0].color_0 = vertex_colors
node.translation = np.array(([
position[0] - barycenter[0],
position[1] - barycenter[1],
position[2] - barycenter[2],
]))
# Place a light
light = DirectionalLight(color=np.ones(3), intensity=15.0)
quaternion_yaw = pyrender.quaternion.from_yaw(math.pi / 4)
quaternion_pitch = pyrender.quaternion.from_pitch(-math.pi / 5)
quaternion = pyrender.quaternion.multiply(quaternion_pitch, quaternion_yaw)
quaternion = quaternion / np.linalg.norm(quaternion)
node = Node(light=light,
rotation=quaternion,
translation=np.array([1, 1, 1]))
scene.add_node(node)
return scene, cube_nodes
def _create_node_from_points(points,
name=None,
pose=None,
color=None,
material=None,
tube_radius=None,
n_divs=20):
"""Helper method that creates a pyrender.Node from an array of points"""
points = np.asanyarray(points)
if points.ndim == 1:
points = np.array([points])
if pose is None:
pose = np.eye(4)
else:
pose = pose.matrix
# Create vertex colors if needed
if color is not None:
color = np.asanyarray(color, dtype=np.float)
if color.ndim == 1 or len(color) != len(points):
color = np.repeat(color[np.newaxis, :], len(points), axis=0)
if tube_radius is not None:
poses = None
mesh = trimesh.creation.uv_sphere(tube_radius, [n_divs, n_divs])
if color is not None:
mesh.visual.vertex_colors = color[0]
poses = np.tile(np.eye(4),
(len(points), 1)).reshape(len(points), 4, 4)
poses[:, :3, 3::4] = points[:, :, None]
m = Mesh.from_trimesh(mesh, material=material, poses=poses)
else:
m = Mesh.from_points(points, colors=color)
return Node(mesh=m, name=name, matrix=pose)
def _create_node_from_points(points,
name=None,
pose=None,
color=None,
material=None,
radius=None,
n_divs=20):
"""Helper method that creates a pyrender.Node from an array of points"""
points = np.asanyarray(points)
if points.ndim == 1:
points = np.array([points])
if pose is None:
pose = np.eye(4)
else:
pose = pose.matrix
color = np.asanyarray(color,
dtype=np.float) if color is not None else None
# If color specified per point, use sprites
if color is not None and color.ndim > 1:
Visualizer3D._kwargs.update({"point_size": 1000 * radius})
m = Mesh.from_points(points, colors=color)
# otherwise, we can make pretty spheres
else:
mesh = trimesh.creation.uv_sphere(radius, [n_divs, n_divs])
if color is not None:
mesh.visual.vertex_colors = color
poses = None
poses = np.tile(np.eye(4),
(len(points), 1)).reshape(len(points), 4, 4)
poses[:, :3, 3::4] = points[:, :, None]
m = Mesh.from_trimesh(mesh, material=material, poses=poses)
return Node(mesh=m, name=name, matrix=pose)
def render_sensor(
point_set,
render_sensor_path="/Users/macbookpro15/Documents/mujoco_hand_exps/data/sensor_render"
):
"""
pointset: it is collectiono of sensor points for all timesteps
"""
# first take one of the point, subtract the center from it which
# I know is the 0-th position out of the 220 points
# form the mesh from this
if not os.path.exists(render_sensor_path):
os.makedirs(render_sensor_path)
time_steps = len(point_set)
for t in range(time_steps):
sensor = trimesh.load_mesh(
f'../data/mesh_dir/mesh_{t}_out/mc_mesh_out.ply')
sensor_mesh = Mesh.from_trimesh(sensor)
# Light for the scene
direc_l = DirectionalLight(color=np.ones(3), intensity=1.0)
spot_l = SpotLight(color=np.ones(3),
intensity=10.0,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6)
point_l = PointLight(color=np.ones(3), intensity=10.0)
# add camera to the scene
cam = PerspectiveCamera(yfov=(np.pi / 3.0))
cam_pose = np.array([[0.0, -np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.5], [1.0, 0.0, 0.0, 0.0],
[0.0, np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.4], [0.0, 0.0, 0.0, 1.0]])
# create the scene
scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02, 1.0]))
point_mesh_node = scene.add(sensor_mesh)
direc_l_node = scene.add(direc_l, pose=cam_pose)
spot_l_node = scene.add(spot_l, pose=cam_pose)
cam_node = scene.add(cam, pose=cam_pose)
print('rendering the scene offline')
r = OffscreenRenderer(viewport_width=640, viewport_height=480)
color, depth = r.render(scene)
r.delete()
plt.figure()
plt.imshow(color)
plt.savefig(f'{render_sensor_path}/img_{t}.jpg')
def __init__(self, path_OBJ, camera_pose, min_corner, max_corner,
symmetric_transforms, viewport_size=(128, 128),
y_fov=3.14159 / 4.0, light_intensity=[0.5, 30]):
self.light_intensity = light_intensity
self.symmetric_transforms = symmetric_transforms
self.min_corner, self.max_corner = min_corner, max_corner
self.scene = Scene(bg_color=[0, 0, 0, 0])
self.light = self._build_light(light_intensity, camera_pose)
self.camera = self._build_camera(y_fov, viewport_size, camera_pose)
self.pixel_mesh = self.scene.add(color_object(path_OBJ))
self.mesh = self.scene.add(
Mesh.from_trimesh(trimesh.load(path_OBJ), smooth=True))
self.renderer = OffscreenRenderer(viewport_size[0], viewport_size[1])
self.flags_RGBA = RenderFlags.RGBA
self.flags_FLAT = RenderFlags.RGBA | RenderFlags.FLAT
def _create_node_from_mesh(mesh,
name=None,
pose=None,
color=None,
material=None,
poses=None,
wireframe=False,
smooth=True):
"""Helper method that creates a pyrender.Node from a trimesh.Trimesh"""
# Create default pose
if pose is None:
pose = np.eye(4)
elif isinstance(pose, RigidTransform):
pose = pose.matrix
# Create vertex colors if needed
if color is not None:
color = np.asanyarray(color, dtype=np.float)
if color.ndim == 1 or len(color) != len(mesh.vertices):
color = np.repeat(color[np.newaxis, :],
len(mesh.vertices),
axis=0)
mesh.visual.vertex_colors = color
if material is None and mesh.visual.kind != 'texture':
if color is not None:
material = None
else:
material = MetallicRoughnessMaterial(baseColorFactor=np.array(
[1.0, 1.0, 1.0, 1.0]),
metallicFactor=0.2,
roughnessFactor=0.8)
m = Mesh.from_trimesh(mesh,
material=material,
poses=poses,
wireframe=wireframe,
smooth=smooth)
return Node(mesh=m, name=name, matrix=pose)
def __init__(self,
filepath,
viewport_size=(128, 128),
y_fov=3.14159 / 4.,
distance=0.30,
top_only=False,
light=5.0,
theta_steps=10,
phi_steps=10):
self.scene = Scene(bg_color=[0, 0, 0])
self.camera = self.scene.add(
PerspectiveCamera(y_fov, aspectRatio=np.divide(*viewport_size)))
self.mesh = self.scene.add(
Mesh.from_trimesh(trimesh.load(filepath), smooth=True))
self.world_origin = self.mesh.mesh.centroid
self.light = self.scene.add(DirectionalLight([1.0, 1.0, 1.0], light))
self.distance = distance
# 0.1 values are to avoid gimbal lock
theta_max = np.pi / 2.0 if top_only else np.pi
self.thetas = np.linspace(0.1, theta_max - 0.1, theta_steps)
self.phis = np.linspace(0.1, 2 * np.pi - 0.1, phi_steps)
self.renderer = OffscreenRenderer(*viewport_size)
self.RGBA = RenderFlags.RGBA
path_name = './models/flower_models/obj/'+flower_type+'/'
# Load the obj file
flower_trimesh = trimesh.load(path_name+file_name)
# All meshes used default to Scenes. We use dump() to
# retrieve a list of individual sub-meshes (Trimesh)
if (type(flower_trimesh) == trimesh.scene.scene.Scene):
trimesh_list = flower_trimesh.dump()
else:
trimesh_list = [flower_trimesh]
# Convert from Trimesh to Mesh
mesh_list = []
for tri_mesh in trimesh_list:
mesh_list.append(Mesh.from_trimesh(tri_mesh))
# Get indices of flower_mesh and center_mesh in the mesh_list
index_1, index_2, index_3 = get_flower_mesh_indices(file_name)
# Different options for the base/ceiling
base_types = ['soil','plain','ceiling','hydroponic']
base_type = base_types[0]
ceiling_type = base_types[2]
hydroponic_type = base_types[3]
# Base mesh
base_trimesh = trimesh.load('./models/scene/'+base_type+'_base.obj')
base_mesh = Mesh.from_trimesh(base_trimesh)
# Alternative base mesh
def accept_messages(self):
"""
Polls our socket for changes that have been sent by the server.
"""
print("Starting message acceptor")
while True:
try:
packet_header = self.client_socket.recv(
HVTPConstants.HVTP_HEADER_SIZE_IN_BYTES)
print("Received!")
if len(packet_header) == 0:
print("Connection closed by server")
sys.exit()
# Read the header
i = 0
offs = 4
hvtp_magic = packet_header[(i *
offs):(offs + i * offs)].decode(
HVTPConstants.HVTP_ENCODING)
i += 1
hvtp_version = int.from_bytes(
packet_header[(i * offs):(offs + i * offs)],
byteorder='big',
signed=False)
i += 1
hvtp_length = int.from_bytes(
packet_header[(i * offs):(offs + i * offs)],
byteorder='big',
signed=False)
i += 1
hvtp_type = packet_header[(i * offs):(offs + i * offs)].decode(
HVTPConstants.HVTP_ENCODING)
print(hvtp_magic)
print(hvtp_version)
print(hvtp_length)
print(hvtp_type)
# Read the rest of the packet
payload = self.client_socket.recv(hvtp_length)
loaded_payload = trimesh.load(BytesIO(payload),
file_type='glb')
mesh = Mesh.from_trimesh(list(loaded_payload.dump()))
# Add to the scene
self.add_to_scene(mesh)
except IOError as e:
# This is normal on non blocking connections - when there are no incoming data error is going to be raised
# Some operating systems will indicate that using AGAIN, and some using WOULDBLOCK error code
# We are going to check for both - if one of them - that's expected, means no incoming data, continue as normal
# If we got different error code - something happened
if e.errno != errno.EAGAIN and e.errno != errno.EWOULDBLOCK:
print('Reading error: {}'.format(str(e)))
sys.exit()
except Exception as e:
# Any other exception - something happened, exit
print('Reading error: {}'.format(str(e)))
sys.exit()
def configure():
os.environ[
"PYOPENGL_PLATFORM"] = "egl" # opengl seems to only work with TPU
os.system("PYOPENGL_PLATFORM=egl python3 -c 'from OpenGL import EGL'")
assert os.environ['PYOPENGL_PLATFORM'] == 'egl'
assert gl.glGetString(gl.GL_VERSION) is None
assert gl.glGetString(gl.GL_VENDOR) is None
if __name__ == '__main__':
configure()
fuze_trimesh = trimesh.load('../../dataset/models/fuze.obj')
print(fuze_trimesh)
fuze_mesh = Mesh.from_trimesh(fuze_trimesh)
# Drill trimesh
drill_trimesh = trimesh.load('../../dataset/models/drill.obj')
drill_mesh = Mesh.from_trimesh(drill_trimesh)
drill_pose = np.eye(4)
drill_pose[0, 3] = 0.1
drill_pose[2, 3] = -np.min(drill_trimesh.vertices[:, 2])
# Wood trimesh
wood_trimesh = trimesh.load('../../dataset/models/wood.obj')
wood_mesh = Mesh.from_trimesh(wood_trimesh)
# Water bottle trimesh
bottle_gltf = trimesh.load('../../dataset/models/WaterBottle.glb')
bottle_trimesh = bottle_gltf.geometry[list(bottle_gltf.geometry.keys())[0]]
from pyrender import Mesh, Scene, Viewer
from io import BytesIO
import numpy as np
import trimesh
import requests
import time
duck_source = "https://github.com/KhronosGroup/glTF-Sample-Models/raw/master/2.0/Duck/glTF-Binary/Duck.glb"
duck = trimesh.load(BytesIO(requests.get(duck_source).content),
file_type='glb')
duckmesh = Mesh.from_trimesh(list(duck.geometry.values())[0])
scene = Scene(ambient_light=np.array([1.0, 1.0, 1.0, 1.0]))
scene.add(duckmesh)
orig_positions = duckmesh.primitives[0].positions
vwr = Viewer(scene,
run_in_thread=True,
render_flags={
"cull_faces": False,
"all_wireframe": False
},
use_raymond_lighting=True,
bg_color=[255, 0, 255])
while vwr.is_active:
with vwr.render_lock:
duckmesh.primitives[0].positions = orig_positions + np.random.randn(
*orig_positions.shape) * 2
time.sleep(0.1)
def test_meshes():
with pytest.raises(TypeError):
x = Mesh()
with pytest.raises(TypeError):
x = Primitive()
with pytest.raises(ValueError):
x = Primitive([], mode=10)
# Basics
x = Mesh([])
assert x.name is None
assert x.is_visible
assert x.weights is None
x.name = 'str'
# From Trimesh
x = Mesh.from_trimesh(trimesh.creation.box())
assert isinstance(x, Mesh)
assert len(x.primitives) == 1
assert x.is_visible
assert np.allclose(x.bounds, np.array([
[-0.5, -0.5, -0.5],
[0.5, 0.5, 0.5]
]))
assert np.allclose(x.centroid, np.zeros(3))
assert np.allclose(x.extents, np.ones(3))
assert np.allclose(x.scale, np.sqrt(3))
assert not x.is_transparent
# Test some primitive functions
x = x.primitives[0]
with pytest.raises(ValueError):
x.normals = np.zeros(10)
with pytest.raises(ValueError):
x.tangents = np.zeros(10)
with pytest.raises(ValueError):
x.texcoord_0 = np.zeros(10)
with pytest.raises(ValueError):
x.texcoord_1 = np.zeros(10)
with pytest.raises(TypeError):
x.material = np.zeros(10)
assert x.targets is None
assert np.allclose(x.bounds, np.array([
[-0.5, -0.5, -0.5],
[0.5, 0.5, 0.5]
]))
assert np.allclose(x.centroid, np.zeros(3))
assert np.allclose(x.extents, np.ones(3))
assert np.allclose(x.scale, np.sqrt(3))
x.material.baseColorFactor = np.array([0.0, 0.0, 0.0, 0.0])
assert x.is_transparent
# From two trimeshes
x = Mesh.from_trimesh([trimesh.creation.box(),
trimesh.creation.cylinder(radius=0.1, height=2.0)],
smooth=False)
assert isinstance(x, Mesh)
assert len(x.primitives) == 2
assert x.is_visible
assert np.allclose(x.bounds, np.array([
[-0.5, -0.5, -1.0],
[0.5, 0.5, 1.0]
]))
assert np.allclose(x.centroid, np.zeros(3))
assert np.allclose(x.extents, [1.0, 1.0, 2.0])
assert np.allclose(x.scale, np.sqrt(6))
assert not x.is_transparent
# From bad data
with pytest.raises(TypeError):
x = Mesh.from_trimesh(None)
# With instancing
poses = np.tile(np.eye(4), (5,1,1))
poses[:,0,3] = np.array([0,1,2,3,4])
x = Mesh.from_trimesh(trimesh.creation.box(), poses=poses)
assert np.allclose(x.bounds, np.array([
[-0.5, -0.5, -0.5],
[4.5, 0.5, 0.5]
]))
poses = np.eye(4)
x = Mesh.from_trimesh(trimesh.creation.box(), poses=poses)
poses = np.eye(3)
with pytest.raises(ValueError):
x = Mesh.from_trimesh(trimesh.creation.box(), poses=poses)
# From textured meshes
fm = trimesh.load('tests/data/fuze.obj')
x = Mesh.from_trimesh(fm)
assert isinstance(x, Mesh)
assert len(x.primitives) == 1
assert x.is_visible
assert not x.is_transparent
assert x.primitives[0].material.baseColorTexture is not None
x = Mesh.from_trimesh(fm, smooth=False)
fm.visual = fm.visual.to_color()
fm.visual.face_colors = np.array([1.0, 0.0, 0.0, 1.0])
x = Mesh.from_trimesh(fm, smooth=False)
with pytest.raises(ValueError):
x = Mesh.from_trimesh(fm, smooth=True)
fm.visual.vertex_colors = np.array([1.0, 0.0, 0.0, 0.5])
x = Mesh.from_trimesh(fm, smooth=False)
x = Mesh.from_trimesh(fm, smooth=True)
assert x.primitives[0].color_0 is not None
assert x.is_transparent
bm = trimesh.load('tests/data/WaterBottle.glb').dump()[0]
x = Mesh.from_trimesh(bm)
assert x.primitives[0].material.baseColorTexture is not None
assert x.primitives[0].material.emissiveTexture is not None
assert x.primitives[0].material.metallicRoughnessTexture is not None
# From point cloud
x = Mesh.from_points(fm.vertices)
# Determine path name
path_name = './models/flower_models/obj/' + flower_type + '/'
# Load the obj file
flower_trimesh = trimesh.load(path_name + file_name)
if (type(flower_trimesh) == trimesh.scene.scene.Scene):
trimesh_list = flower_trimesh.dump()
else:
trimesh_list = [flower_trimesh]
# Convert from Trimesh to Mesh
mesh_list = []
for tri_mesh in trimesh_list:
mesh_list.append(Mesh.from_trimesh(tri_mesh))
# Get indices of flower_mesh and center_mesh in the mesh_list
index_1, index_2 = get_flower_mesh_indices(file_name)
# Different options for the base
base_types = ['soil', 'plain', 'other']
base_type = base_types[0]
# Base mesh
base_trimesh = trimesh.load('./models/scene/' + base_type + '_base.obj')
base_mesh = Mesh.from_trimesh(base_trimesh)
base_pose = spatial_transform_Matrix(scale=1 / 2, t_z=-0.01)
# Inf base mesh (big square that sits underneath the scene)
inf_base_trimesh = trimesh.load('./models/scene/' + base_type + '_base.obj')
def test_nodes():
x = Node()
assert x.name is None
assert x.camera is None
assert x.children == []
assert x.skin is None
assert np.allclose(x.matrix, np.eye(4))
assert x.mesh is None
assert np.allclose(x.rotation, [0, 0, 0, 1])
assert np.allclose(x.scale, np.ones(3))
assert np.allclose(x.translation, np.zeros(3))
assert x.weights is None
assert x.light is None
x.name = 'node'
# Test node light/camera/mesh tests
c = PerspectiveCamera(yfov=2.0)
m = Mesh([])
d = DirectionalLight()
x.camera = c
assert x.camera == c
with pytest.raises(TypeError):
x.camera = m
x.camera = d
x.camera = None
x.mesh = m
assert x.mesh == m
with pytest.raises(TypeError):
x.mesh = c
x.mesh = d
x.light = d
assert x.light == d
with pytest.raises(TypeError):
x.light = m
x.light = c
# Test transformations getters/setters/etc...
# Set up test values
x = np.array([1.0, 0.0, 0.0])
y = np.array([0.0, 1.0, 0.0])
t = np.array([1.0, 2.0, 3.0])
s = np.array([0.5, 2.0, 1.0])
Mx = transformations.rotation_matrix(np.pi / 2.0, x)
qx = np.roll(transformations.quaternion_about_axis(np.pi / 2.0, x), -1)
Mxt = Mx.copy()
Mxt[:3, 3] = t
S = np.eye(4)
S[:3, :3] = np.diag(s)
Mxts = Mxt.dot(S)
My = transformations.rotation_matrix(np.pi / 2.0, y)
qy = np.roll(transformations.quaternion_about_axis(np.pi / 2.0, y), -1)
Myt = My.copy()
Myt[:3, 3] = t
x = Node(matrix=Mx)
assert np.allclose(x.matrix, Mx)
assert np.allclose(x.rotation, qx)
assert np.allclose(x.translation, np.zeros(3))
assert np.allclose(x.scale, np.ones(3))
x.matrix = My
assert np.allclose(x.matrix, My)
assert np.allclose(x.rotation, qy)
assert np.allclose(x.translation, np.zeros(3))
assert np.allclose(x.scale, np.ones(3))
x.translation = t
assert np.allclose(x.matrix, Myt)
assert np.allclose(x.rotation, qy)
x.rotation = qx
assert np.allclose(x.matrix, Mxt)
x.scale = s
assert np.allclose(x.matrix, Mxts)
x = Node(matrix=Mxt)
assert np.allclose(x.matrix, Mxt)
assert np.allclose(x.rotation, qx)
assert np.allclose(x.translation, t)
assert np.allclose(x.scale, np.ones(3))
x = Node(matrix=Mxts)
assert np.allclose(x.matrix, Mxts)
assert np.allclose(x.rotation, qx)
assert np.allclose(x.translation, t)
assert np.allclose(x.scale, s)
# Individual element getters
x.scale[0] = 0
assert np.allclose(x.scale[0], 0)
x.translation[0] = 0
assert np.allclose(x.translation[0], 0)
x.matrix = np.eye(4)
x.matrix[0, 0] = 500
assert x.matrix[0, 0] == 1.0
# Failures
with pytest.raises(ValueError):
x.matrix = 5 * np.eye(4)
with pytest.raises(ValueError):
x.matrix = np.eye(5)
with pytest.raises(ValueError):
x.matrix = np.eye(4).dot([5, 1, 1, 1])
with pytest.raises(ValueError):
x.rotation = np.array([1, 2])
with pytest.raises(ValueError):
x.rotation = np.array([1, 2, 3])
with pytest.raises(ValueError):
x.rotation = np.array([1, 2, 3, 4])
with pytest.raises(ValueError):
x.translation = np.array([1, 2, 3, 4])
with pytest.raises(ValueError):
x.scale = np.array([1, 2, 3, 4])
xyz = np.transpose(xyz)
# N,3
return xyz
if __name__ == '__main__':
#==============================================================================
# Mesh creation
#==============================================================================
# Drill trimesh
drill_trimesh = trimesh.load('./models/drill.obj')
drill_trimesh.visual.vertex_colors = np.array((1, 0, 0, 1))
drill_trimesh.visual.face_colors = np.array((1, 0, 0, 1))
drill_mesh = Mesh.from_trimesh(drill_trimesh)
#drill_pose = np.eye(4)
#drill_pose[0,3] = 0.1
#drill_pose[2,3] = -np.min(drill_trimesh.vertices[:,2])
#==============================================================================
# Camera creation
#==============================================================================
cam = PerspectiveCamera(yfov=(np.pi / 3.0),
aspectRatio=1.0 * 640 / 480,
znear=0.1,
zfar=6)
cam_pose = np.array([[0.0, -np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.5], [1.0, 0.0, 0.0, 0.0],
def test_scenes():
# Basics
s = Scene()
assert np.allclose(s.bg_color, np.ones(4))
assert np.allclose(s.ambient_light, np.zeros(3))
assert len(s.nodes) == 0
assert s.name is None
s.name = 'asdf'
s.bg_color = None
s.ambient_light = None
assert np.allclose(s.bg_color, np.ones(4))
assert np.allclose(s.ambient_light, np.zeros(3))
assert s.nodes == set()
assert s.cameras == set()
assert s.lights == set()
assert s.point_lights == set()
assert s.spot_lights == set()
assert s.directional_lights == set()
assert s.meshes == set()
assert s.camera_nodes == set()
assert s.light_nodes == set()
assert s.point_light_nodes == set()
assert s.spot_light_nodes == set()
assert s.directional_light_nodes == set()
assert s.mesh_nodes == set()
assert s.main_camera_node is None
assert np.all(s.bounds == 0)
assert np.all(s.centroid == 0)
assert np.all(s.extents == 0)
assert np.all(s.scale == 0)
# From trimesh scene
tms = trimesh.load('tests/data/WaterBottle.glb')
s = Scene.from_trimesh_scene(tms)
assert len(s.meshes) == 1
assert len(s.mesh_nodes) == 1
# Test bg color formatting
s = Scene(bg_color=[0, 1.0, 0])
assert np.allclose(s.bg_color, np.array([0.0, 1.0, 0.0, 1.0]))
# Test constructor for nodes
n1 = Node()
n2 = Node()
n3 = Node()
nodes = [n1, n2, n3]
s = Scene(nodes=nodes)
n1.children.append(n2)
s = Scene(nodes=nodes)
n3.children.append(n2)
with pytest.raises(ValueError):
s = Scene(nodes=nodes)
n3.children = []
n2.children.append(n3)
n3.children.append(n2)
with pytest.raises(ValueError):
s = Scene(nodes=nodes)
# Test node accessors
n1 = Node()
n2 = Node()
n3 = Node()
nodes = [n1, n2]
s = Scene(nodes=nodes)
assert s.has_node(n1)
assert s.has_node(n2)
assert not s.has_node(n3)
# Test node poses
for n in nodes:
assert np.allclose(s.get_pose(n), np.eye(4))
with pytest.raises(ValueError):
s.get_pose(n3)
with pytest.raises(ValueError):
s.set_pose(n3, np.eye(4))
tf = np.eye(4)
tf[:3, 3] = np.ones(3)
s.set_pose(n1, tf)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), np.eye(4))
nodes = [n1, n2, n3]
tf2 = np.eye(4)
tf2[:3, :3] = np.diag([-1, -1, 1])
n1.children.append(n2)
n1.matrix = tf
n2.matrix = tf2
s = Scene(nodes=nodes)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), tf.dot(tf2))
assert np.allclose(s.get_pose(n3), np.eye(4))
n1 = Node()
n2 = Node()
n3 = Node()
n1.children.append(n2)
s = Scene()
s.add_node(n1)
with pytest.raises(ValueError):
s.add_node(n2)
s.set_pose(n1, tf)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), tf)
s.set_pose(n2, tf2)
assert np.allclose(s.get_pose(n2), tf.dot(tf2))
# Test node removal
n1 = Node()
n2 = Node()
n3 = Node()
n1.children.append(n2)
n2.children.append(n3)
s = Scene(nodes=[n1, n2, n3])
s.remove_node(n2)
assert len(s.nodes) == 1
assert n1 in s.nodes
assert len(n1.children) == 0
assert len(n2.children) == 1
s.add_node(n2, parent_node=n1)
assert len(n1.children) == 1
n1.matrix = tf
n3.matrix = tf2
assert np.allclose(s.get_pose(n3), tf.dot(tf2))
# Now test ADD function
s = Scene()
m = Mesh([], name='m')
cp = PerspectiveCamera(yfov=2.0)
co = OrthographicCamera(xmag=1.0, ymag=1.0)
dl = DirectionalLight()
pl = PointLight()
sl = SpotLight()
n1 = s.add(m, name='mn')
assert n1.mesh == m
assert len(s.nodes) == 1
assert len(s.mesh_nodes) == 1
assert n1 in s.mesh_nodes
assert len(s.meshes) == 1
assert m in s.meshes
assert len(s.get_nodes(node=n2)) == 0
n2 = s.add(m, pose=tf)
assert len(s.nodes) == len(s.mesh_nodes) == 2
assert len(s.meshes) == 1
assert len(s.get_nodes(node=n1)) == 1
assert len(s.get_nodes(node=n1, name='mn')) == 1
assert len(s.get_nodes(name='mn')) == 1
assert len(s.get_nodes(obj=m)) == 2
assert len(s.get_nodes(obj=m, obj_name='m')) == 2
assert len(s.get_nodes(obj=co)) == 0
nsl = s.add(sl, name='sln')
npl = s.add(pl, parent_name='sln')
assert nsl.children[0] == npl
ndl = s.add(dl, parent_node=npl)
assert npl.children[0] == ndl
nco = s.add(co)
ncp = s.add(cp)
assert len(s.light_nodes) == len(s.lights) == 3
assert len(s.point_light_nodes) == len(s.point_lights) == 1
assert npl in s.point_light_nodes
assert len(s.spot_light_nodes) == len(s.spot_lights) == 1
assert nsl in s.spot_light_nodes
assert len(s.directional_light_nodes) == len(s.directional_lights) == 1
assert ndl in s.directional_light_nodes
assert len(s.cameras) == len(s.camera_nodes) == 2
assert s.main_camera_node == nco
s.main_camera_node = ncp
s.remove_node(ncp)
assert len(s.cameras) == len(s.camera_nodes) == 1
assert s.main_camera_node == nco
s.remove_node(n2)
assert len(s.meshes) == 1
s.remove_node(n1)
assert len(s.meshes) == 0
s.remove_node(nsl)
assert len(s.lights) == 0
s.remove_node(nco)
assert s.main_camera_node is None
s.add_node(n1)
s.clear()
assert len(s.nodes) == 0
# Trigger final errors
with pytest.raises(ValueError):
s.main_camera_node = None
with pytest.raises(ValueError):
s.main_camera_node = ncp
with pytest.raises(ValueError):
s.add(m, parent_node=n1)
with pytest.raises(ValueError):
s.add(m, name='asdf')
s.add(m, name='asdf')
s.add(m, parent_name='asdf')
with pytest.raises(ValueError):
s.add(m, parent_name='asfd')
with pytest.raises(TypeError):
s.add(None)
s.clear()
# Test bounds
m1 = Mesh.from_trimesh(trimesh.creation.box())
m2 = Mesh.from_trimesh(trimesh.creation.box())
m3 = Mesh.from_trimesh(trimesh.creation.box())
n1 = Node(mesh=m1)
n2 = Node(mesh=m2, translation=[1.0, 0.0, 0.0])
n3 = Node(mesh=m3, translation=[0.5, 0.0, 1.0])
s.add_node(n1)
s.add_node(n2)
s.add_node(n3)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [1.5, 0.5, 1.5]])
s.clear()
s.add_node(n1)
s.add_node(n2, parent_node=n1)
s.add_node(n3, parent_node=n2)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [2.0, 0.5, 1.5]])
tf = np.eye(4)
tf[:3, 3] = np.ones(3)
s.set_pose(n3, tf)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [2.5, 1.5, 1.5]])
s.remove_node(n2)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]])
s.clear()
assert np.allclose(s.bounds, 0.0)
