def setup_standard_scene(camera):
"""
Creates an empty scene with some standard lighting and the given camera.
Parameters
----------
camera:
pyrender camera that should be used for rendering
Returns
-------
an empty scene with lighting and camera set up.
"""
directional_light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=2.0)
scene = pyrender.Scene()
scene.add(camera)
scene.add(directional_light)
return scene
def __call__(self, vertices, camera_translation, image):
material = pyrender.MetallicRoughnessMaterial(
metallicFactor=0.2,
alphaMode='OPAQUE',
baseColorFactor=(0.8, 0.3, 0.3, 1.0))
camera_translation[0] *= -1.
mesh = trimesh.Trimesh(vertices, self.faces)
rot = trimesh.transformations.rotation_matrix(np.radians(180),
[1, 0, 0])
mesh.apply_transform(rot)
mesh = pyrender.Mesh.from_trimesh(mesh, material=material)
scene = pyrender.Scene(ambient_light=(0.5, 0.5, 0.5))
scene.add(mesh, 'mesh')
camera_pose = np.eye(4)
camera_pose[:3, 3] = camera_translation
camera = pyrender.IntrinsicsCamera(fx=self.focal_length,
fy=self.focal_length,
cx=self.camera_center[0],
cy=self.camera_center[1])
scene.add(camera, pose=camera_pose)
light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=1)
light_pose = np.eye(4)
light_pose[:3, 3] = np.array([0, -1, 1])
scene.add(light, pose=light_pose)
light_pose[:3, 3] = np.array([0, 1, 1])
scene.add(light, pose=light_pose)
light_pose[:3, 3] = np.array([1, 1, 2])
scene.add(light, pose=light_pose)
color, rend_depth = self.renderer.render(
scene, flags=pyrender.RenderFlags.RGBA)
color = color.astype(np.float32) / 255.0
valid_mask = (rend_depth > 0)[:, :, None]
output_img = (color[:, :, :3] * valid_mask + (1 - valid_mask) * image)
return output_img
def img_renderer(pointcloud):
pcd = o3d.io.read_point_cloud(pointcloud)
pcd = pcd.voxel_down_sample(voxel_size=0.05)
pcd.estimate_normals()
# estimate radius for rolling ball
distances = pcd.compute_nearest_neighbor_distance()
avg_dist = np.mean(distances)
radius = 5 * avg_dist
mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_ball_pivoting(
pcd,
o3d.utility.DoubleVector([radius, radius * 2]))
# create the triangular mesh with the vertices and faces from open3d
tri_mesh = trimesh.Trimesh(np.asarray(mesh.vertices), np.asarray(mesh.triangles),
vertex_normals=np.asarray(mesh.vertex_normals))
#tri_mesh = trimesh.convex.is_convex(tri_mesh)
mesh = pyrender.Mesh.from_trimesh(tri_mesh)
scene = pyrender.Scene()
scene.add(mesh)
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
s = np.sqrt(2)/2
camera_pose = np.array([
[0.0, -s, s, 0.3],
[1.0, 0.0, 0.0, 0.0],
[0.0, s, s, 0.35],
[0.0, 0.0, 0.0, 1.0], ])
scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3), intensity=3.0,
innerConeAngle=np.pi/16.0,
outerConeAngle=np.pi/6.0)
scene.add(light, pose=camera_pose)
pyrender.Viewer(scene)
r = pyrender.OffscreenRenderer(512, 512)
color, _ = r.render(scene)
plt.figure(figsize=(8,8))
plt.imshow(color)
def render(obsinfo, bg_color=[0.0, 0.0, 0.0], wireframe=False):
scene = pyrender.Scene(bg_color=bg_color)
camera = pyrender.PerspectiveCamera(
yfov=np.radians(obsinfo.fov.fovy), aspectRatio=obsinfo.fov.aspect
)
# rot_z(180) . rot_y(180)
camera_pose = np.array(
[[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]],
dtype=np.float64,
)
scene.add(camera, pose=camera_pose)
star_image = np.zeros(
shape=(obsinfo.height, obsinfo.width, 4), dtype=np.uint8
)
for star in obsinfo.stars:
star_image += render_star(star, obsinfo.width, obsinfo.height)
for solar_object in obsinfo.solar_objects:
mesh, pose = render_solar_object(solar_object, wireframe)
scene.add(mesh, pose=pose)
# light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=3.8e27)
light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=3.8e17)
pose = np.identity(4)
pose[0:3, 3] = -obsinfo.pos
scene.add(light, pose=pose)
# Render the scene
r = pyrender.OffscreenRenderer(obsinfo.width, obsinfo.height)
flags = pyrender.RenderFlags.RGBA | pyrender.RenderFlags.SHADOWS_DIRECTIONAL
foreground, _ = r.render(scene, flags=flags)
# background layer: star_image, foreground layer:foreground
bg_color.append(1.0)
bg_color_int = (np.array(bg_color) * 255).astype(int)
return np.where(
foreground != bg_color_int,
foreground,
np.where(star_image != [0, 0, 0, 0], star_image, bg_color_int),
)
def render_mesh(img, mesh, face, cam_param):
# mesh
mesh = trimesh.Trimesh(mesh, face)
rot = trimesh.transformations.rotation_matrix(np.radians(180), [1, 0, 0])
mesh.apply_transform(rot)
material = pyrender.MetallicRoughnessMaterial(metallicFactor=0.0,
alphaMode='OPAQUE',
baseColorFactor=(1.0, 1.0,
0.9, 1.0))
mesh = pyrender.Mesh.from_trimesh(mesh, material=material, smooth=False)
scene = pyrender.Scene(ambient_light=(0.3, 0.3, 0.3))
scene.add(mesh, 'mesh')
focal, princpt = cam_param['focal'], cam_param['princpt']
camera = pyrender.IntrinsicsCamera(fx=focal[0],
fy=focal[1],
cx=princpt[0],
cy=princpt[1])
scene.add(camera)
# renderer
renderer = pyrender.OffscreenRenderer(viewport_width=img.shape[1],
viewport_height=img.shape[0],
point_size=1.0)
# light
light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=0.8)
light_pose = np.eye(4)
light_pose[:3, 3] = np.array([0, -1, 1])
scene.add(light, pose=light_pose)
light_pose[:3, 3] = np.array([0, 1, 1])
scene.add(light, pose=light_pose)
light_pose[:3, 3] = np.array([1, 1, 2])
scene.add(light, pose=light_pose)
# render
rgb, depth = renderer.render(scene, flags=pyrender.RenderFlags.RGBA)
rgb = rgb[:, :, :3].astype(np.float32)
valid_mask = (depth > 0)[:, :, None]
# save to image
img = rgb * valid_mask + img * (1 - valid_mask)
return img
def render_mesh(model):
dist = 2
angle = 20
height = -0.2
scene = pyrender.Scene(ambient_light=[.1, .1, .3], bg_color=(0, 0, 0))
scene.add(pyrender.Mesh.from_trimesh(model, smooth=False))
light = pyrender.DirectionalLight(color=[1, 1, 1], intensity=2e3)
scene.add(light, pose=np.eye(4))
c = np.cos(angle * np.pi / 180)
s = np.sin(angle * np.pi / 180)
camera_pose = np.array([[c, 0, s, dist * s], [0, 1, 0, height],
[-1 * s, 0, c, dist * c], [0, 0, 0, 1]])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, znear=0.5, zfar=5)
scene.add(camera, pose=camera_pose)
renderer = pyrender.OffscreenRenderer(512, 512)
color, _ = renderer.render(scene)
return color[:, :, ::-1]
def export_3d_image(self, filename="3d_render.png", tolerance=0.005):
"""Creates a 3D rendered image (png) of the reactor
:param filename: output filename of the image created
:type filename: [ParamType](, optional)
:param tolerance: the mesh tolerance
:type tolerance: float
:return: filename of the created image
:rtype: str
"""
scene = pyrender.Scene(ambient_light=np.array([0.1, 0.1, 0.1, 1.0]))
for entry in self.shapes_and_components:
if entry.render_mesh is None:
scene.add(entry._create_render_mesh(tolerance))
# sets the field of view (fov) and the aspect ratio of the image
camera = pyrender.camera.PerspectiveCamera(yfov=math.radians(90.0),
aspectRatio=2.0)
# sets the position of the camera using a matrix
c = 2**-0.5
camera_pose = np.array([[1, 0, 0, 0], [0, c, -c, -500], [0, c, c, 500],
[0, 0, 0, 1]])
scene.add(camera, pose=camera_pose)
# adds some basic lighting to the scene
light = pyrender.DirectionalLight(color=np.ones(3), intensity=1.0)
scene.add(light, pose=camera_pose)
# Render the scene
renderer = pyrender.OffscreenRenderer(1000, 500)
colours, depth = renderer.render(scene)
image = Image.fromarray(colours, "RGB")
Path(filename).parent.mkdir(parents=True, exist_ok=True)
image.save(filename, "PNG")
print("\n saved 3d image to ", filename)
return filename
def save_image_face_heatmap(facedata, vec, errors, id, name="face_heat", path=""):
plt.clf()
plt.close()
colors_heat = []
# Map errors to RGB colors
min_error = 0
max_error = 6
for i, er in enumerate(errors[id]):
c_er = abs(er)
if c_er > max_error:
c_er = max_error
c = rgb(min_error, max_error, c_er)
colors_heat.append(c)
# Generate colored mesh
predict_trimeshh = facedata.vec2meshTrimesh2(vec, col=colors_heat)
trimeshh = pyrender.Mesh.from_trimesh(predict_trimeshh, smooth=False)
# Create scene for rendering, etc.
scene = pyrender.Scene(ambient_light=[.1, .1, .3], bg_color=[255, 255, 255])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
light = pyrender.DirectionalLight(color=[1, 1, 1], intensity=2e3)
scene.add(trimeshh, pose=np.eye(4))
scene.add(light, pose=np.eye(4))
# non chiedermi mai come ho trovato questi valori bellissimi (T/N: just use these values)
camera_pose = np.array([
[0.94063, 0.01737, -0.41513, -88.15790],
[-0.06728, 0.98841, -0.16663, -35.36127],
[0.33266, 0.15078, 1.14014, 241.71166],
[0.00000, 0.00000, 0.00000, 1.00000]
])
scene.add(camera, pose=camera_pose)
r = pyrender.OffscreenRenderer(512, 512)
color, _ = r.render(scene)
plt.figure(figsize=(8, 8))
plt.imshow(color)
name_image = path+name+str(".png")
# plt.colorbar() # Could be a nice addition, however unneeded
plt.savefig(name_image)
plt.clf()
def __init__(self, faces, img_res=512):
self.renderer = pyrender.OffscreenRenderer(viewport_width=img_res,
viewport_height=img_res,
point_size=1.0)
self.camera_center = [img_res // 2, img_res // 2]
self.faces = faces
self.img_res = img_res
# set the scene
self.scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0, 0.0],
ambient_light=(0.3, 0.3, 0.3))
light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=1.)
light_pose = np.eye(4)
light_pose[:3, 3] = [0, -1, 1]
self.scene.add(light, pose=light_pose)
light_pose[:3, 3] = [0, 1, 1]
self.scene.add(light, pose=light_pose)
def test2():
model_path = 'data/models/basic/cow.obj'
# load the cow model
tm = trimesh.load(model_path)
tm.visual.vertex_colors = np.random.uniform(
size=tm.visual.vertex_colors.shape)
tm.visual.face_colors = np.random.uniform(size=tm.visual.face_colors.shape)
mesh = pyrender.Mesh.from_trimesh(tm, smooth=False)
light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=2.0)
cam = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.414)
nm = pyrender.Node(mesh=mesh, matrix=np.eye(4))
nl = pyrender.Node(light=light, matrix=np.eye(4))
nc = pyrender.Node(camera=cam, matrix=np.eye(4))
scene = pyrender.Scene(ambient_light=[1.0, 1.0, 1.0], bg_color=gray)
scene.add_node(nm)
scene.add_node(nl, parent_node=nm)
scene.add_node(nc, parent_node=nm)
pyrender.Viewer(scene, use_raymond_lighting=True)
def __init__(self, obj, intrinsic: torch.Tensor):
self.intrinsic = intrinsic
self.scene = pyrender.Scene(bg_color=(0, 0, 0, 0),
ambient_light=(0.1, 0.1, 0.1))
fx = self.intrinsic[0, 0].item()
fy = self.intrinsic[1, 1].item()
cx = self.intrinsic[0, 2].item()
cy = self.intrinsic[1, 2].item()
self.camera = pyrender.IntrinsicsCamera(fx, fy, cx, cy)
# Create lights.
self.light_nodes = []
self.extrinsic = None
self.camera_node = self.scene.add(self.camera, name='camera')
self.obj = obj
self.object_node = _create_object_node(self.obj)
self.scene.add_node(self.object_node)
def set_render(self, vertices, faces, visual=None, normalize=True):
self.tri_mesh = trimesh.Trimesh(vertices=vertices,
faces=faces,
visual=visual)
self.render = pyrender.OffscreenRenderer(self.width, self.height)
self.py_mesh = pyrender.Mesh.from_trimesh(self.tri_mesh)
self.py_scene = pyrender.Scene(ambient_light=[0.0, 0.0, 0.0],
bg_color=[-1.0, -1.0, -1.0])
self.py_scene.add(self.py_mesh, pose=self.global_tr)
self.py_scene.add(self.camera, pose=self.camera_pose)
self.vertices = vertices
self.faces = faces
if normalize:
self.vertices -= self.py_scene.centroid
bounds = self.tri_mesh.bounding_box_oriented.extents
self.vertices /= [bounds[0]] * 3
self.vertices += 1 / 2
def __init__(self, is_shading=True, d_light=3., scale=None):
self.is_shading = is_shading
self.light = (.3, .3, .3) if self.is_shading else (1., 1., 1.)
self.scene = pyrender.Scene(bg_color=[255, 255, 255],
ambient_light=self.light)
self.size = None
self.viewer = None
self.T = None
self.K_no_scale = None
self.K = None
self.camera = None
self.camera_node = None
self.d_light = d_light
self.light_nodes = None
self.scale = scale
def get_frame(self, phi, theta):
scene = pyrender.Scene(bg_color=np.zeros(4))
scene.add(self.mesh)
camera = pyrender.PerspectiveCamera(yfov=v_fov * pi / 180)
camera_rot_y = np.array([
[1.0, 0.0, 0.0, 0.0],
[0.0, cos(phi), -sin(phi), 0.0],
[0.0, sin(phi), cos(phi), 0.0],
[0.0, 0.0, 0.0, 1.0],
])
camera_rot_x = np.array([
[cos(theta), 0.0, sin(theta), 0.0],
[0.0, 1.0, 0.0, 0.0],
[-sin(theta), 0.0, cos(theta), 0.0],
[0.0, 0.0, 0.0, 1.0],
])
camera_rot_z = np.eye(4)
# camera_rot_z = np.array([
# [1, 0.0, 0.0, 0.0],
# [0.0, cos(gamma), sin(gamma), 0.0],
# [0.0, -sin(gamma), cos(gamma), 0.0],
# [0.0, 0.0, 0.0, 1.0],
# ])
camera_pose = np.matmul(np.matmul(camera_rot_y, camera_rot_x),
camera_rot_z)
camera_pose[0][3] = 0.0
camera_pose[1][3] = 0.0
camera_pose[2][3] = 7.0
# camera_pose = np.array([
# [1.0, 0.0, 0.0, -20],
# [0.0, 1.0, 0.0, 3],
# [0.0, 0.0, 1.0, 150],
# [0.0, 0.0, 0.0, 1.0],
# ])
scene.add(camera, pose=camera_pose)
# Set up the light
light = pyrender.PointLight(color=np.ones(3), intensity=30.0)
scene.add(light, pose=camera_pose)
color, depth = self.renderer.render(scene)
return color, depth
def __call__(self, vertices, faces):
material = pyrender.MetallicRoughnessMaterial(
metallicFactor=0.2,
alphaMode='OPAQUE',
baseColorFactor=(0.8, 0.3, 0.3, 1.0))
camera_translation = np.array([0.0, 0.0, 50.0])
mesh = trimesh.Trimesh(vertices[0], faces[0], process=False)
# rot = trimesh.transformations.rotation_matrix(
# np.radians(180), [1, 0, 0])
# mesh.apply_transform(rot)
mesh = pyrender.Mesh.from_trimesh(mesh, material=material)
scene = pyrender.Scene(ambient_light=(0.5, 0.5, 0.5))
scene.add(mesh, 'mesh')
camera_pose = np.eye(4)
camera_pose[:3, 3] = camera_translation
camera = pyrender.IntrinsicsCamera(fx=self.focal_length,
fy=self.focal_length,
cx=self.camera_center[0],
cy=self.camera_center[1])
scene.add(camera, pose=camera_pose)
light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=1)
light_pose = np.eye(4)
light_pose[:3, 3] = np.array([0, -1, 1])
scene.add(light, pose=light_pose)
light_pose[:3, 3] = np.array([0, 1, 1])
scene.add(light, pose=light_pose)
light_pose[:3, 3] = np.array([1, 1, 2])
scene.add(light, pose=light_pose)
color, rend_depth = self.renderer.render(
scene, flags=pyrender.RenderFlags.RGBA)
color = color.astype(np.float32) / 255.0
return torch.from_numpy(color).float().unsqueeze(0)
def img_renderer(img, mesh, light_est):
# compose scene
scene = pyrender.Scene(ambient_light=[.1, .1, .3], bg_color=[0, 0, 0])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
light = pyrender.DirectionalLight(color=[1, 1, 1], intensity=2e3)
scene.add(mesh, pose=np.eye(4))
scene.add(light_est, pose=np.eye(4))
# c = 2**-0.5
# scene.add(camera, pose=[[ 1, 0, 0, 0],
# [ 0, c, -c, -2],
# [ 0, c, c, 2],
# [ 0, 0, 0, 1]])
# render scene
r = pyrender.OffscreenRenderer(512, 512)
color, _ = r.render(scene)
plt.figure(figsize=(8, 8)), plt.imshow(color)
def __init__(self, resolution=(256, 256)):
self.resolution = resolution
self.faces = np.load(config.SMPL_FACES_PATH)
self.renderer = pyrender.OffscreenRenderer(
viewport_width=self.resolution[0],
viewport_height=self.resolution[1],
point_size=1.0)
# set the scene
self.scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0, 0.0],
ambient_light=(0.3, 0.3, 0.3))
light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=1.)
light_pose = np.eye(4)
light_pose[:3, 3] = [0, -1, 1]
self.scene.add(light, pose=light_pose)
light_pose[:3, 3] = [0, 1, 1]
self.scene.add(light, pose=light_pose)
def generate_random_scene(model):
scene = pyrender.Scene( # Create new scene
ambient_light=[int(random.uniform(.3, .7) * 255) for _ in range(3)])
scene, model_node = _add_model(scene, model) # Add model to scene
scene, cam = _add_camera(scene) # Add camera
for l_type in ('directional_lights',
'point_lights'): # Adds light to the scene
scene = _add_lighting(scene, light_type=l_type)
'''
scene.add(
pyrender.DirectionalLight(
color=np.ones(3),
intensity=10.0,
), pose = scene.get_pose(scene.main_camera_node)
)
'''
return scene, model_node, cam
def render_mesh(mesh, height, width):
scene = pyrender.Scene(ambient_light=[.3, .3, .3],
bg_color=[255, 255, 255])
rgb_per_v = np.zeros_like(mesh.v)
rgb_per_v[:, 0] = 0.53
rgb_per_v[:, 1] = 0.81
rgb_per_v[:, 2] = 0.98
tri_mesh = trimesh.Trimesh(vertices=0.001 * mesh.v,
faces=mesh.f,
vertex_colors=rgb_per_v)
render_mesh = pyrender.Mesh.from_trimesh(tri_mesh, smooth=True)
scene.add(render_mesh, pose=np.eye(4))
camera = pyrender.camera.OrthographicCamera(xmag=0.001 * 0.5 * width,
ymag=0.001 * 0.5 * height,
znear=0.01,
zfar=10)
camera_pose = np.eye(4)
camera_pose[:3,
3] = np.array([0.001 * 0.5 * width, 0.001 * 0.5 * height, 1.0])
scene.add(camera, pose=camera_pose)
light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=1.0)
light_pose = np.eye(4)
light_pose[:3, 3] = np.array([1.0, 1.0, 1.0])
scene.add(light, pose=light_pose.copy())
light_pose[:3, 3] = np.array([0.0, 1.0, 1.0])
scene.add(light, pose=light_pose.copy())
try:
r = pyrender.OffscreenRenderer(viewport_width=width,
viewport_height=height)
color, _ = r.render(scene)
except:
print('Rendering failed')
color = np.zeros((height, width, 3))
return color[..., ::-1].copy()
def render4mesh(self, idx, ratio=1):
#the ratio=10 can make the rendered image be black
vertices4view = self.get4viewManovertices(idx)
import trimesh
import pyrender
from pyrender import RenderFlags
np_rot_x = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]],
dtype=np.float32)
np_rot_x = np.reshape(np.tile(np_rot_x, [1, 1]), [3, 3])
recolorlist = []
for iv in range(4):
xyz = vertices4view[iv, :778, :3, 0].copy()
cv = xyz @ np_rot_x
tmesh = trimesh.Trimesh(vertices=cv / 1000 * ratio,
faces=self.mano_right.faces)
# tmesh.visual.vertex_colors = [.9, .7, .7, 1]
# tmesh.show()
mesh = pyrender.Mesh.from_trimesh(tmesh)
scene = pyrender.Scene()
scene.add(mesh)
pycamera = pyrender.IntrinsicsCamera(self.camera[iv].fx,
self.camera[iv].fy,
self.camera[iv].cx,
self.camera[iv].cy,
znear=0.0001,
zfar=3000)
ccamera_pose = self.camera_pose[self.rootcameraidx]
scene.add(pycamera, pose=ccamera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=2.0,
innerConeAngle=np.pi / 16.0)
# light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=2.0)
scene.add(light, pose=ccamera_pose)
r = pyrender.OffscreenRenderer(640, 480)
# flags = RenderFlags.SHADOWS_DIRECTIONAL
recolor, depth = r.render(scene)
# cv2.imshow("depth", depth)
recolorlist.append(recolor[:, :, :3])
meshcolor = np.hstack(recolorlist)
return meshcolor
def main(leap_path, smpl_param_file, bm_path, device):
# load SMPL parameters
smpl_body = torch.load(smpl_param_file, map_location=torch.device('cpu'))
smpl_body = {
key: val.to(device=device) if torch.is_tensor(val) else val
for key, val in smpl_body.items()
}
# load LEAP
leap_model = LEAPBodyModel(leap_path,
bm_path=os.path.join(bm_path,
smpl_body['gender'],
'model.npz'),
num_betas=smpl_body['betas'].shape[1],
batch_size=smpl_body['betas'].shape[0],
device=device)
leap_model.set_parameters(betas=smpl_body['betas'],
pose_body=smpl_body['pose_body'],
pose_hand=smpl_body['pose_hand'])
leap_model.forward_parametric_model()
# uniform points
np_query_points, tensor_query_points = sample_points(leap_model.posed_vert)
occupancy = leap_model(tensor_query_points) < 0.5
inside_points = (occupancy <
0.5).squeeze().detach().cpu().numpy() # 0.5 is threshold
posed_mesh = leap_model.extract_posed_mesh()
# can_mesh = leap_model.extract_canonical_mesh() # mesh in canonical pose
# visualize
scene = pyrender.Scene(ambient_light=[.1, 0.1, 0.1],
bg_color=[1.0, 1.0, 1.0])
scene.add(pyrender.Mesh.from_trimesh(posed_mesh))
scene.add(vis_create_pc(np_query_points[inside_points],
color=(1., 0., 0.))) # red - inside points
scene.add(
vis_create_pc(np_query_points[~inside_points],
color=(0., 1., 0.))) # blue - outside points
pyrender.Viewer(scene, use_raymond_lighting=True, run_in_thread=False)
def show_urdf_transform_manager(tm,
frame,
collision_objects=False,
visuals=False,
frames=False,
s=1.0):
"""Render URDF file with pyrender.
Parameters
----------
tm : UrdfTransformManager
Transformation manager
frame : str
Base frame for rendering
collision_objects : bool, optional (default: False)
Render collision objects
visuals : bool, optional (default: False)
Render visuals
frames : bool, optional (default: False)
Render frames
s : float, optional (default: 1)
Axis scale
"""
scene = pr.Scene()
if collision_objects:
if hasattr(tm, "collision_objects"):
_add_objects(scene, tm, tm.collision_objects, frame)
if visuals:
if hasattr(tm, "visuals"):
_add_objects(scene, tm, tm.visuals, frame)
if frames:
for node in tm.nodes:
_add_frame(scene, tm, node, frame, s)
pr.Viewer(scene, use_raymond_lighting=True)
def load_dynamic_contour(template_flame_path='None', contour_embeddings_path='None', angle=0):
template_mesh = Mesh(filename=template_flame_path)
contour_embeddings_path = contour_embeddings_path
dynamic_lmks_embeddings = np.load(contour_embeddings_path, allow_pickle=True).item()
lmk_face_idx = dynamic_lmks_embeddings['lmk_face_idx'][angle]
lmk_b_coords = dynamic_lmks_embeddings['lmk_b_coords'][angle]
dynamic_lmks = mesh_points_by_barycentric_coordinates(template_mesh.v, template_mesh.f, lmk_face_idx, lmk_b_coords)
# Visualization of the pose dependent contour on the template mesh
vertex_colors = np.ones([template_mesh.v.shape[0], 4]) * [0.3, 0.3, 0.3, 0.8]
tri_mesh = trimesh.Trimesh(template_mesh.v, template_mesh.f,
vertex_colors=vertex_colors)
mesh = pyrender.Mesh.from_trimesh(tri_mesh)
scene = pyrender.Scene()
scene.add(mesh)
sm = trimesh.creation.uv_sphere(radius=0.005)
sm.visual.vertex_colors = [0.9, 0.1, 0.1, 1.0]
tfs = np.tile(np.eye(4), (len(dynamic_lmks), 1, 1))
tfs[:, :3, 3] = dynamic_lmks
joints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs)
scene.add(joints_pcl)
pyrender.Viewer(scene, use_raymond_lighting=True)
def _render_joint(self):
"""Renders and saves hand joint visualizations."""
print('Rendering joint')
for i in range(self._loader.num_frames):
print('{:03d}/{:03d}'.format(i + 1, self._loader.num_frames))
self._loader.step()
for c in range(self._loader.num_cameras):
# Create pyrender scene.
scene = pyrender.Scene(bg_color=np.array([0.0, 0.0, 0.0, 0.0]),
ambient_light=np.array([1.0, 1.0, 1.0]))
# Add camera.
scene.add(self._cameras[c], pose=np.eye(4))
joint_3d = self._loader.mano_joint_3d[c]
# Add MANO joints.
for o in range(self._loader.num_mano):
if np.all(joint_3d[o] == -1):
continue
j = joint_3d[o].copy()
j[:, 1] *= -1
j[:, 2] *= -1
tfs = np.tile(np.eye(4), (21, 1, 1))
tfs[:, :3, 3] = j
mesh = pyrender.Mesh.from_trimesh(self._mesh_j[o],
poses=tfs)
scene.add(mesh)
color, _ = self._r.render(scene)
im = self._loader.pcd_rgb[c]
color = self._blend(im, color)
color_file = self._render_dir[c] + "/joints_{:06d}.jpg".format(
i)
cv2.imwrite(color_file, color[:, :, ::-1])
def main():
dataset = get_dataset('s0_train')
# Load pre-generated grasps for YCB objects.
ycb_grasp_file = os.path.join(os.path.dirname(__file__), "..", "assets",
"ycb_farthest_100_grasps.json")
with open(ycb_grasp_file, 'r') as f:
ycb_grasps = json.load(f)
# Load simplified panda gripper mesh.
gripper_mesh_file = os.path.join(os.path.dirname(__file__), "..", "assets",
"panda_tubes.obj")
gripper_mesh = trimesh.load(gripper_mesh_file)
gripper_material = pyrender.MetallicRoughnessMaterial(
alphaMode="BLEND",
doubleSided=True,
baseColorFactor=(0.00, 1.00, 0.04, 1.00),
metallicFactor=0.0)
# Visualize pre-generated grasps for each YCB object.
for ycb_id, name in dataset.ycb_classes.items():
if name not in ycb_grasps.keys():
print('{} does not have pre-generated grasps: skip.'.format(name))
continue
scene = pyrender.Scene(ambient_light=np.array([0.5, 0.5, 0.5, 1.0]))
obj_mesh = trimesh.load(dataset.obj_file[ycb_id])
scene.add(pyrender.Mesh.from_trimesh(obj_mesh))
for grasp in ycb_grasps[name]:
grasp_mesh = copy.deepcopy(gripper_mesh).apply_transform(grasp),
scene.add(pyrender.Mesh.from_trimesh(grasp_mesh, gripper_material))
print('Visualizing pre-generated grasps of {}'.format(name))
print('Close the window to visualize next object.')
pyrender.Viewer(scene, use_raymond_lighting=True)
def __init__(self, ply_path, recon_path):
self.reconstruction = Reconstruction(recon_path)
self.ply_data = PlyData.read(ply_path)
self.vertices = self.ply_data.elements[0]
self.faces = self.ply_data.elements[1]
self.tmesh = trimesh.load(ply_path)
# Transform vertices from UTM to ENU.
vertices_enu = self.reconstruction.utm_to_enu(self.tmesh.vertices)
print 'tmesh.vertices_enu:', vertices_enu[0:2, :]
self.tmesh.vertices = vertices_enu
# self.tmesh.export('./mesh_enu.ply')
# Recolor the facets.
num_facets = self.tmesh.facets.size
print 'number of facets:', num_facets
facet_index = long(0)
# TODO(snavely): Why are some facets showing up as gray? Are
# they somehow facing the wrong direction? Do those faces not
# show up in the list of facets?
for facet in self.tmesh.facets:
# Random trimesh colors have random hue but nearly full
# saturation and value. Useful for visualization and
# debugging.
# tmesh.visual.face_colors[facet] = trimesh.visual.random_color()
# self.rectifying_homography(None, facet_index, 100)
r, g, b = self.color_index_to_color(facet_index + 1)
# Last 255 is for alpha channel (fully opaque).
self.tmesh.visual.face_colors[facet] = np.array((r, g, b, 255))
facet_index = facet_index + 1
self.mesh = pyrender.Mesh.from_trimesh(self.tmesh, smooth=False)
self.scene = pyrender.Scene(ambient_light=(1.0, 1.0, 1.0))
self.scene.add(self.mesh)
self.ply_textured = None
def create_scene(self):
self.scene = pyrender.Scene()
# load table
table_trimesh = trimesh.load(self.table_mesh_path)
table_mesh = pyrender.Mesh.from_trimesh(table_trimesh)
self.scene.add(table_mesh)
# setup camera
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
self.camera_pose = np.array([
[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, self.dist],
[0, 0, 0, 1]
])
nc = pyrender.Node(camera=camera, matrix=self.camera_pose)
self.scene.add_node(nc)
# get camera projection matrix
self.proj_matrix = camera.get_projection_matrix(self.img_w, self.img_h)
# add light
light = pyrender.SpotLight(color=np.ones(3), intensity=1.0,
innerConeAngle=np.pi/16.0,
outerConeAngle=np.pi/6.0)
light_pose = np.array([
[1, 0, 0, 0.2],
[0, 1, 0, 0.2],
[0, 0, 1, 1],
[0, 0, 0, 1]
])
self.scene.add(light, pose=light_pose)
# load mesh
mesh_file = os.path.join(self.mesh_path, self.mesh)
fuze_trimesh = trimesh.load(mesh_file)
mesh_file = pyrender.Mesh.from_trimesh(fuze_trimesh)
self.scene.add(mesh_file, pose=self.obj_pose)
def main(model_folder, model_type='smplx', ext='npz',
gender='neutral', plot_joints=False,
use_face_contour=False):
model = smplx.create(model_folder, model_type=model_type,
gender=gender, use_face_contour=use_face_contour,
ext=ext)
print(model)
betas = torch.randn([1, 10], dtype=torch.float32)
expression = torch.randn([1, 10], dtype=torch.float32)
output = model(betas=betas, expression=expression,
return_verts=True)
vertices = output.vertices.detach().cpu().numpy().squeeze()
joints = output.joints.detach().cpu().numpy().squeeze()
print('Vertices shape =', vertices.shape)
print('Joints shape =', joints.shape)
vertex_colors = np.ones([vertices.shape[0], 4]) * [0.3, 0.3, 0.3, 0.8]
tri_mesh = trimesh.Trimesh(vertices, model.faces,
vertex_colors=vertex_colors)
mesh = pyrender.Mesh.from_trimesh(tri_mesh)
scene = pyrender.Scene()
scene.add(mesh)
if plot_joints:
sm = trimesh.creation.uv_sphere(radius=0.005)
sm.visual.vertex_colors = [0.9, 0.1, 0.1, 1.0]
tfs = np.tile(np.eye(4), (len(joints), 1, 1))
tfs[:, :3, 3] = joints
joints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs)
scene.add(joints_pcl)
pyrender.Viewer(scene, use_raymond_lighting=True)
def render_example():
# generate mesh
sphere = trimesh.creation.icosphere(subdivisions=4, radius=0.8)
sphere.vertices += 1e-2 * np.random.randn(*sphere.vertices.shape)
mesh = pyrender.Mesh.from_trimesh(sphere, smooth=False)
# compose scene
scene = pyrender.Scene(ambient_light=[.1, .1, .3], bg_color=[0, 0, 0])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
light = pyrender.DirectionalLight(color=[1, 1, 1], intensity=2e3)
scene.add(mesh, pose=np.eye(4))
scene.add(light, pose=np.eye(4))
c = 2**-0.5
scene.add(camera,
pose=[[1, 0, 0, 0], [0, c, -c, -2], [0, c, c, 2], [0, 0, 0, 1]])
# render scene
r = pyrender.OffscreenRenderer(512, 512)
color, _ = r.render(scene)
plt.figure(figsize=(8, 8)), plt.imshow(color)
def render(obj, cam_pos):
# create scene
scene = pyrender.Scene()
# MESHES
obj_trimesh = trimesh.load(obj)
obj_trimesh.vertices = scaleVertices(obj_trimesh.vertices,
feature_range=(-1, 1))
mesh = pyrender.Mesh.from_trimesh(obj_trimesh)
nm = pyrender.Node(mesh=mesh, matrix=np.eye(4))
scene.add_node(nm)
# CAMERA
cam = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0)
nc = pyrender.Node(camera=cam, matrix=cam_pos)
scene.add_node(nc)
# RENDER
flags = RenderFlags.DEPTH_ONLY
r = pyrender.OffscreenRenderer(400, 400)
depth = r.render(scene, flags=flags)
return (depth)
