def save_image_face(facedata, vec, name="face", path=""):
plt.clf()
plt.close()
# Generate mesh
predict_trimeshh = facedata.vec2meshTrimesh2(vec)
trimeshh = pyrender.Mesh.from_trimesh(predict_trimeshh, smooth=False)
# Create scene for rendering
scene = pyrender.Scene(ambient_light=[.1, .1, .3], bg_color=[0, 0, 0])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
light = pyrender.DirectionalLight(color=[1, 1, 1], intensity=2e3)
scene.add(trimeshh, pose=np.eye(4))
scene.add(light, pose=np.eye(4))
camera_pose = np.array([
[0.94063, 0.01737, -0.41513, -88.15790],
[-0.06728, 0.98841, -0.16663, -35.36127],
[0.33266, 0.15078, 1.14014, 241.71166],
[0.00000, 0.00000, 0.00000, 1.00000]
])
scene.add(camera, pose=camera_pose)
# Use this in order to visualize the mesh live:
# view = pyrender.Viewer(scene)
# cam = view.get_my_camera_node_viewer() # Allow access to procted attributes
# print(view.scene.get_pose(scene.main_camera_node))
r = pyrender.OffscreenRenderer(512, 512)
color, _ = r.render(scene)
plt.figure(figsize=(8, 8))
plt.imshow(color)
img_name = path + name + ".png"
plt.savefig(img_name)
plt.clf()
def _init_camera_renderer(self):
"""
If not in visualizing mode, initialize camera with given intrinsics
"""
if self._viewer:
return
if self._intrinsics in ['kinect_azure', 'realsense']:
camera = pyrender.IntrinsicsCamera(self._fx, self._fy, self._cx,
self._cy, self._znear,
self._zfar)
self._camera_node = self._scene.add(camera,
pose=np.eye(4),
name='camera')
self.renderer = pyrender.OffscreenRenderer(
viewport_width=self._width,
viewport_height=self._height,
point_size=1.0)
else:
camera = pyrender.PerspectiveCamera(
yfov=self._fov, aspectRatio=1.0,
znear=0.001) # do not change aspect ratio
self._camera_node = self._scene.add(camera,
pose=tra.euler_matrix(
np.pi, 0, 0),
name='camera')
self.renderer = pyrender.OffscreenRenderer(400, 400)
def render_depth(path, override=None):
# Load the FUZE bottle trimesh and put it in a scene
fuze_trimesh = trimesh.load(path)
fuze_trimesh.vertices = (
fuze_trimesh.vertices - np.amin(fuze_trimesh.vertices, axis=0)) / (
np.amax(fuze_trimesh.vertices, axis=0) -
np.amin(fuze_trimesh.vertices, axis=0) + 0.001) - 0.5
mesh = pyrender.Mesh.from_trimesh(fuze_trimesh)
scene = pyrender.Scene()
scene.add(mesh)
# Set up the camera -- z-axis away from the scene, x-axis right, y-axis up
camera = pyrender.PerspectiveCamera(yfov=np.pi / 5.0, znear=0.15)
camera_pose = getCameraPose(override)
scene.add(camera, pose=camera_pose)
# Set up the light -- a single spot light in the same spot as the camera
light = pyrender.SpotLight(color=np.ones(3),
intensity=3.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_pose)
# Render the scene
r = pyrender.OffscreenRenderer(200, 200)
depth = r.render(scene, flags=pyrender.RenderFlags.DEPTH_ONLY)
depth = depth / (np.max(depth) + 0.0001)
return depth
def mesh_to_png(file_name,
mesh,
width=640,
height=480,
z_camera_translation=400):
mesh = trimesh.base.Trimesh(vertices=mesh.vertices,
faces=mesh.triangles,
vertex_colors=mesh.colors)
mesh = pyrender.Mesh.from_trimesh(mesh, smooth=True, wireframe=False)
# compose scene
scene = pyrender.Scene(ambient_light=np.array([1.7, 1.7, 1.7, 1.0]),
bg_color=[255, 255, 255])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
light = pyrender.DirectionalLight(color=[1, 1, 1], intensity=2e3)
scene.add(mesh, pose=np.eye(4))
scene.add(light, pose=np.eye(4))
# Added camera translated z_camera_translation in the 0z direction w.r.t. the origin
scene.add(camera,
pose=[[1, 0, 0, 0], [0, 1, 0, 0],
[0, 0, 1, z_camera_translation], [0, 0, 0, 1]])
# render scene
r = pyrender.OffscreenRenderer(width, height)
color, _ = r.render(scene)
imsave(file_name, color)
def render_pred_coeff(pred_coeff):
pred_coeff = np.asarray(pred_coeff).tolist()
# make scene
scene = pyrender.Scene()
# initialize camera
f_len = 366.474487
rend_size = 256.
camera = pyrender.PerspectiveCamera(yfov=np.arctan(rend_size*0.5/f_len)*2, aspectRatio=1)
camera_pose = np.array([[1.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0],
[0.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 1.0]])
scene.add(camera, pose=camera_pose)
# initialize light
light_posi1 = np.array([[1.0, 0.0, 0.0, 1.0],
[0.0, 0.0, 1.0, -1.0],
[0.0, 1.0, 0.0, -2.0],
[0.0, 0.0, 0.0, 1.0]])
light_posi2 = np.array([[1.0, 0.0, 0.0, -1.0],
[0.0, 0.0, 1.0, -1.0],
[0.0, 1.0, 0.0, -2.0],
[0.0, 0.0, 0.0, 1.0]])
light_posi3 = np.array([[1.0, 0.0, 0.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 1.0, 0.0, -2.0],
[0.0, 0.0, 0.0, 1.0]])
light = pyrender.SpotLight(color=np.array([0.65098039, 0.74117647, 0.85882353]),
intensity=100,
innerConeAngle=np.pi/16.0,
outerConeAngle=np.pi/6.0)
scene.add(light, pose=light_posi1)
scene.add(light, pose=light_posi2)
scene.add(light, pose=light_posi3)
# get renderer
r = pyrender.OffscreenRenderer(viewport_width=rend_size, viewport_height=rend_size)
# get verts from smpl coefficients
smpl_op = load_model("./tf_smpl/neutral_smpl_with_cocoplus_reg.pkl")
verts_bias = pred_coeff[3:6]
smpl_op.pose[:] = np.asarray([0]*3 + pred_coeff[6:75])
smpl_op.betas[:] = np.array(pred_coeff[75:85])
verts = np.array(smpl_op)
rot_mat = cv2.Rodrigues(np.asarray(pred_coeff[3:6]))[0]
verts = np.tensordot(verts, rot_mat, axes=([1],[1]))
verts = verts + pred_coeff[:3]
# make trimesh
faces = np.load("./tf_smpl/smpl_faces.npy").astype(np.int32)
this_trimesh = trimesh.Trimesh(vertices = verts, faces = faces)
this_mesh = pyrender.Mesh.from_trimesh(this_trimesh)
scene.add(this_mesh)
rend_img, _ = r.render(scene)
return rend_img
def _create_scene_and_offscreen_render():
"""We will add (and remove) the meshes later.
Unfortunately this requires some tuning of the position and rotation.
"""
scene = pyrender.Scene()
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
# The position / translation [px, py, pz] and then rotation matrix.
p = [0.45, -0.45, 0.90]
theta = -45 * DEG_TO_RAD
RX = np.array([
[1.0, 0.0, 0.0],
[0.0, np.cos(theta), np.sin(theta)],
[0.0, -np.sin(theta), np.cos(theta)],
])
R = RX
camera_pose = np.array([
[R[0,0], R[0,1], R[0,2], p[0]],
[R[1,0], R[1,1], R[1,2], p[1]],
[R[2,0], R[2,1], R[2,2], p[2]],
[ 0.0, 0.0, 0.0, 1.0],
])
scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(2), intensity=1.0,
innerConeAngle=np.pi/16.0,
outerConeAngle=np.pi/6.0)
scene.add(light, pose=camera_pose)
# Only for debugging
#v = pyrender.Viewer(scene, use_raymond_lighting=True)
rend = pyrender.OffscreenRenderer(640, 480)
return scene, rend
def __init__(self, width=1200, height=800, use_offscreen=True):
super(MeshViewer, self).__init__()
self.use_offscreen = use_offscreen
self.render_wireframe = False
self.mat_constructor = pyrender.MetallicRoughnessMaterial
self.trimesh_to_pymesh = pyrender.Mesh.from_trimesh
self.scene = pyrender.Scene(bg_color=colors['white'],
ambient_light=(0.3, 0.3, 0.3))
pc = pyrender.PerspectiveCamera(yfov=np.pi / 3.0,
aspectRatio=float(width) / height)
camera_pose = np.eye(4)
camera_pose[:3, 3] = np.array([0, 0, 2.5])
self.scene.add(pc, pose=camera_pose, name='pc-camera')
self.figsize = (width, height)
if self.use_offscreen:
self.viewer = pyrender.OffscreenRenderer(*self.figsize)
self.use_raymond_lighting(5.)
else:
self.viewer = pyrender.Viewer(self.scene,
use_raymond_lighting=True,
viewport_size=self.figsize,
cull_faces=False,
run_in_thread=True)
self.set_background_color(colors['white'])
def render(obj_path, camera_mat, return_depth=False, im_size=128):
fuze_trimesh = trimesh.load(obj_path)
if type(fuze_trimesh) == trimesh.base.Trimesh:
m = pyrender.Mesh.from_trimesh(fuze_trimesh)
scene = pyrender.Scene()
scene.add_node(pyrender.Node(mesh=m))
else:
assert type(
fuze_trimesh) == trimesh.scene.scene.Scene, "Unrognized file"
scene = pyrender.Scene.from_trimesh_scene(fuze_trimesh)
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
s = np.sqrt(2) / 2
scene.add(camera, pose=camera_mat)
light = pyrender.SpotLight(color=np.ones(3),
intensity=4.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_mat)
light = pyrender.SpotLight(color=np.ones(3),
intensity=6.0,
innerConeAngle=0.2 * np.pi)
light_pose = np.array(
[[0, 1, 0, 0], [0, 0, 1, 1], [1, 0, 0, 0], [0, 0, 0, 1]],
dtype=np.float32)
scene.add(light, pose=light_pose)
r = pyrender.OffscreenRenderer(im_size, im_size)
color, depth = r.render(scene)
r.delete()
if return_depth:
return color, depth
return color
def captureDepth(model, rotations, imageWidth=224, imageHeight=224, cameraZTranslation=2.5, lightIntensity=2.0, depthBegin=1, depthEnd=5):
# Construct an offline scene
scene = pyrender.Scene()
# add parts
for part in model.parts:
partMesh = pyrender.Mesh.from_trimesh(part.mesh, smooth=False)
scene.add(partMesh)
# add camera
renderCamera = pyrender.PerspectiveCamera(
yfov=np.pi / 3.0, aspectRatio=imageWidth/imageHeight)
cameraNode = pyrender.Node(camera=renderCamera, matrix=np.eye(4))
cameraNode.translation[2] = cameraZTranslation
scene.add_node(cameraNode)
# add light
light = pyrender.DirectionalLight(
color=[1.0, 1.0, 1.0], intensity=lightIntensity)
lightNode = pyrender.Node(light=light, matrix=np.eye(4))
scene.add_node(lightNode)
# initialize offscreen renderer
offscreenRenderer = pyrender.OffscreenRenderer(
viewport_width=imageWidth, viewport_height=imageHeight, point_size=1.0)
# render
result = []
for rotation in rotations:
result.append(renderSceneWithRotation(offscreenRenderer,
scene, rotation, depthBegin, depthEnd))
return result
def __init__(
self,
pyrender_scene,
fov=np.pi / 6.0,
width=400,
height=400,
aspect_ratio=1.0,
z_near=0.001,
):
"""Create an image renderer for a scene.
Args:
pyrender_scene (pyrender.Scene): Scene description including object meshes and their poses.
fov (float, optional): Field of view of camera. Defaults to np.pi/6.
width (int, optional): Width of camera sensor (in pixels). Defaults to 400.
height (int, optional): Height of camera sensor (in pixels). Defaults to 400.
aspect_ratio (float, optional): Aspect ratio of camera sensor. Defaults to 1.0.
z_near (float, optional): Near plane closer to which nothing is rendered. Defaults to 0.001.
"""
self._fov = fov
self._width = width
self._height = height
self._z_near = z_near
self._scene = pyrender_scene
self._camera = pyrender.PerspectiveCamera(yfov=fov,
aspectRatio=aspect_ratio,
znear=z_near)
def _init_scene(self):
self._scene = pyrender.Scene()
camera = pyrender.PerspectiveCamera(
yfov=self._fov, aspectRatio=1.0,
znear=0.001) # do not change aspect ratio
theta, phi = 0, np.pi / 4 + np.random.rand() * (np.pi / 8)
radius = 3 + (np.random.rand() - 1)
x = radius * np.sin(phi) * np.cos(theta)
y = radius * np.sin(phi) * np.sin(theta)
z = radius * np.cos(phi)
camera_pos = np.array([x, y, z])
#camera_pos = np.array([0,0,10])
#light_pos = np.array([0,0,10])
camera_pose = tra.euler_matrix(0, phi, 0)
camera_pose[:3, 3] = camera_pos
camera_pose[:3, :3] = camera_pose[:3, :3]
self._scene.add(camera, pose=camera_pose, name='camera')
self.camera_pose = camera_pose
light = pyrender.SpotLight(color=np.ones(4),
intensity=50.,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6.0)
self._scene.add(light, pose=camera_pose, name='light')
def _render_pyrender(self, T_camera2world, fovy, height, width):
# FIXME: pyrender and pybullet images are not perfectly aligned
raise NotImplementedError
import pyrender
scene = self.scene
node_camera = scene.add(
obj=pyrender.PerspectiveCamera(yfov=np.deg2rad(fovy),
aspectRatio=width / height),
pose=morefusion.extra.trimesh.to_opengl_transform(T_camera2world),
)
for _ in range(4):
direction = self._random_state.uniform(-1, 1, (3, ))
direction /= np.linalg.norm(direction)
scene.add(
obj=pyrender.DirectionalLight(
intensity=self._random_state.uniform(0.5, 5), ),
pose=trimesh.transformations.rotation_matrix(
angle=np.deg2rad(self._random_state.uniform(0, 45)),
direction=direction,
),
parent_node=node_camera,
)
renderer = pyrender.OffscreenRenderer(viewport_width=width,
viewport_height=height)
rgb, depth = renderer.render(scene)
scene.remove_node(node_camera)
return rgb, depth
def __init__(self, size=(512, 512), config=None):
config = {
'ambient_light': (0.1, 0.1, 0.1),
'bg_color': (255, 255, 255),
} if config is None else config
self.renderer = pyrender.OffscreenRenderer(*size)
self.scene = pyrender.Scene(bg_color=config['bg_color'], ambient_light=config['ambient_light'])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=size[0] / size[1], znear=0.01, zfar=10)
self.camera_node = self.scene.add(camera, pose=self.make_camera_pose())
self.dir_light_nodes = []
n_lights = 10
for i in range(n_lights):
dir_light = pyrender.DirectionalLight(color=np.ones(3), intensity=10.0/n_lights)
dir_light_pose = np.eye(4)
dir_light_pose[:3,:3] = Rotation.from_euler(
'zyx',
[0, i * 360 / n_lights, -30],
degrees=True
).as_matrix()
dir_light_node = self.scene.add(dir_light, pose=dir_light_pose)
self.dir_light_nodes.append(dir_light_node)
self.persistent = [self.camera_node] + self.dir_light_nodes
def __init__(self,
width=1200,
height=800,
body_color=(1.0, 1.0, 0.9, 1.0),
registered_keys=None):
super(MeshViewer, self).__init__()
if registered_keys is None:
registered_keys = dict()
self.mat_constructor = pyrender.MetallicRoughnessMaterial
self.mesh_constructor = trimesh.Trimesh
self.trimesh_to_pymesh = pyrender.Mesh.from_trimesh
self.transf = trimesh.transformations.rotation_matrix
self.body_color = body_color
self.scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0, 1.0],
ambient_light=(0.3, 0.3, 0.3))
pc = pyrender.PerspectiveCamera(yfov=np.pi / 3.0,
aspectRatio=float(width) / height)
camera_pose = np.eye(4)
camera_pose[:3, 3] = np.array([0, 0, 3])
self.scene.add(pc, pose=camera_pose)
self.viewer = pyrender.Viewer(self.scene,
use_raymond_lighting=True,
viewport_size=(width, height),
cull_faces=False,
run_in_thread=True,
registered_keys=registered_keys)
def render_mesh(self, mesh, width, height):
# for i in range(20):
scene = pyrender.Scene()
#Adding the mesh into the scene
scene.add(pyrender.Mesh.from_trimesh(mesh))
#Adding the camera into the scene
camera = pyrender.PerspectiveCamera(yfov=self.camera_fov,
aspectRatio=self.aspect_ratio)
pitch_cos = np.cos(self.camera_rot[0])
pitch_sin = np.sin(self.camera_rot[0])
roll_cos = np.cos(self.camera_rot[1])
roll_sin = np.sin(self.camera_rot[1])
yaw_cos = np.cos(self.camera_rot[2])
yaw_sin = np.sin(self.camera_rot[2])
pitch = np.mat([[1.0, 0.0, 0.0, 0.0],
[0.0, pitch_cos, -pitch_sin, 0.0],
[0.0, pitch_sin, pitch_cos, 0.0], [0.0, 0.0, 0.0,
1.0]])
roll = np.mat([[roll_cos, 0.0, roll_sin, 0.0], [0.0, 1.0, 0.0, 0.0],
[-roll_sin, 0.0, roll_cos, 0.0], [0.0, 0.0, 0.0, 1.0]])
yaw = np.mat([[yaw_cos, -yaw_sin, 0.0, 0.0],
[yaw_sin, yaw_cos, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0],
[0.0, 0.0, 0.0, 1.0]])
camera_transform = np.mat([[1.0, 0.0, 0.0, self.camera_pos[0]],
[0.0, 1.0, 0.0, self.camera_pos[1]],
[0.0, 0.0, 1.0, self.camera_pos[2]],
[0.0, 0.0, 0.0, 1.0]])
s = np.sqrt(2) / 2
camera_pose = np.mat([
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0],
[0.0, 0.0, 0.0, 1.0],
])
#Multiply yaw*roll*pitch
camera_rotation = yaw * roll * pitch
camera_pose = camera_transform * camera_rotation * camera_pose #*camera_z_rot*camera_y_rot*camera_x_rot
scene.add(camera, pose=np.array(camera_pose))
#Adding lights into the scene
for light in self.lights:
scene.add(light[0], pose=light[1])
#Render the scene and return the resultant image
r = pyrender.OffscreenRenderer(width, height)
color, depth = r.render(scene)
# io.imsave(str(i)+"_test.png", color)
return color
def model_to_projection(model):
""" 通过相机模型得到OpenGL的ndc无限投影矩阵
model: 相机模型, [yfov, znear, aspectRatio], [可视角度, 焦距(近平面距离), 长宽比例]
"""
camera_ = pyrender.PerspectiveCamera(
yfov=model[0], znear=model[1], aspectRatio=model[2])
return camera_.get_projection_matrix()
def generate_thumbnail(inname, outname, light_color=[1.0, 1.0, 1.0]):
"""Generate a thumbnail image of a GLB mesh.
"""
print("Loading", inname)
tmesh_obj = trimesh.load(inname)
mesh = glbutils.getSceneMesh(tmesh_obj)
corners = bound_corners(mesh.bounds)
tm_scene = trimesh.scene.scene.Scene(geometry=[mesh])
cam = tm_scene.camera
cam_transform = cam.look_at(corners)
pyren_mesh = pyrender.Mesh.from_trimesh(mesh)
pyren_scene = pyrender.Scene()
pyren_scene.add(pyren_mesh)
pyren_cam = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
pyren_scene.add(pyren_cam, pose=cam_transform)
pyren_light = pyrender.DirectionalLight(color=np.asarray(light_color),
intensity=3.0)
pyren_scene.add(pyren_light)
r = pyrender.OffscreenRenderer(400, 400)
color, depth = r.render(pyren_scene)
plt.imsave(outname, color)
print(outname, "written")
def create_from_scans(mesh,
bounding_radius=1,
scan_count=100,
scan_resolution=400,
calculate_normals=True):
scans = []
camera = pyrender.PerspectiveCamera(yfov=1.0472,
aspectRatio=1.0,
znear=bounding_radius * 1,
zfar=bounding_radius * 3)
renderApp = RenderApp(scan_resolution, mesh, camera)
for phi, theta in get_equidistant_camera_angles(scan_count, 2):
camera_transform = get_camera_transform_looking_at_origin(
phi, theta, camera_distance=1.5 * bounding_radius)
scans.append(
Scan(render_app=renderApp,
camera=camera,
theta=theta,
camera_transform=camera_transform,
calculate_normals=calculate_normals))
return SurfacePointCloud(
mesh,
points=np.concatenate([scan.points for scan in scans], axis=0),
normals=np.concatenate([scan.normals for scan in scans], axis=0)
if calculate_normals else None,
edges=np.concatenate([scan.edges for scan in scans], axis=0)
if calculate_normals else None,
edges_sharp=np.concatenate([scan.edges_sharp for scan in scans],
axis=0) if calculate_normals else None,
scans=scans)
def render_hmr_smpl(hmr_coeff, f_len=500., rend_size=224., req_model=False):
# hmr_coeff is a 85-vector, named as theta in hmr
hmr_coeff = np.asarray(hmr_coeff).tolist()
# make scene
scene = pyrender.Scene()
# initialize camera
camera = pyrender.PerspectiveCamera(
yfov=np.arctan(rend_size * 0.5 / f_len) * 2, aspectRatio=1)
camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0],
[0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]])
scene.add(camera, pose=camera_pose)
# initialize light
light_posi1 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, -1.0],
[0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]])
light_posi2 = np.array([[1.0, 0.0, 0.0, -1.0], [0.0, 0.0, 1.0, -1.0],
[0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]])
light_posi3 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0],
[0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]])
light = pyrender.SpotLight(color=np.array(
[0.65098039, 0.74117647, 0.85882353]),
intensity=100,
innerConeAngle=np.pi / 16.0,
outerConeAngle=np.pi / 6.0)
scene.add(light, pose=light_posi1)
scene.add(light, pose=light_posi2)
scene.add(light, pose=light_posi3)
# get renderer
r = pyrender.OffscreenRenderer(viewport_width=rend_size,
viewport_height=rend_size)
# get verts from smpl coefficients
smpl_op = load_model("./tf_smpl/neutral_smpl_with_cocoplus_reg.pkl")
smpl_op.pose[:] = np.asarray(hmr_coeff[3:75])
smpl_op.betas[:] = np.array(hmr_coeff[75:85])
verts = np.array(smpl_op)
global_t = np.array(
[hmr_coeff[1], hmr_coeff[2], f_len / (0.5 * rend_size * hmr_coeff[0])])
verts = verts + global_t
faces = np.load("./tf_smpl/smpl_faces.npy").astype(np.int32)
# smooth and expand
om_mesh = make_trimesh(verts, faces)
om_mesh = smooth_mesh(om_mesh, 4)
om_mesh = expand_mesh(om_mesh, 0.026)
this_trimesh = trimesh.Trimesh(vertices=om_mesh.points(),
faces=om_mesh.face_vertex_indices())
this_mesh = pyrender.Mesh.from_trimesh(this_trimesh)
scene.add(this_mesh)
rend_img, depth = r.render(scene)
if req_model is True:
return rend_img, verts, faces, depth
else:
return rend_img
def show(self, q=None, obstacles=None):
cfg = self.get_config(q)
# print(cfg)
fk = self.robot.link_fk(cfg=cfg)
scene = pyrender.Scene()
# adding robot to the scene
for tm in fk:
pose = fk[tm]
init_pose, link_mesh = self.get_link_mesh(tm)
mesh = pyrender.Mesh.from_trimesh(link_mesh, smooth=False)
scene.add(mesh, pose=np.matmul(pose, init_pose))
# adding base box to the scene
# table = box([0.5, 0.5, 0.01])
# table.apply_translation([0, 0, -0.1])
cam = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.414)
# nc = pyrender.Node(camera=cam, matrix=np.eye(4))
# scene.add_node(nc)
init_cam_pose = np.eye(4)
init_cam_pose[2, 3] = 2.5
scene.add(cam, pose=init_cam_pose)
# scene.add(pyrender.Mesh.from_trimesh(table))
# adding obstacles to the scene
for ob in obstacles:
scene.add(pyrender.Mesh.from_trimesh(ob, smooth=False))
pyrender.Viewer(scene, use_raymond_lighting=True)
def rendering(R, fuze_trimesh):
mesh = pyrender.Mesh.from_trimesh(fuze_trimesh, poses=[R])
scene = pyrender.Scene()
scene.add(mesh)
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3, aspectRatio=1)
m1 = np.array([
[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
],
dtype='float')
m2 = np.array([
[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0.4],
[0, 0, 0, 1],
],
dtype='float')
camera_pose = m1.dot(m2)
scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=3.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_pose)
r = pyrender.OffscreenRenderer(400, 400)
color, depth = r.render(scene)
color = cv2.cvtColor(color, cv2.COLOR_BGR2RGB)
return color
def addCamera(self, projMat=None):
self.camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0,
aspectRatio=1.0)
if projMat is not None:
self.camera.projMatrix = projMat
# self.camera.set_projection_matrix(projMat)
self.scene.add(self.camera, pose=np.eye(4))
def _add_camera(scene):
'''Adds a camera to the scene'''
cam = pyrender.PerspectiveCamera( # Create perspective camera
yfov=np.pi / 3.0, znear=0.05, zfar=100)
cam = scene.add(cam) # Add camera to scene
return scene, cam
def __init__(self,
mesh,
rotation_y,
rotation_x,
bounding_radius=1,
resolution=400,
calculate_normals=True):
camera_distance = 2 * bounding_radius
self.camera_transform = get_camera_transform(
rotation_y, rotation_x, camera_distance=camera_distance)
self.camera_direction = np.matmul(self.camera_transform,
np.array([0, 0, 1, 0]))[:3]
self.camera_position = np.matmul(self.camera_transform,
np.array([0, 0, 0, 1]))[:3]
self.resolution = resolution
z_near = camera_distance - bounding_radius
z_far = camera_distance + bounding_radius
camera = pyrender.PerspectiveCamera(
yfov=2 * math.asin(bounding_radius / camera_distance),
aspectRatio=1.0,
znear=z_near,
zfar=z_far)
self.projection_matrix = camera.get_projection_matrix()
color, depth = render_normal_and_depth_buffers(mesh, camera,
self.camera_transform,
resolution)
indices = np.argwhere(depth != 0)
depth[depth == 0] = float('inf')
# This reverts the processing that pyrender does and calculates the original depth buffer in clipping space
self.depth = (z_far + z_near -
(2.0 * z_near * z_far) / depth) / (z_far - z_near)
points = np.ones((indices.shape[0], 4))
points[:, [1, 0]] = indices.astype(float) / (resolution - 1) * 2 - 1
points[:, 1] *= -1
points[:, 2] = self.depth[indices[:, 0], indices[:, 1]]
clipping_to_world = np.matmul(self.camera_transform,
np.linalg.inv(self.projection_matrix))
points = np.matmul(points, clipping_to_world.transpose())
points /= points[:, 3][:, np.newaxis]
self.points = points[:, :3]
if calculate_normals:
normals = color[indices[:, 0], indices[:, 1]] / 255 * 2 - 1
camera_to_points = self.camera_position - self.points
normal_orientation = np.einsum('ij,ij->i', camera_to_points,
normals)
normals[normal_orientation < 0] *= -1
self.normals = normals
else:
self.normals = None
def animate(self, q_traj=None, obstacles=None):
import time
fps = 10.0
cfgs = [self.get_config(q) for q in q_traj]
# Create the scene
fk = self.robot.visual_trimesh_fk(cfg=cfgs[0])
node_map = {}
init_pose_map = {}
scene = pyrender.Scene()
for tm in fk:
pose = fk[tm]
mesh = pyrender.Mesh.from_trimesh(tm, smooth=False)
node = scene.add(mesh, pose=pose)
node_map[tm] = node
# adding base box to the scene
#table = cylinder(radius=0.7, height=0.02) #([0.5, 0.5, 0.02])
#table.apply_translation([0, 0, -0.015])
#table.visual.vertex_colors = [205, 243, 8, 255]
#scene.add(pyrender.Mesh.from_trimesh(table))
cam = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.414)
init_cam_pose = np.eye(4)
init_cam_pose[2, 3] = 2.5
scene.add(cam, pose=init_cam_pose)
# adding obstacles to the scene
for i, ob in enumerate(obstacles):
if i < len(obstacles) - 1:
ob.visual.vertex_colors = [255, 0, 0, 255]
else:
ob.visual.vertex_colors = [205, 243, 8, 255]
scene.add(pyrender.Mesh.from_trimesh(ob, smooth=False))
# Pop the visualizer asynchronously
v = pyrender.Viewer(scene,
run_in_thread=True,
use_raymond_lighting=True)
time.sleep(1.0)
# Now, run our loop
i = 0
while v.is_active:
cfg = cfgs[i]
fk = self.robot.visual_trimesh_fk(cfg=cfg)
if i < len(cfgs) - 1:
i += 1
else:
i = 0
time.sleep(1.0)
v.render_lock.acquire()
for mesh in fk:
pose = fk[mesh]
node_map[mesh].matrix = pose
v.render_lock.release()
time.sleep(1.0 / fps)
def rendering(mat):
pcd = open3d.geometry.PointCloud()
pcd.points = open3d.utility.Vector3dVector(mat["verts"])
pcd.normals = open3d.utility.Vector3dVector(mat["normal_vec"])
# pcd.estimate_normals()
# estimate radius for rolling ball
distances = pcd.compute_nearest_neighbor_distance()
avg_dist = np.mean(distances)
radius = 1.5 * avg_dist
mesh = open3d.geometry.TriangleMesh.create_from_point_cloud_ball_pivoting(
pcd, open3d.utility.DoubleVector([radius, radius * 5])
)
# mesh, density = open3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd)
# create the triangular mesh with the vertices and faces from open3d
tri_mesh = trimesh.Trimesh(
np.asarray(mesh.vertices),
np.asarray(mesh.triangles),
vertex_normals=np.asarray(mesh.vertex_normals),
process=False,
)
# tri_mesh.export("test.stl")
# tri_mesh.convex.is_convex(tri_mesh)
mesh = pyrender.Mesh.from_trimesh(tri_mesh, smooth=False)
# mesh = pyrender.Mesh.from_points(pcd.points, normals=pcd.normals)
center = np.zeros(3)
# center = (mat["verts"].min(axis=0) + mat["verts"].max(axis=0)) / 2
# center[1] = mat["verts"][:, 1].min() + 10
center[1] = -5
center[2] = mat["verts"][:, 2].min()
# compose scene
scene = pyrender.Scene(ambient_light=[0.1, 0.1, 0.1], bg_color=[0.7, 0.7, 0.7])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 4.0)
# camera = pyrender.IntrinsicsCamera(1, 1, 0, 0)
# light = pyrender.DirectionalLight(color=[1, 1, 1], intensity=1)
light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=2.5e3)
scene.add(mesh, pose=np.eye(4))
scene.add(light, pose=[[1, 0, 0, center[0]], [0, 1, 0, center[1]], [0, 0, -1, center[2] - 20], [0, 0, 0, 1]])
scene.add(camera, pose=[[1, 0, 0, center[0]], [0, -1, 0, center[1]], [0, 0, -1, center[2] - 40], [0, 0, 0, 1]])
# render scene
# r = pyrender.OffscreenRenderer(800, 768)
# color, _ = r.render(scene)
# r.delete()
pyrender.Viewer(scene)
return color
def __init__(self, models, models_ui, parent=None, size=(640, 480)):
super().__init__(parent)
self.width = int(size[0])
self.height = int(size[1])
self.resizeGL(self.width, self.height)
self.models = models
self.models_ui = models_ui
self.timer = 0
timer = QTimer(self)
timer.timeout.connect(self.update)
timer.start(1000 / 60.0)
self.app = parent
self.dist = 12
self.angle = 0.0
self.lastpos = (-1, -1)
self.mouseWithin = False
self.scene = pyrender.Scene(bg_color=[0.2, 0.2, 0.2, 1])
self.camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0,
aspectRatio=1.0)
s = np.sqrt(2) / 2
camera_pose = np.array([
[0.0, -s, s, 1.3],
[1.0, 0.0, 0.0, 0.0],
[0.0, s, s, 1.35],
[0.0, 0.0, 0.0, 1.0],
])
self.scene.add(self.camera, pose=camera_pose)
self.side_light_1 = pyrender.PointLight(color=np.ones(3),
intensity=22,
name="side-light-1",
range=100)
self.side_light_2 = pyrender.PointLight(color=np.ones(3),
intensity=22,
name="side-light-2",
range=100)
self.scene.add(self.side_light_1, pose=np.array(translate((2, 1, 1))))
self.scene.add(self.side_light_2,
pose=np.array(translate((-1, 1, -2))))
self.offscreenRenderer = pyrender.OffscreenRenderer(
self.width, self.height)
self.color, depth = self.offscreenRenderer.render(self.scene)
def write_projection_pyrender(pfp, v, f, map): # TODO - Uncompleted
# RENDER_INFO = {'Height': 480, 'Width': 640, 'fx': 575, 'fy': 575, 'cx': 319.5, 'cy': 239.5}
mesh = pyrender.Mesh.from_points(v)
scene = pyrender.Scene()
scene.add(mesh)
scene.add(pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.414))
# pyrender.Viewer(scene, use_raymond_lighting=True)
r = pyrender.OffscreenRenderer(viewport_width=640,
viewport_height=480,
point_size=1.0)
color, depth = r.render(scene)
def show_meshes(meshes,
angle_x=-0.7854,
angle_y=0,
angle_z=0.31416,
ax=None,
resolution=(1200, 1200)):
if ax is None:
fig, ax = plt.subplots(1, 1)
scene = pyrender.Scene(
ambient_light=[0.0, 0.0, 0.0],
bg_color=[1.0, 1.0, 1.0],
)
for mesh in meshes:
mesh = mesh.copy()
re = trimesh.transformations.euler_matrix(angle_x, angle_y, angle_z,
'rxyz')
mesh.apply_transform(re)
scene.add(pyrender.Mesh.from_trimesh(mesh))
camera = pyrender.PerspectiveCamera(yfov=np.pi / 4.0, aspectRatio=1.0)
camera_pose = np.eye(4)
camera_pose[:3, 3] = [0, 0, 240]
scene.add(camera, pose=camera_pose)
ligth_poses = [
np.array([[-0.000, -0.866, 0.500, 0.], [1.000, -0.000, -0.000, 0.],
[0.000, 0.500, 0.866, 0.], [0.000, 0.000, 0.000, 1.]]),
np.array([[0.866, 0.433, -0.250, 0.], [-0.500, 0.750, -0.433, 0.],
[0.000, 0.500, 0.866, 0.], [0.000, 0.000, 0.000, 1.]]),
np.array([[-0.866, 0.433, -0.250, 0.], [-0.500, -0.750, 0.433, 0.],
[0.000, 0.500, 0.866, 0.], [0.000, 0.000, 0.000, 1.]])
]
for pose in ligth_poses:
light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=1.0)
scene.add(light, pose=pose)
r = pyrender.OffscreenRenderer(*resolution)
color, depth = r.render(scene)
if ax is None:
fig, ax = plt.subplots(1, 1, figsize=(10, 10))
ax.axis('off')
ind_ax0 = np.where(np.any(np.any(color != 255, axis=2), axis=1))[0]
ind_ax1 = np.where(np.any(np.any(color != 255, axis=2), axis=0))[0]
ax.imshow(color[ind_ax0[0]:(ind_ax0[-1] + 1),
ind_ax1[0]:(ind_ax1[-1] + 1), :])
return ax
def __init__(self,
render_scene=None,
viewport_size=None,
render_flags=None,
viewer_flags=None,
registered_keys=None,
run_in_thread=True,
video_filename=None,
**kwargs):
"""
Use render_scene to specify camera or lighting or additional geometries if required.
Additional viewer flags:
- axes_scale - size of coordinate axes
"""
if render_scene is None:
render_scene = pyrender.Scene()
render_scene.bg_color = np.array([0.75] * 3)
if render_scene.main_camera_node is None:
cam = pyrender.PerspectiveCamera(yfov=np.deg2rad(60),
aspectRatio=1.414,
znear=0.005)
cam_pose = np.eye(4)
cam_pose[:3, 3] = RoboticTrackball.spherical_to_cartesian(
3., 0., np.deg2rad(45.), target=np.zeros(3))
cam_pose[:3, :3] = RoboticTrackball.look_at_rotation(
eye=cam_pose[:3, 3], target=np.zeros(3), up=[0, 0, 1])
nc = pyrender.Node(camera=cam, matrix=cam_pose)
render_scene.add_node(nc)
render_scene.main_camera_node = nc
_viewer_flags = {
'view_center': np.zeros(3),
'window_title': 'PyPhysX Scene Viewer',
'show_world_axis': True,
'show_mesh_axes': False,
'axes_scale': 0.5,
'use_raymond_lighting': True,
'plane_grid_spacing': 1.,
'plane_grid_num_of_lines': 10,
'spheres_count': [8, 8],
}
if viewer_flags is not None:
_viewer_flags.update(viewer_flags)
super().__init__(render_scene, viewport_size, render_flags,
_viewer_flags, registered_keys, run_in_thread,
**kwargs)
self.nodes_and_actors = []
fps = self.viewer_flags['refresh_rate']
self.video_writer = imageio.get_writer(
video_filename, fps=fps) if video_filename is not None else None
