def render(obj_path, camera_mat, return_depth=False, im_size=128):
fuze_trimesh = trimesh.load(obj_path)
if type(fuze_trimesh) == trimesh.base.Trimesh:
m = pyrender.Mesh.from_trimesh(fuze_trimesh)
scene = pyrender.Scene()
scene.add_node(pyrender.Node(mesh=m))
else:
assert type(
fuze_trimesh) == trimesh.scene.scene.Scene, "Unrognized file"
scene = pyrender.Scene.from_trimesh_scene(fuze_trimesh)
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
s = np.sqrt(2) / 2
scene.add(camera, pose=camera_mat)
light = pyrender.SpotLight(color=np.ones(3),
intensity=4.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_mat)
light = pyrender.SpotLight(color=np.ones(3),
intensity=6.0,
innerConeAngle=0.2 * np.pi)
light_pose = np.array(
[[0, 1, 0, 0], [0, 0, 1, 1], [1, 0, 0, 0], [0, 0, 0, 1]],
dtype=np.float32)
scene.add(light, pose=light_pose)
r = pyrender.OffscreenRenderer(im_size, im_size)
color, depth = r.render(scene)
r.delete()
if return_depth:
return color, depth
return color
def _create_scene_and_offscreen_render():
"""We will add (and remove) the meshes later.
Unfortunately this requires some tuning of the position and rotation.
"""
scene = pyrender.Scene()
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
# The position / translation [px, py, pz] and then rotation matrix.
p = [0.45, -0.45, 0.90]
theta = -45 * DEG_TO_RAD
RX = np.array([
[1.0, 0.0, 0.0],
[0.0, np.cos(theta), np.sin(theta)],
[0.0, -np.sin(theta), np.cos(theta)],
])
R = RX
camera_pose = np.array([
[R[0,0], R[0,1], R[0,2], p[0]],
[R[1,0], R[1,1], R[1,2], p[1]],
[R[2,0], R[2,1], R[2,2], p[2]],
[ 0.0, 0.0, 0.0, 1.0],
])
scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(2), intensity=1.0,
innerConeAngle=np.pi/16.0,
outerConeAngle=np.pi/6.0)
scene.add(light, pose=camera_pose)
# Only for debugging
#v = pyrender.Viewer(scene, use_raymond_lighting=True)
rend = pyrender.OffscreenRenderer(640, 480)
return scene, rend
def render_depth(path, override=None):
# Load the FUZE bottle trimesh and put it in a scene
fuze_trimesh = trimesh.load(path)
fuze_trimesh.vertices = (
fuze_trimesh.vertices - np.amin(fuze_trimesh.vertices, axis=0)) / (
np.amax(fuze_trimesh.vertices, axis=0) -
np.amin(fuze_trimesh.vertices, axis=0) + 0.001) - 0.5
mesh = pyrender.Mesh.from_trimesh(fuze_trimesh)
scene = pyrender.Scene()
scene.add(mesh)
# Set up the camera -- z-axis away from the scene, x-axis right, y-axis up
camera = pyrender.PerspectiveCamera(yfov=np.pi / 5.0, znear=0.15)
camera_pose = getCameraPose(override)
scene.add(camera, pose=camera_pose)
# Set up the light -- a single spot light in the same spot as the camera
light = pyrender.SpotLight(color=np.ones(3),
intensity=3.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_pose)
# Render the scene
r = pyrender.OffscreenRenderer(200, 200)
depth = r.render(scene, flags=pyrender.RenderFlags.DEPTH_ONLY)
depth = depth / (np.max(depth) + 0.0001)
return depth
def rendering(R, fuze_trimesh):
mesh = pyrender.Mesh.from_trimesh(fuze_trimesh, poses=[R])
scene = pyrender.Scene()
scene.add(mesh)
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3, aspectRatio=1)
m1 = np.array([
[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0],
[0, 0, 0, 1],
],
dtype='float')
m2 = np.array([
[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, 0.4],
[0, 0, 0, 1],
],
dtype='float')
camera_pose = m1.dot(m2)
scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=3.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_pose)
r = pyrender.OffscreenRenderer(400, 400)
color, depth = r.render(scene)
color = cv2.cvtColor(color, cv2.COLOR_BGR2RGB)
return color
def render_hmr_smpl(hmr_coeff, f_len=500., rend_size=224., req_model=False):
# hmr_coeff is a 85-vector, named as theta in hmr
hmr_coeff = np.asarray(hmr_coeff).tolist()
# make scene
scene = pyrender.Scene()
# initialize camera
camera = pyrender.PerspectiveCamera(
yfov=np.arctan(rend_size * 0.5 / f_len) * 2, aspectRatio=1)
camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0],
[0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]])
scene.add(camera, pose=camera_pose)
# initialize light
light_posi1 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, -1.0],
[0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]])
light_posi2 = np.array([[1.0, 0.0, 0.0, -1.0], [0.0, 0.0, 1.0, -1.0],
[0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]])
light_posi3 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0],
[0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]])
light = pyrender.SpotLight(color=np.array(
[0.65098039, 0.74117647, 0.85882353]),
intensity=100,
innerConeAngle=np.pi / 16.0,
outerConeAngle=np.pi / 6.0)
scene.add(light, pose=light_posi1)
scene.add(light, pose=light_posi2)
scene.add(light, pose=light_posi3)
# get renderer
r = pyrender.OffscreenRenderer(viewport_width=rend_size,
viewport_height=rend_size)
# get verts from smpl coefficients
smpl_op = load_model("./tf_smpl/neutral_smpl_with_cocoplus_reg.pkl")
smpl_op.pose[:] = np.asarray(hmr_coeff[3:75])
smpl_op.betas[:] = np.array(hmr_coeff[75:85])
verts = np.array(smpl_op)
global_t = np.array(
[hmr_coeff[1], hmr_coeff[2], f_len / (0.5 * rend_size * hmr_coeff[0])])
verts = verts + global_t
faces = np.load("./tf_smpl/smpl_faces.npy").astype(np.int32)
# smooth and expand
om_mesh = make_trimesh(verts, faces)
om_mesh = smooth_mesh(om_mesh, 4)
om_mesh = expand_mesh(om_mesh, 0.026)
this_trimesh = trimesh.Trimesh(vertices=om_mesh.points(),
faces=om_mesh.face_vertex_indices())
this_mesh = pyrender.Mesh.from_trimesh(this_trimesh)
scene.add(this_mesh)
rend_img, depth = r.render(scene)
if req_model is True:
return rend_img, verts, faces, depth
else:
return rend_img
def _init_scene(self):
self._scene = pyrender.Scene()
camera = pyrender.PerspectiveCamera(
yfov=self._fov, aspectRatio=1.0,
znear=0.001) # do not change aspect ratio
theta, phi = 0, np.pi / 4 + np.random.rand() * (np.pi / 8)
radius = 3 + (np.random.rand() - 1)
x = radius * np.sin(phi) * np.cos(theta)
y = radius * np.sin(phi) * np.sin(theta)
z = radius * np.cos(phi)
camera_pos = np.array([x, y, z])
#camera_pos = np.array([0,0,10])
#light_pos = np.array([0,0,10])
camera_pose = tra.euler_matrix(0, phi, 0)
camera_pose[:3, 3] = camera_pos
camera_pose[:3, :3] = camera_pose[:3, :3]
self._scene.add(camera, pose=camera_pose, name='camera')
self.camera_pose = camera_pose
light = pyrender.SpotLight(color=np.ones(4),
intensity=50.,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6.0)
self._scene.add(light, pose=camera_pose, name='light')
def render_pred_coeff(pred_coeff):
pred_coeff = np.asarray(pred_coeff).tolist()
# make scene
scene = pyrender.Scene()
# initialize camera
f_len = 366.474487
rend_size = 256.
camera = pyrender.PerspectiveCamera(yfov=np.arctan(rend_size*0.5/f_len)*2, aspectRatio=1)
camera_pose = np.array([[1.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0],
[0.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 1.0]])
scene.add(camera, pose=camera_pose)
# initialize light
light_posi1 = np.array([[1.0, 0.0, 0.0, 1.0],
[0.0, 0.0, 1.0, -1.0],
[0.0, 1.0, 0.0, -2.0],
[0.0, 0.0, 0.0, 1.0]])
light_posi2 = np.array([[1.0, 0.0, 0.0, -1.0],
[0.0, 0.0, 1.0, -1.0],
[0.0, 1.0, 0.0, -2.0],
[0.0, 0.0, 0.0, 1.0]])
light_posi3 = np.array([[1.0, 0.0, 0.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 1.0, 0.0, -2.0],
[0.0, 0.0, 0.0, 1.0]])
light = pyrender.SpotLight(color=np.array([0.65098039, 0.74117647, 0.85882353]),
intensity=100,
innerConeAngle=np.pi/16.0,
outerConeAngle=np.pi/6.0)
scene.add(light, pose=light_posi1)
scene.add(light, pose=light_posi2)
scene.add(light, pose=light_posi3)
# get renderer
r = pyrender.OffscreenRenderer(viewport_width=rend_size, viewport_height=rend_size)
# get verts from smpl coefficients
smpl_op = load_model("./tf_smpl/neutral_smpl_with_cocoplus_reg.pkl")
verts_bias = pred_coeff[3:6]
smpl_op.pose[:] = np.asarray([0]*3 + pred_coeff[6:75])
smpl_op.betas[:] = np.array(pred_coeff[75:85])
verts = np.array(smpl_op)
rot_mat = cv2.Rodrigues(np.asarray(pred_coeff[3:6]))[0]
verts = np.tensordot(verts, rot_mat, axes=([1],[1]))
verts = verts + pred_coeff[:3]
# make trimesh
faces = np.load("./tf_smpl/smpl_faces.npy").astype(np.int32)
this_trimesh = trimesh.Trimesh(vertices = verts, faces = faces)
this_mesh = pyrender.Mesh.from_trimesh(this_trimesh)
scene.add(this_mesh)
rend_img, _ = r.render(scene)
return rend_img
def render(self, camera_pose, target_id="", render_pc=True):
"""Render RGB/depth image, point cloud, and segmentation mask of the scene.
Args:
camera_pose (np.ndarray): Homogenous 4x4 matrix describing the pose of the camera in scene coordinates.
target_id (str, optional): Object ID which is used to create the segmentation mask. Defaults to ''.
render_pc (bool, optional): If true, point cloud is also returned. Defaults to True.
Returns:
np.ndarray: Color image.
np.ndarray: Depth image.
np.ndarray: Point cloud.
np.ndarray: Segmentation mask.
"""
# Keep local to free OpenGl resources after use
renderer = pyrender.OffscreenRenderer(viewport_width=self._width,
viewport_height=self._height)
# add camera and light to scene
scene = self._scene.as_pyrender_scene()
scene.add(self._camera, pose=camera_pose, name="camera")
light = pyrender.SpotLight(
color=np.ones(4),
intensity=3.0,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6.0,
)
scene.add(light, pose=camera_pose, name="light")
# render the full scene
color, depth = renderer.render(scene)
segmentation = np.zeros(depth.shape, dtype=np.uint8)
# hide all objects
for node in scene.mesh_nodes:
node.mesh.is_visible = False
# Render only target object and add to segmentation mask
for node in scene.mesh_nodes:
if node.name == target_id:
node.mesh.is_visible = True
_, object_depth = renderer.render(scene)
mask = np.logical_and((np.abs(object_depth - depth) < 1e-6),
np.abs(depth) > 0)
segmentation[mask] = 1
for node in scene.mesh_nodes:
node.mesh.is_visible = True
if render_pc:
pc = self._to_pointcloud(depth)
else:
pc = None
return color, depth, pc, segmentation
def __init__(self, imgH, imgW):
self.imgW = imgW
self.imgH = imgH
self.scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0])
self.nodesDict = {}
self.light = pyrender.SpotLight(color=np.ones(3), intensity=3.0,
innerConeAngle=np.pi / 16.0, outerConeAngle=np.pi / 6.0)
self.scene.add(self.light, pose=np.eye(4))
def _init_scene(self):
self._scene = pyrender.Scene()
camera = pyrender.PerspectiveCamera(
yfov=self._fov, aspectRatio=1.0,
znear=0.001) # do not change aspect ratio
camera_pose = tra.euler_matrix(np.pi, 0, 0)
self._scene.add(camera, pose=camera_pose, name='camera')
light = pyrender.SpotLight(color=np.ones(4),
intensity=3.,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6.0)
self._scene.add(light, pose=camera_pose, name='light')
self.renderer = pyrender.OffscreenRenderer(400, 400)
def get_empty_scene(self):
"""Creates an empty scene with a camera and spot light."""
scene = pyrender.Scene(ambient_light=[0.2, 0.2, 0.2],
bg_color=[0.1, 0.1, 0.1])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1)
camera_pose = translation_matrix([0, 3, 4]) @ quaternion_matrix(
quaternion_about_axis(np.deg2rad(-45.0), [1, 0, 0]))
scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=3.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_pose)
return scene
def img_renderer(pointcloud):
pcd = o3d.io.read_point_cloud(pointcloud)
pcd = pcd.voxel_down_sample(voxel_size=0.05)
pcd.estimate_normals()
# estimate radius for rolling ball
distances = pcd.compute_nearest_neighbor_distance()
avg_dist = np.mean(distances)
radius = 5 * avg_dist
mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_ball_pivoting(
pcd,
o3d.utility.DoubleVector([radius, radius * 2]))
# create the triangular mesh with the vertices and faces from open3d
tri_mesh = trimesh.Trimesh(np.asarray(mesh.vertices), np.asarray(mesh.triangles),
vertex_normals=np.asarray(mesh.vertex_normals))
#tri_mesh = trimesh.convex.is_convex(tri_mesh)
mesh = pyrender.Mesh.from_trimesh(tri_mesh)
scene = pyrender.Scene()
scene.add(mesh)
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
s = np.sqrt(2)/2
camera_pose = np.array([
[0.0, -s, s, 0.3],
[1.0, 0.0, 0.0, 0.0],
[0.0, s, s, 0.35],
[0.0, 0.0, 0.0, 1.0], ])
scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3), intensity=3.0,
innerConeAngle=np.pi/16.0,
outerConeAngle=np.pi/6.0)
scene.add(light, pose=camera_pose)
pyrender.Viewer(scene)
r = pyrender.OffscreenRenderer(512, 512)
color, _ = r.render(scene)
plt.figure(figsize=(8,8))
plt.imshow(color)
def render4mesh(self, idx, ratio=1):
#the ratio=10 can make the rendered image be black
vertices4view = self.get4viewManovertices(idx)
import trimesh
import pyrender
from pyrender import RenderFlags
np_rot_x = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]],
dtype=np.float32)
np_rot_x = np.reshape(np.tile(np_rot_x, [1, 1]), [3, 3])
recolorlist = []
for iv in range(4):
xyz = vertices4view[iv, :778, :3, 0].copy()
cv = xyz @ np_rot_x
tmesh = trimesh.Trimesh(vertices=cv / 1000 * ratio,
faces=self.mano_right.faces)
# tmesh.visual.vertex_colors = [.9, .7, .7, 1]
# tmesh.show()
mesh = pyrender.Mesh.from_trimesh(tmesh)
scene = pyrender.Scene()
scene.add(mesh)
pycamera = pyrender.IntrinsicsCamera(self.camera[iv].fx,
self.camera[iv].fy,
self.camera[iv].cx,
self.camera[iv].cy,
znear=0.0001,
zfar=3000)
ccamera_pose = self.camera_pose[self.rootcameraidx]
scene.add(pycamera, pose=ccamera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=2.0,
innerConeAngle=np.pi / 16.0)
# light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=2.0)
scene.add(light, pose=ccamera_pose)
r = pyrender.OffscreenRenderer(640, 480)
# flags = RenderFlags.SHADOWS_DIRECTIONAL
recolor, depth = r.render(scene)
# cv2.imshow("depth", depth)
recolorlist.append(recolor[:, :, :3])
meshcolor = np.hstack(recolorlist)
return meshcolor
def texture_ply(self):
# drop the RGB properties, and add two new properties (u, v)
# vert_list = []
# for vert in self.vertices.data:
# vert = vert.tolist() # convert to tuple
# vertices_utm = np.reshape(self.vertices.data
vertices_utm = np.stack(
(self.vertices['x'], self.vertices['y'], self.vertices['z']),
axis=1)
vertices_enu = self.reconstruction.utm_to_enu(vertices_utm)
# vert_list.append(xyz)
# vert_list.append(vert[0:3]+(u, v))
# vertices = np.array(vert_list,
# dtype=[('x', '<f4'), ('y', '<f4'), ('z', '<f4')])
# ('u', '<f4'), ('v', '<f4')])
# vert_el = PlyElement.describe(vertices, 'vertex',
# comments=['point coordinate, texture coordinate'])
# self.ply_textured = PlyData([vert_el, self.faces], text=True)
print 'ply_vertices:', vertices_enu[0:2, :]
renderer = pyrender.OffscreenRenderer(1000, 1000)
for image, camera in self.reconstruction.cameras.items():
print camera.project(vertices_enu[0, :])
test_camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
test_camera_pose = np.array([[1, 0, 0, 0], [0, 1, 0, 0],
[0, 0, 1, 1000.0], [0, 0, 0, 1]])
# self.scene.add(camera.pyrender_camera, camera.pose)
self.scene.add(test_camera, pose=test_camera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=3.0,
innerConeAngle=np.pi / 16.0)
# self.scene.add(light, pose=camera.pose)
# self.scene.add(light, pose=test_camera_pose)
# renderer = pyrender.OffscreenRenderer(camera.width, camera.height)
color, depth = renderer.render(self.scene)
# png.from_array(color, 'RGB').save(image + '_render.png')
png.from_array(color, 'RGB').save('test_render.png')
def create_scene(self):
self.scene = pyrender.Scene()
# load table
table_trimesh = trimesh.load(self.table_mesh_path)
table_mesh = pyrender.Mesh.from_trimesh(table_trimesh)
self.scene.add(table_mesh)
# setup camera
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
self.camera_pose = np.array([
[1, 0, 0, 0],
[0, 1, 0, 0],
[0, 0, 1, self.dist],
[0, 0, 0, 1]
])
nc = pyrender.Node(camera=camera, matrix=self.camera_pose)
self.scene.add_node(nc)
# get camera projection matrix
self.proj_matrix = camera.get_projection_matrix(self.img_w, self.img_h)
# add light
light = pyrender.SpotLight(color=np.ones(3), intensity=1.0,
innerConeAngle=np.pi/16.0,
outerConeAngle=np.pi/6.0)
light_pose = np.array([
[1, 0, 0, 0.2],
[0, 1, 0, 0.2],
[0, 0, 1, 1],
[0, 0, 0, 1]
])
self.scene.add(light, pose=light_pose)
# load mesh
mesh_file = os.path.join(self.mesh_path, self.mesh)
fuze_trimesh = trimesh.load(mesh_file)
mesh_file = pyrender.Mesh.from_trimesh(fuze_trimesh)
self.scene.add(mesh_file, pose=self.obj_pose)
def __init__(self, mesh):
import pyrender
# rotate 180 around x because the Z dir of the reference grid is down
T = np.eye(4)
T[0:3, 0:3] = rot_x(np.pi)
mesh.apply_transform(T)
# Load the trimesh and put it in a scene
mesh = pyrender.Mesh.from_trimesh(mesh)
scene = pyrender.Scene(bg_color=np.array([0, 0, 0, 0]))
scene.add(mesh)
# add temp cam
self.camera = pyrender.IntrinsicsCamera(K[0, 0],
K[1, 1],
K[0, 2],
K[1, 2],
zfar=10000,
name="cam")
light_pose = np.array([
[1.0, 0, 0, 0.0],
[0, 1.0, 0.0, 10.0],
[0.0, 0, 1, 100.0],
[0.0, 0.0, 0.0, 1.0],
])
self.cam_node = scene.add(self.camera, pose=light_pose)
# Set up the light -- a single spot light in z+
light = pyrender.SpotLight(color=255 * np.ones(3),
intensity=3000.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=light_pose)
self.scene = scene
self.r = pyrender.OffscreenRenderer(width, height)
# add the A flag for the masking
self.flag = pyrender.constants.RenderFlags.RGBA
def process_mesh(self):
mesh_path = os.path.join(self.folder_name, self.name + '.ply')
print('Attempting to load mesh: %s ...' % self.name)
if not os.path.exists(mesh_path):
print("ERROR: mesh file does not exist: %s" % mesh_path)
# Load the mesh and put it in a scene
#input_trimesh = trimesh.load('./example_data/pointclouds/teapot_normal_dense.obj')
input_trimesh = trimesh.load_mesh(mesh_path)
trimesh_scene = input_trimesh.scene()
input_trimesh.vertices -= trimesh_scene.centroid
volume = input_trimesh.bounding_sphere.volume
radius = np.power((3*volume/(4*np.pi)),1/3)
#input_trimesh.vertices *= (1/(4*radius))
input_trimesh.apply_scale(1/(1.2*input_trimesh.scale))
print("Mesh has %d vertices" % input_trimesh.vertices.shape[0] )
#scene = fuze_trimesh.scene()
self.mesh = pyrender.Mesh.from_trimesh(input_trimesh)
self.scene = pyrender.Scene(ambient_light=(0.5, 0.5, 0.5))
#scene = pyrender.Scene.from_trimesh_scene(fuze_trimesh)
self.scene.add(self.mesh)
# Set up the camera -- z-axis away from the scene, x-axis right, y-axis up
# Add lights
light = pyrender.SpotLight(color=np.ones(3), intensity=24.0,
innerConeAngle=np.pi / 16.0)
light_node = self.scene.add(light, pose= lookAt(np.array([2,2,2]), [0,0,0]))
light_node = self.scene.add(light, pose= lookAt(np.array([2,6,3]), [0,0,0]))
light_node = self.scene.add(light, pose= lookAt(np.array([2,-1,-3]), [0,0,0]))
light_node = self.scene.add(light, pose= lookAt(np.array([-4,4,-3]), [0,0,0]))
light_node = self.scene.add(light, pose= lookAt(np.array([-2,-2,-3]), [0,0,0]))
# Set up the camera -- z-axis away from the scene, x-axis right, y-axis up
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
s = np.sqrt(2) / 2
camera_pose = np.array([
[0.0, -s, s, 0.3],
[1.0, 0.0, 0.0, 0.0],
[0.0, s, s, 0.35],
[0.0, 0.0, 0.0, 1.0],
])
scene.add(camera, pose=camera_pose)
# Set up the light -- a single spot light in the same spot as the camera
light = pyrender.SpotLight(color=np.ones(3),
intensity=3.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_pose)
# Render the scene
r = pyrender.OffscreenRenderer(640, 480)
color, depth = r.render(scene)
# Show the images
import matplotlib.pyplot as plt
plt.figure()
plt.subplot(1, 2, 1)
plt.axis('off')
plt.imshow(color)
plt.subplot(1, 2, 2)
bin_pose[:3, :3] = bin_R
bin_pose[:3, 3] = bin_t
bin_render_mesh = pyrender.Mesh.from_trimesh(bin_trimesh)
scene.add(bin_render_mesh, pose=bin_pose, name=binId)
for key in obj_keys:
obj_t, obj_q = pybullet.getBasePositionAndOrientation(key, physicsClient)
obj_R = Rotation.from_quat(obj_q).as_matrix()
obj_pose = np.eye(4)
obj_pose[:3, :3] = obj_R
obj_pose[:3, 3] = obj_t
obj_render_mesh = pyrender.Mesh.from_trimesh(obj_trimesh)
scene.add(obj_render_mesh, pose=obj_pose, name=str(key))
# Add light
light = pyrender.SpotLight(color=np.ones(3), intensity=4.0)
light_pose = np.eye(4)
light_pose[2, 3] = 1.8
scene.add(light, pose=light_pose)
# Add camera
width = 2048
height = 1536
fx = 3700
fy = 3700
cx = width / 2
cy = height / 2
camera = pyrender.IntrinsicsCamera(fx, fy, cx, cy)
camera_pose = np.eye(4)
camera_pose[2, 3] = 1.6
scene.add(camera, pose=camera_pose)
def render_mesh_pc_bed_pyrender_everything_synth(self, smpl_verts, smpl_faces, camera_point, bedangle, RESULTS_DICT,
smpl_verts_gt = None, pmat = None, smpl_render_points = False, markers = None,
dropout_variance=None, tf_corners = None, save_name = 'test_synth'):
pmat *= 0.75
pmat[pmat>0] += 10
viz_popup = False
#print np.min(smpl_verts[:, 0])
#print np.min(smpl_verts[:, 1])
shift_estimate_sideways = np.min([-0.15, np.min(smpl_verts[:, 1])])
#print shift_estimate_sideways
shift_estimate_sideways = 0.8 - shift_estimate_sideways
top_smpl_vert = np.max(smpl_verts[:, 0])
extend_top_bottom = np.max([np.max(smpl_verts[:, 0]), 64*.0286]) - 64*.0286
print extend_top_bottom, 'extend top bot'
shift_both_amount = np.max([0.9, np.max(smpl_verts[:, 1])]) #if smpl is bigger than 0.9 shift less
shift_both_amount = 1.5 - shift_both_amount + (0.15 + np.min([-0.15, np.min(smpl_verts[:, 1])]))
smpl_verts_quad = np.concatenate((smpl_verts, np.ones((smpl_verts.shape[0], 1))), axis = 1)
smpl_verts_quad = np.swapaxes(smpl_verts_quad, 0, 1)
smpl_verts_quad_gt = np.concatenate((smpl_verts_gt, np.ones((smpl_verts_gt.shape[0], 1))), axis = 1)
smpl_verts_quad_gt = np.swapaxes(smpl_verts_quad_gt, 0, 1)
#print smpl_verts_quad.shape
shift_ground_truth = 1.3
transform_A = np.identity(4)
transform_A[1, 3] = shift_both_amount
transform_B = np.identity(4)
transform_B[1, 3] = shift_estimate_sideways + shift_both_amount#4.0 #move things over
smpl_verts_B = np.swapaxes(np.matmul(transform_B, smpl_verts_quad), 0, 1)[:, 0:3]
transform_C = np.identity(4)
transform_C[1, 3] = shift_estimate_sideways + shift_both_amount+shift_ground_truth #move things over
smpl_verts_C = np.swapaxes(np.matmul(transform_C, smpl_verts_quad_gt), 0, 1)[:, 0:3]
from matplotlib import cm
human_mesh_vtx_all, human_mesh_face_all = self.get_human_mesh_parts(smpl_verts_B, smpl_faces, segment_limbs=False)
#GET MESH WITH PMAT
tm_curr = trimesh.base.Trimesh(vertices=np.array(human_mesh_vtx_all[0]), faces = np.array(human_mesh_face_all[0]))
tm_list = [tm_curr]
original_mesh = [tm_curr]
mesh_list = []
mesh_list.append(pyrender.Mesh.from_trimesh(tm_list[0], material = self.human_mat, smooth=True))#wireframe = False)) #this is for the main human
human_mesh_vtx_all_gt, human_mesh_face_all_gt = self.get_human_mesh_parts(smpl_verts_C, smpl_faces, segment_limbs=False)
#GET MESH GT WITH PMAT
tm_curr_gt = trimesh.base.Trimesh(vertices=np.array(human_mesh_vtx_all_gt[0]), faces = np.array(human_mesh_face_all_gt[0]))
tm_list_gt = [tm_curr_gt]
original_mesh_gt = [tm_curr_gt]
mesh_list_gt = []
mesh_list_gt.append(pyrender.Mesh.from_trimesh(tm_list_gt[0], material = self.human_mat_gt, smooth=True))#wireframe = False)) #this is for the main human
fig = plt.figure()
if self.render == True:
artag_meshes = []
artag_tm = trimesh.base.Trimesh(vertices=self.artag_r + [0.0, shift_estimate_sideways + shift_both_amount, 0.0], faces=self.artag_f, face_colors = self.artag_facecolors)
artag_meshes.append(pyrender.Mesh.from_trimesh(artag_tm, smooth = False))
artag_meshes_gt = []
artag_tm_gt = trimesh.base.Trimesh(vertices=self.artag_r + [0.0, shift_estimate_sideways + shift_both_amount+shift_ground_truth, 0.0], faces=self.artag_f, face_colors = self.artag_facecolors_gt)
artag_meshes_gt.append(pyrender.Mesh.from_trimesh(artag_tm_gt, smooth = False))
if pmat is not None:
pmat_verts, pmat_faces, pmat_facecolors = self.get_3D_pmat_markers(pmat, bedangle)
pmat_verts = np.array(pmat_verts)
pmat_verts = np.concatenate((np.swapaxes(pmat_verts, 0, 1), np.ones((1, pmat_verts.shape[0]))), axis = 0)
pmat_verts = np.swapaxes(np.matmul(transform_A, pmat_verts), 0, 1)[:, 0:3]
pmat_tm = trimesh.base.Trimesh(vertices=pmat_verts, faces=pmat_faces, face_colors = pmat_facecolors)
pmat_mesh = pyrender.Mesh.from_trimesh(pmat_tm, smooth = False)
pmat_verts2, _, pmat_facecolors2 = self.get_3D_pmat_markers(pmat, bedangle, solidcolor = True)
pmat_verts2 = np.array(pmat_verts2)
pmat_verts2 = np.concatenate((np.swapaxes(pmat_verts2, 0, 1), np.ones((1, pmat_verts2.shape[0]))), axis = 0)
pmat_verts2 = np.swapaxes(np.matmul(transform_B, pmat_verts2), 0, 1)[:, 0:3]
pmat_tm2 = trimesh.base.Trimesh(vertices=pmat_verts2, faces=pmat_faces, face_colors = pmat_facecolors2)
pmat_mesh2 = pyrender.Mesh.from_trimesh(pmat_tm2, smooth = False)
else:
pmat_mesh = None
pmat_mesh2 = None
#print "Viewing"
if self.first_pass == True:
for mesh_part in mesh_list:
self.scene.add(mesh_part)
for mesh_part_gt in mesh_list_gt:
self.scene.add(mesh_part_gt)
if pmat_mesh is not None:
self.scene.add(pmat_mesh)
if pmat_mesh2 is not None:
self.scene.add(pmat_mesh2)
for artag_mesh in artag_meshes:
if artag_mesh is not None:
self.scene.add(artag_mesh)
for artag_mesh_gt in artag_meshes_gt:
if artag_mesh_gt is not None:
self.scene.add(artag_mesh_gt)
lighting_intensity = 20.
#self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, lighting_intensity=lighting_intensity,
# point_size=2, run_in_thread=True, viewport_size=(1200, 1200))
self.first_pass = False
self.node_list = []
for mesh_part in mesh_list:
for node in self.scene.get_nodes(obj=mesh_part):
self.node_list.append(node)
self.node_list_gt = []
for mesh_part_gt in mesh_list_gt:
for node in self.scene.get_nodes(obj=mesh_part_gt):
self.node_list_gt.append(node)
self.artag_nodes = []
for artag_mesh in artag_meshes:
if artag_mesh is not None:
for node in self.scene.get_nodes(obj=artag_mesh):
self.artag_nodes.append(node)
self.artag_nodes_gt = []
for artag_mesh_gt in artag_meshes_gt:
if artag_mesh_gt is not None:
for node in self.scene.get_nodes(obj=artag_mesh_gt):
self.artag_nodes_gt.append(node)
if pmat_mesh is not None:
for node in self.scene.get_nodes(obj=pmat_mesh):
self.pmat_node = node
if pmat_mesh2 is not None:
for node in self.scene.get_nodes(obj=pmat_mesh2):
self.pmat_node2 = node
camera_pose = np.eye(4)
# camera_pose[0,0] = -1.0
# camera_pose[1,1] = -1.0
camera_pose[0, 0] = np.cos(np.pi/2)
camera_pose[0, 1] = np.sin(np.pi/2)
camera_pose[1, 0] = -np.sin(np.pi/2)
camera_pose[1, 1] = np.cos(np.pi/2)
rot_udpim = np.eye(4)
rot_y = 180*np.pi/180.
rot_udpim[1,1] = np.cos(rot_y)
rot_udpim[2,2] = np.cos(rot_y)
rot_udpim[1,2] = np.sin(rot_y)
rot_udpim[2,1] = -np.sin(rot_y)
camera_pose = np.matmul(rot_udpim, camera_pose)
camera_pose[0, 3] = 64*0.0286/2 # -1.0
camera_pose[1, 3] = 1.2 + 0.8
camera_pose[2, 3] = -1.0
if viz_popup == True:
self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True,
lighting_intensity=10.,
point_size=5, run_in_thread=True, viewport_size=(1000, 1000))
#camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
magnify =(64*.0286)
camera = pyrender.OrthographicCamera(xmag=magnify, ymag = magnify)
self.scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3), intensity=250.0, innerConeAngle=np.pi / 10.0,
outerConeAngle=np.pi / 2.0)
light_pose = np.copy(camera_pose)
# light_pose[1, 3] = 2.0
light_pose[0, 3] = 0.8
light_pose[1, 3] = -0.5
light_pose[2, 3] = -2.5
light_pose2 = np.copy(camera_pose)
light_pose2[0, 3] = 2.5
light_pose2[1, 3] = 1.0
light_pose2[2, 3] = -5.0
light_pose3 = np.copy(camera_pose)
light_pose3[0, 3] = 1.0
light_pose3[1, 3] = 5.0
light_pose3[2, 3] = -4.0
#light_pose2[0, 3] = 1.0
#light_pose2[1, 3] = 2.0 #across
#light_pose2[2, 3] = -1.5
# light_pose[1, ]
self.scene.add(light, pose=light_pose)
self.scene.add(light, pose=light_pose2)
self.scene.add(light, pose=light_pose3)
else:
if viz_popup == True:
self.viewer.render_lock.acquire()
#reset the human mesh
for idx in range(len(mesh_list)):
self.scene.remove_node(self.node_list[idx])
self.scene.add(mesh_list[idx])
for node in self.scene.get_nodes(obj=mesh_list[idx]):
self.node_list[idx] = node
#reset the human mesh
for idx in range(len(mesh_list_gt)):
self.scene.remove_node(self.node_list_gt[idx])
self.scene.add(mesh_list_gt[idx])
for node in self.scene.get_nodes(obj=mesh_list_gt[idx]):
self.node_list_gt[idx] = node
#reset the artag meshes
for artag_node in self.artag_nodes:
self.scene.remove_node(artag_node)
for artag_mesh in artag_meshes:
if artag_mesh is not None:
self.scene.add(artag_mesh)
self.artag_nodes = []
for artag_mesh in artag_meshes:
if artag_mesh is not None:
for node in self.scene.get_nodes(obj=artag_mesh):
self.artag_nodes.append(node)
#reset the artag meshes
for artag_node_gt in self.artag_nodes_gt:
self.scene.remove_node(artag_node_gt)
for artag_mesh_gt in artag_meshes_gt:
if artag_mesh_gt is not None:
self.scene.add(artag_mesh_gt)
self.artag_nodes_gt = []
for artag_mesh_gt in artag_meshes_gt:
if artag_mesh_gt is not None:
for node in self.scene.get_nodes(obj=artag_mesh_gt):
self.artag_nodes_gt.append(node)
#reset the pmat mesh
if pmat_mesh is not None:
self.scene.remove_node(self.pmat_node)
self.scene.add(pmat_mesh)
for node in self.scene.get_nodes(obj=pmat_mesh):
self.pmat_node = node
#reset the pmat mesh
if pmat_mesh2 is not None:
self.scene.remove_node(self.pmat_node2)
self.scene.add(pmat_mesh2)
for node in self.scene.get_nodes(obj=pmat_mesh2):
self.pmat_node2 = node
#print self.scene.get_nodes()
if viz_popup == True:
self.viewer.render_lock.release()
#time.sleep(100)
if viz_popup == False:
r = pyrender.OffscreenRenderer(880, 880)
# r.render(self.scene)
color_render, depth = r.render(self.scene)
# plt.subplot(1, 2, 1)
plt.axis('off')
#im_to_show = np.concatenate((color_render, color_im), axis = 1)
im_to_show = np.copy(color_render)
im_to_show = im_to_show[130-int(extend_top_bottom*300):750+int(extend_top_bottom*300), :, :]
#plt.imshow(color)
plt.imshow(im_to_show)
# plt.subplot(1, 2, 2)
# plt.axis('off')
# plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show()
fig.set_size_inches(15., 10.)
fig.tight_layout()
#save_name = 'f_hbh_'+'{:04}'.format(self.pic_num)
print "saving!"
fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/'+save_name+'_v2.png', dpi=300)
self.pic_num += 1
#plt.show()
#if self.pic_num == 20:
# print "DONE"
# time.sleep(1000000)
#print "got here"
#print X.shape
return RESULTS_DICT
def vis(model_id, mesh_or_scene):
"""
based on https://colab.research.google.com/drive/1Z71mHIc-Sqval92nK290vAsHZRUkCjUx
setup:
```bash
sudo apt update
sudo wget https://github.com/mmatl/travis_debs/raw/master/xenial/mesa_18.3.3-0.deb
sudo dpkg -i ./mesa_18.3.3-0.deb || true
sudo apt install -f
git clone https://github.com/mmatl/pyopengl.git
pip install ./pyopengl
```
"""
# scene = pyrender.Scene()
if isinstance(mesh_or_scene, trimesh.Scene):
# for mesh in mesh_or_scene.geometry.values():
# scene.add(pyrender.Mesh.from_trimesh(mesh))
# tm_scene = mesh_or_scene
meshes = mesh_or_scene.geometry.values()
else:
# scene.add(pyrender.Mesh.from_trimesh(mesh_or_scene))
# tm_scene = mesh_or_scene.scene()
meshes = [mesh_or_scene]
camera_transform = np.array([
[
0.58991882,
-0.3372407,
0.73366511,
0.86073076,
],
[
0.04221561,
0.92024442,
0.38906047,
0.45644301,
],
[
-0.80635825,
-0.19854197,
0.55710632,
0.65359322,
],
[
0.0,
0.0,
0.0,
1.0,
],
])
trimesh.scene.cameras.Camera(fov=(45.0, 45.0))
light = trimesh.scene.lighting.SpotLight(color=0.5 * np.ones(3),
intensity=0.0,
innerConeAngle=np.pi / 16.0)
tm_scene = trimesh.scene.Scene(meshes, lights=[light])
for name in [light.name, tm_scene.camera.name]:
tm_scene.graph[name] = camera_transform
# tm_scene = trimesh.scene.Scene(meshes)
# views.set_scene_view(tm_scene, resolution, **view_fn(model_id))
tm_scene.camera.resolution = resolution
tm_scene.camera_transform = camera_transform
tm_scene.show()
# dist = np.linalg.norm(tm_scene.camera_transform[:3, 3])
camera = pyrender.PerspectiveCamera(yfov=np.pi / 180 *
tm_scene.camera.fov[1],
# znear=dist-0.5, zfar=dist+0.5
)
scene = pyrender.Scene.from_trimesh_scene(tm_scene)
scene.add(camera, pose=camera_transform)
light = pyrender.SpotLight(color=np.ones(3),
intensity=10.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_transform)
# viewer = pyrender.Viewer(scene, use_direct_lighting=True)
r = pyrender.OffscreenRenderer(*resolution)
color, depth = r.render(scene)
# Show the images
plt.figure()
plt.subplot(1, 2, 1)
plt.axis("off")
plt.imshow(color)
plt.subplot(1, 2, 2)
plt.axis("off")
plt.imshow(depth, cmap=plt.cm.gray_r) # pylint: disable=no-member
plt.show()
def render_mesh_pc_bed_pyrender_everything(self,
smpl_verts,
smpl_faces,
camera_point,
bedangle,
RESULTS_DICT,
pc=None,
pmat=None,
smpl_render_points=False,
markers=None,
dropout_variance=None,
color_im=None,
tf_corners=None,
current_pose_type_ct=None,
participant=None):
#print np.min(smpl_verts[:, 0])
#print np.min(smpl_verts[:, 1])
shift_estimate_sideways = np.min([-0.15, np.min(smpl_verts[:, 1])])
#print shift_estimate_sideways
shift_estimate_sideways = 0.8 - shift_estimate_sideways
top_smpl_vert = np.max(smpl_verts[:, 0])
extend_top_bottom = np.max([np.max(smpl_verts[:, 0]), 64 * .0286
]) - 64 * .0286
print extend_top_bottom, 'extend top bot'
shift_both_amount = np.max([0.9, np.max(smpl_verts[:, 1])
]) #if smpl is bigger than 0.9 shift less
shift_both_amount = 1.5 - shift_both_amount + (
0.15 + np.min([-0.15, np.min(smpl_verts[:, 1])]))
#print np.max(smpl_verts[:, 1]), 'max smpl'
#shift_both_amount = 0.6
#smpl_verts[:, 2] += 0.5
#pc[:, 2] += 0.5
pc[:, 0] = pc[:, 0] # - 0.17 - 0.036608
pc[:, 1] = pc[:, 1] # + 0.09
#adjust the point cloud
#segment_limbs = True
#if pmat is not None:
# if np.sum(pmat) < 5000:
# smpl_verts = smpl_verts * 0.001
smpl_verts_quad = np.concatenate(
(smpl_verts, np.ones((smpl_verts.shape[0], 1))), axis=1)
smpl_verts_quad = np.swapaxes(smpl_verts_quad, 0, 1)
#print smpl_verts_quad.shape
transform_A = np.identity(4)
transform_A[1, 3] = shift_both_amount
transform_B = np.identity(4)
transform_B[
1,
3] = shift_estimate_sideways + shift_both_amount #4.0 #move things over
smpl_verts_B = np.swapaxes(np.matmul(transform_B, smpl_verts_quad), 0,
1)[:, 0:3]
transform_C = np.identity(4)
transform_C[1, 3] = 2.0 #2.0 #move things over
smpl_verts_C = np.swapaxes(np.matmul(transform_C, smpl_verts_quad), 0,
1)[:, 0:3]
from matplotlib import cm
human_mesh_vtx_all, human_mesh_face_all = self.get_human_mesh_parts(
smpl_verts_B, smpl_faces, segment_limbs=False)
#GET MESH WITH PMAT
tm_curr = trimesh.base.Trimesh(vertices=np.array(
human_mesh_vtx_all[0]),
faces=np.array(human_mesh_face_all[0]))
tm_list = [tm_curr]
original_mesh = [tm_curr]
mesh_list = []
mesh_list.append(
pyrender.Mesh.from_trimesh(tm_list[0],
material=self.human_mat,
smooth=False)
) #wireframe = False)) #this is for the main human
print np.shape(color_im)
print tf_corners
top_idx = float(tf_corners[0, 1])
bot_idx = float(tf_corners[2, 1])
perc_total = (bot_idx - top_idx) / 880.
print perc_total
fig = plt.figure()
if self.render == True:
#print m.r
#print artag_r
#create mini meshes for AR tags
artag_meshes = []
if markers is not None:
for marker in markers:
if markers[2] is None:
artag_meshes.append(None)
elif marker is None:
artag_meshes.append(None)
else:
#print marker - markers[2]
if marker is markers[2]:
print "is markers 2", marker
#artag_tm = trimesh.base.Trimesh(vertices=self.artag_r, faces=self.artag_f, face_colors = self.artag_facecolors_root)
#artag_meshes.append(pyrender.Mesh.from_trimesh(artag_tm, smooth = False))
else:
artag_tm = trimesh.base.Trimesh(
vertices=self.artag_r + [
0.0, shift_estimate_sideways +
shift_both_amount, 0.0
],
faces=self.artag_f,
face_colors=self.artag_facecolors)
artag_meshes.append(
pyrender.Mesh.from_trimesh(artag_tm,
smooth=False))
if pmat is not None:
pmat_verts, pmat_faces, pmat_facecolors = self.get_3D_pmat_markers(
pmat, bedangle)
pmat_verts = np.array(pmat_verts)
pmat_verts = np.concatenate((np.swapaxes(
pmat_verts, 0, 1), np.ones((1, pmat_verts.shape[0]))),
axis=0)
pmat_verts = np.swapaxes(np.matmul(transform_A, pmat_verts), 0,
1)[:, 0:3]
pmat_tm = trimesh.base.Trimesh(vertices=pmat_verts,
faces=pmat_faces,
face_colors=pmat_facecolors)
pmat_mesh = pyrender.Mesh.from_trimesh(pmat_tm, smooth=False)
pmat_verts2, _, pmat_facecolors2 = self.get_3D_pmat_markers(
pmat, bedangle, solidcolor=True)
pmat_verts2 = np.array(pmat_verts2)
pmat_verts2 = np.concatenate((np.swapaxes(
pmat_verts2, 0, 1), np.ones((1, pmat_verts2.shape[0]))),
axis=0)
pmat_verts2 = np.swapaxes(np.matmul(transform_B, pmat_verts2),
0, 1)[:, 0:3]
pmat_tm2 = trimesh.base.Trimesh(vertices=pmat_verts2,
faces=pmat_faces,
face_colors=pmat_facecolors2)
pmat_mesh2 = pyrender.Mesh.from_trimesh(pmat_tm2, smooth=False)
else:
pmat_mesh = None
pmat_mesh2 = None
#print "Viewing"
if self.first_pass == True:
for mesh_part in mesh_list:
self.scene.add(mesh_part)
if pmat_mesh is not None:
self.scene.add(pmat_mesh)
if pmat_mesh2 is not None:
self.scene.add(pmat_mesh2)
for artag_mesh in artag_meshes:
if artag_mesh is not None:
self.scene.add(artag_mesh)
lighting_intensity = 20.
#self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, lighting_intensity=lighting_intensity,
# point_size=2, run_in_thread=True, viewport_size=(1200, 1200))
self.first_pass = False
self.node_list = []
for mesh_part in mesh_list:
for node in self.scene.get_nodes(obj=mesh_part):
self.node_list.append(node)
self.artag_nodes = []
for artag_mesh in artag_meshes:
if artag_mesh is not None:
for node in self.scene.get_nodes(obj=artag_mesh):
self.artag_nodes.append(node)
if pmat_mesh is not None:
for node in self.scene.get_nodes(obj=pmat_mesh):
self.pmat_node = node
if pmat_mesh2 is not None:
for node in self.scene.get_nodes(obj=pmat_mesh2):
self.pmat_node2 = node
camera_pose = np.eye(4)
# camera_pose[0,0] = -1.0
# camera_pose[1,1] = -1.0
camera_pose[0, 0] = np.cos(np.pi / 2)
camera_pose[0, 1] = np.sin(np.pi / 2)
camera_pose[1, 0] = -np.sin(np.pi / 2)
camera_pose[1, 1] = np.cos(np.pi / 2)
rot_udpim = np.eye(4)
rot_y = 180 * np.pi / 180.
rot_udpim[1, 1] = np.cos(rot_y)
rot_udpim[2, 2] = np.cos(rot_y)
rot_udpim[1, 2] = np.sin(rot_y)
rot_udpim[2, 1] = -np.sin(rot_y)
camera_pose = np.matmul(rot_udpim, camera_pose)
camera_pose[0, 3] = 64 * 0.0286 / 2 # -1.0
camera_pose[1, 3] = 1.2
camera_pose[2, 3] = -1.0
# self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True,
# lighting_intensity=10.,
# point_size=5, run_in_thread=True, viewport_size=(1000, 1000))
# camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
magnify = (64 * .0286) * 0.5 / perc_total
camera = pyrender.OrthographicCamera(xmag=magnify,
ymag=magnify)
self.scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=200.0,
innerConeAngle=np.pi / 10.0,
outerConeAngle=np.pi / 2.0)
light_pose = np.copy(camera_pose)
# light_pose[1, 3] = 2.0
light_pose[0, 3] = 0.8
light_pose[1, 3] = -0.5
light_pose[2, 3] = -1.5
light_pose2 = np.copy(camera_pose)
light_pose2[0, 3] = 2.5
light_pose2[1, 3] = 1.0
light_pose2[2, 3] = -4.0
# light_pose[1, ]
self.scene.add(light, pose=light_pose)
self.scene.add(light, pose=light_pose2)
else:
#self.viewer.render_lock.acquire()
#reset the human mesh
for idx in range(len(mesh_list)):
self.scene.remove_node(self.node_list[idx])
self.scene.add(mesh_list[idx])
for node in self.scene.get_nodes(obj=mesh_list[idx]):
self.node_list[idx] = node
#reset the artag meshes
for artag_node in self.artag_nodes:
self.scene.remove_node(artag_node)
for artag_mesh in artag_meshes:
if artag_mesh is not None:
self.scene.add(artag_mesh)
self.artag_nodes = []
for artag_mesh in artag_meshes:
if artag_mesh is not None:
for node in self.scene.get_nodes(obj=artag_mesh):
self.artag_nodes.append(node)
#reset the pmat mesh
if pmat_mesh is not None:
self.scene.remove_node(self.pmat_node)
self.scene.add(pmat_mesh)
for node in self.scene.get_nodes(obj=pmat_mesh):
self.pmat_node = node
#reset the pmat mesh
if pmat_mesh2 is not None:
self.scene.remove_node(self.pmat_node2)
self.scene.add(pmat_mesh2)
for node in self.scene.get_nodes(obj=pmat_mesh2):
self.pmat_node2 = node
#print self.scene.get_nodes()
#self.viewer.render_lock.release()
#time.sleep(100)
r = pyrender.OffscreenRenderer(880, 880)
# r.render(self.scene)
color_render, depth = r.render(self.scene)
# plt.subplot(1, 2, 1)
plt.axis('off')
if 880. - bot_idx > top_idx:
print 'shift im down by', 880. - bot_idx - top_idx
downshift = int((880. - bot_idx) / 2 - top_idx / 2 + 0.5)
color_im[downshift:880] = color_im[0:880 - downshift]
elif top_idx > (880. - bot_idx):
print 'shift im up by', top_idx - (880. - bot_idx)
upshift = int(top_idx / 2 - (880. - bot_idx) / 2 + 0.5)
color_im[0:880 - upshift] = color_im[upshift:880]
print tf_corners
print np.shape(color_render), np.shape(color_im)
color_im = np.concatenate(
(color_im[:, :, 2:3], color_im[:, :, 1:2], color_im[:, :, 0:1]),
axis=2)
color_im = color_im[:,
int(tf_corners[0, 0] - 10):int(tf_corners[1, 0] +
10), :]
im_to_show = np.concatenate((color_render, color_im), axis=1)
im_to_show = im_to_show[130 - int(extend_top_bottom * 300):750 +
int(extend_top_bottom * 300), :, :]
#plt.imshow(color)
plt.imshow(im_to_show)
# plt.subplot(1, 2, 2)
# plt.axis('off')
# plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show()
fig.set_size_inches(15., 10.)
fig.tight_layout()
#save_name = 'f_hbh_'+'{:04}'.format(self.pic_num)
save_name = participant + '_' + current_pose_type_ct
fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/' +
participant + '/estimated_poses/' + save_name + '.png',
dpi=300)
#fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/TEST.png', dpi=300)
#plt.savefig('test2png.png', dpi=100)
self.pic_num += 1
#plt.show()
#if self.pic_num == 20:
# print "DONE"
# time.sleep(1000000)
#print "got here"
#print X.shape
return RESULTS_DICT
import trimesh
import pyrender
import numpy as np
import math
import collada
red = [1.0, 0.0, 0.0]
green = [0.0, 1.0, 0.0]
blue = [0.0, 0.0, 1.0]
black = [0.0, 0.0, 0.0]
gray = [0.5, 0.5, 0.5]
white = [1.0, 1.0, 1.0]
point_light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=2.0)
spot_light = pyrender.SpotLight(color=[1.0, 1.0, 1.0],
intensity=2.0,
innerConeAngle=0.05,
outerConeAngle=0.5)
directional_light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0],
intensity=2.0)
perspective_camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0,
aspectRatio=1.414)
ortho_camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0)
def display(mesh, light=None, camera=None):
scene = pyrender.Scene(ambient_light=white)
scene.add(mesh)
if light:
scene.add(light)
if camera:
def _init_light(self):
"""
Set up light
"""
# Load light from config file
light = self.conf.sensor.lights
origin = np.array(light.origin)
xyz = []
if light.polar:
# Apply polar coordinates
thetas = light.xrtheta.thetas
rs = light.xrtheta.rs
xs = light.xrtheta.xs
for i in range(len(thetas)):
theta = np.pi / 180 * thetas[i]
xyz.append(
[xs[i], rs[i] * np.cos(theta), rs[i] * np.sin(theta)])
else:
# Apply cartesian coordinates
xyz = np.array(light.xyz.coords)
colors = np.array(light.colors)
intensities = light.intensities
# Save light nodes
self.light_nodes = []
self.light_poses0 = []
for i in range(len(colors)):
if not self.spot_light_enabled:
# create pyrender.PointLight
color = colors[i]
light_pose_0 = euler2matrix(angles=[0, 0, 0],
translation=xyz[i] + origin)
light = pyrender.PointLight(color=color,
intensity=intensities[i])
elif self.spot_light_enabled:
# create pyrender.SpotLight
color = colors[i]
theta = np.pi / 180 * (thetas[i] - 90)
tuning_angle = -np.pi / 16
light_pose_0 = euler2matrix(
xyz="yzx",
angles=[0, tuning_angle, theta],
translation=xyz[i] + origin,
)
light = pyrender.SpotLight(
color=color,
intensity=intensities[i],
innerConeAngle=0,
outerConeAngle=np.pi / 3,
)
light_node = pyrender.Node(light=light, matrix=light_pose_0)
self.scene.add_node(light_node)
self.light_nodes.append(light_node)
self.light_poses0.append(light_pose_0)
self.current_light_nodes.append(light_node)
# Add extra light node into scene_depth
light_node_depth = pyrender.Node(light=light, matrix=light_pose_0)
self.scene_depth.add_node(light_node_depth)
def renderMeshEGL(mesh,
cam_viewpoint,
number_sampled_points,
z_towards_mesh=True):
"""
Renders a number of points on a mesh from a viewpoint, considering occlusion.
Faces not in sight are detected by casting rays from a camera viewpoint.
Uses EGL hardware acceleration.
Parameters
----------
mesh: trimesh.Mesh
Mesh to sample
cam_viewpoint: (3,) float
Point from which to observe the mesh
number_sampled_points: int
Number of points to sample on the mesh
Returns
----------
points: (n, 3) float
Points sampled on the visible portion of the mesh, in the original mesh reference frame
points_camera_frame: (n, 3) float
Points sampled on the visible portion of the mesh, in the camera reference frame
"""
import os
os.environ["PYOPENGL_PLATFORM"] = "egl"
mesh = pyrender.Mesh.from_trimesh(mesh)
scene = pyrender.Scene()
scene.add(mesh)
# Set up the camera and depth of field
fov = np.pi / 3.0
height = 480
width = 640
fx = fy = width / 2.0 / np.tan(fov / 2.0)
cx = width / 2.0
cy = height / 2.0
camera = pyrender.IntrinsicsCamera(fx=fx, fy=fy, cx=cx, cy=cy)
# z axis is the ray from the mesh center to the camera axis center
# This inversion is necessary because the camera reference system uses the cinematographic
# convention
# https://www.scratchapixel.com/lessons/mathematics-physics-for-computer-graphics/lookat-function
mesh_centroid = mesh.centroid
if z_towards_mesh:
camera_z_axis = (cam_viewpoint - mesh_centroid
) / np.linalg.norm(mesh.centroid - cam_viewpoint)
else:
camera_z_axis = (mesh_centroid - cam_viewpoint
) / np.linalg.norm(mesh.centroid - cam_viewpoint)
# Obtain the camera transformation as a 4x4 and add it
camera_transform_4x4 = trimesh.geometry.align_vectors(
np.array([0, 0, 1], dtype=np.float32), camera_z_axis)
camera_transform_4x4[0:3, 3] = cam_viewpoint
scene.add(camera, pose=camera_transform_4x4)
# Set up the light -- a single spot light in the same spot as the camera
light = pyrender.SpotLight(color=np.ones(3),
intensity=3.0,
innerConeAngle=np.pi / 16.0)
scene.add(light, pose=camera_transform_4x4)
# Render the scene
r = pyrender.OffscreenRenderer(width, height)
color, depth = r.render(scene)
# Clip off depth values where there's nothing
mask = np.where(depth > 0)
x = mask[1]
y = mask[0]
# Compute coordinates given depth and image pixel coordinates
points_x = (x.astype(np.float32) - cx) / fx * depth[y, x]
points_y = (y.astype(np.float32) - cy) / fy * depth[y, x]
points_z = depth[y, x]
points_camera_frame = np.vstack((points_x, points_y, points_z)).T
# Downsample if necessary
if points_camera_frame.shape[0] > number_sampled_points:
choice = np.random.choice(points_camera_frame.shape[0],
number_sampled_points,
replace=False)
points_camera_frame = points_camera_frame[choice, :]
# Obtain point coordinates in the object-centric ref system
points_cam_homogeneous = np.hstack((points_camera_frame,
np.ones(
(points_camera_frame.shape[0], 1),
dtype=np.float32)))
points_world_frame_homogeneous = np.transpose(
np.matmul(np.linalg.inv(camera_transform_4x4),
np.transpose(points_cam_homogeneous)))
points_world_frame = points_world_frame_homogeneous[:, 0:3]
return points_world_frame, points_camera_frame
def render_pose_data(model_path,
num_frames,
width=400,
height=400,
show=False):
r = pyrender.OffscreenRenderer(viewport_width=width,
viewport_height=height,
point_size=1.0)
fuze_trimesh = trimesh.load(model_path)
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
camera_pose = np.array([
[1.0, 0, 0, 0],
[0, 1.0, 0.0, 0],
[0.0, 0, 1, 1.3],
[0.0, 0.0, 0.0, 1.0],
])
if isinstance(fuze_trimesh, trimesh.scene.scene.Scene):
scene = pyrender.Scene.from_trimesh_scene(fuze_trimesh)
nodes = list(scene.nodes)
for i in nodes:
scene.remove_node(i)
meshParent = pyrender.Node("MsehNode", children=nodes)
scene.add_node(meshParent)
mesh_vert = np.vstack([i.vertices for i in fuze_trimesh.dump()])
else:
scene = pyrender.Scene()
meshParent = pyrender.Mesh.from_trimesh(fuze_trimesh)
scene.add(meshParent)
meshParent = list(scene.nodes)[0]
mesh_vert = np.array(fuze_trimesh.vertices)
scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=20.0,
innerConeAngle=np.pi / 6.0,
outerConeAngle=np.pi / 6.0)
scene.add(light, pose=camera_pose)
transform = np.eye(4)
import time
t_s = time.time()
poses = []
images = []
depths = []
for i in range(num_frames):
x = special_ortho_group.rvs(3)
transform[:3, :3] = x
scene.set_pose(meshParent, transform)
color, depth = r.render(scene)
images.append(color)
poses.append(np.copy(transform))
depths.append(depth)
if show:
plt.imshow(color)
plt.show()
dt = time.time() - t_s
print("Rendering Time used {:.4f}s".format(dt))
return images, depths, poses, mesh_vert
def render_trimesh(
trimesh_mesh,
eye,
center,
world_up,
res=(640, 640),
light_intensity=3.0,
ambient_intensity=None,
**kwargs,
):
"""Render a shapenet mesh using default settings.
Args:
trimesh_mesh: trimesh mesh instance, or a list of trimesh meshes
(or point clouds).
eye: array with shape [3,] containing the XYZ world
space position of the camera.
center: array with shape [3,] containing a position
along the center of the camera's gaze.
world_up: np.float32 array with shape [3,] specifying the
world's up direction; the output camera will have no tilt with respect
to this direction.
res: 2-tuple of int, [width, height], resolution (in pixels) of output
images.
light_intensity: float, light intensity.
ambient_intensity: float, ambient light intensity.
kwargs: additional flags to pass to pyrender renderer.
Returns:
color_img: [*res, 3] color image.
depth_img: [*res, 1] depth image.
world_to_cam: [4, 4] camera to world matrix.
projection_matrix: [4, 4] projection matrix, aka cam_to_img matrix.
"""
if not isinstance(trimesh_mesh, list):
trimesh_mesh = [trimesh_mesh]
eye = list2npy(eye).astype(np.float32)
center = list2npy(center).astype(np.float32)
world_up = list2npy(world_up).astype(np.float32)
# setup camera pose matrix
scene = pyrender.Scene(ambient_light=ambient_intensity *
np.ones([3], dtype=float))
for tmesh in trimesh_mesh:
if not (isinstance(tmesh, trimesh.Trimesh)
or isinstance(tmesh, trimesh.PointCloud)):
raise NotImplementedError(
"All instances in trimesh_mesh must be either trimesh.Trimesh "
f"or trimesh.PointCloud. Instead it is {type(tmesh)}.")
if isinstance(tmesh, trimesh.Trimesh):
mesh = pyrender.Mesh.from_trimesh(tmesh)
elif isinstance(tmesh, trimesh.PointCloud):
if tmesh.colors is not None:
colors = np.array(tmesh.colors)
else:
colors = np.ones_like(tmesh.vertices)
mesh = pyrender.Mesh.from_points(np.array(tmesh.vertices),
colors=colors)
scene.add(mesh)
# Set up the camera -- z-axis away from the scene, x-axis right, y-axis up
camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0)
world_to_cam = look_at(eye[None], center[None], world_up[None])
world_to_cam = world_to_cam[0]
cam_pose = np.linalg.inv(world_to_cam)
scene.add(camera, pose=cam_pose)
# Set up the light -- a single spot light in the same spot as the camera
light = pyrender.SpotLight(
color=np.ones(3, dtype=np.float32),
intensity=light_intensity,
innerConeAngle=np.pi / 16.0,
)
scene.add(light, pose=cam_pose)
# Render the scene
r = pyrender.OffscreenRenderer(*res, **kwargs)
color_img, depth_img = r.render(scene)
return (
color_img,
depth_img,
world_to_cam,
camera.get_projection_matrix(*res),
)
def render_only_human_gt(self, m):
bed1_verts = np.array([
[-1.3, -1.35, -3.0],
[-0.9, -0.75, -4.0],
[1.3, -1.35, -3.0],
[0.9, -0.75, -4.0],
[-1.2, 0.05, -4.0],
[-1.2, 0.0, -4.0],
[1.2, 0.05, -4.0],
[1.2, 0.0, -4.0],
[-1.3, -1.35, -3.0],
[-1.3, -1.45, -3.0],
[1.3, -1.35, -3.0],
[1.3, -1.45, -3.0],
[-1.2, 0.05, -4.0],
[-1.0, 1.0, -4.0],
[1.2, 0.05, -4.0],
[1.0, 1.0, -4.0],
])
bed1_faces = np.array([
[0, 1, 2],
[0, 2, 1],
[1, 2, 3],
[1, 3, 2],
[4, 5, 6],
[4, 6, 5],
[5, 6, 7],
[5, 7, 6],
[8, 9, 10],
[8, 10, 9],
[9, 10, 11],
[9, 11, 10],
[12, 13, 14],
[12, 14, 13],
[13, 14, 15],
[13, 15, 14],
])
bed1_facecolors = []
for i in range(bed1_faces.shape[0]):
if 4 <= i < 12:
bed1_facecolors.append([0.8, 0.8, 0.2])
else:
bed1_facecolors.append([1.0, 1.0, 0.8])
smpl_verts = (m.r - m.J_transformed[0, :])
#smpl_verts = np.concatenate((smpl_verts[:, 1:2] - 1.5, smpl_verts[:, 0:1], -smpl_verts[:, 2:3]), axis = 1)
smpl_verts = np.concatenate(
(smpl_verts[:, 0:1] - 1.5, smpl_verts[:, 1:2], smpl_verts[:, 2:3]),
axis=1)
#smpl_verts = np.concatenate((smpl_verts[:, 1:2], smpl_verts[:, 0:1], -smpl_verts[:, 2:3]), axis = 1)
smpl_faces = np.array(m.f)
#smpl_faces = np.concatenate((smpl_faces[:, 0:1],smpl_faces[:, 2:3],smpl_faces[:, 1:2]), axis = 1)
smpl_verts2 = np.concatenate(
(smpl_verts[:, 1:2], smpl_verts[:, 2:3] - 2.0, smpl_verts[:, 0:1]),
axis=1)
smpl_verts3 = np.concatenate(
(smpl_verts[:, 1:2], smpl_verts[:, 2:3] - 2.4, smpl_verts[:, 0:1]),
axis=1)
laying_rot_M = np.array([[1., 0., 0.], [0., 0.866025, -0.5],
[0., 0.5, 0.866025]])
laying_rot_M2 = np.array([[1., 0., 0.], [0., 0.34202, -0.93969],
[0., 0.93969, 0.34202]])
for i in range(smpl_verts2.shape[0]):
smpl_verts2[i, :] = np.matmul(laying_rot_M, smpl_verts2[i, :])
smpl_verts3[i, :] = np.matmul(laying_rot_M2, smpl_verts3[i, :])
#break
#smpl_verts2 = np.concatenate((-smpl_verts[:, 2:3] + 1.5, smpl_verts[:, 1:2], smpl_verts[:, 0:1]), axis = 1)
#smpl_verts3 = np.concatenate((smpl_verts[:, 2:3] - 1.5, smpl_verts[:, 1:2], -smpl_verts[:, 0:1]), axis = 1)
#print smpl_verts2.shape
#tm = trimesh.base.Trimesh(vertices=smpl_verts, faces=smpl_faces)
#mesh = pyrender.Mesh.from_trimesh(tm, material=self.human_mat_for_study, wireframe=False)
tm2 = trimesh.base.Trimesh(vertices=smpl_verts2, faces=smpl_faces)
mesh2 = pyrender.Mesh.from_trimesh(tm2,
material=self.human_mat_for_study,
wireframe=False)
tm3 = trimesh.base.Trimesh(vertices=smpl_verts3, faces=smpl_faces)
mesh3 = pyrender.Mesh.from_trimesh(tm3,
material=self.human_mat_for_study,
wireframe=False)
tm_bed1 = trimesh.base.Trimesh(vertices=bed1_verts,
faces=bed1_faces,
face_colors=np.array(bed1_facecolors))
mesh_bed1 = pyrender.Mesh.from_trimesh(
tm_bed1,
material=self.human_bed_for_study,
wireframe=False,
smooth=False)
fig = plt.figure()
#plt.plot(np.arange(0, 400), np.arange(0, 400)*.5 + 800)
if self.first_pass == True:
self.scene.add(mesh_bed1)
#self.scene.add(mesh)
self.scene.add(mesh2)
self.scene.add(mesh3)
self.first_pass = False
self.node_list_bed1 = []
for node in self.scene.get_nodes(obj=mesh_bed1):
self.node_list_bed1.append(node)
#self.node_list = []
#for node in self.scene.get_nodes(obj=mesh):
# self.node_list.append(node)
self.node_list_2 = []
for node in self.scene.get_nodes(obj=mesh2):
self.node_list_2.append(node)
self.node_list_3 = []
for node in self.scene.get_nodes(obj=mesh3):
self.node_list_3.append(node)
camera_pose = np.eye(4)
# camera_pose[0,0] = -1.0
# camera_pose[1,1] = -1.0
camera_pose[0, 3] = 0.0 # -1.0
camera_pose[1, 3] = 0.0 # -1.0
camera_pose[2, 3] = 4.0
# self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True,
# lighting_intensity=10.,
# point_size=5, run_in_thread=True, viewport_size=(1000, 1000))
# camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
camera = pyrender.OrthographicCamera(xmag=2.0, ymag=2.0)
self.scene.add(camera, pose=camera_pose)
light = pyrender.SpotLight(color=np.ones(3),
intensity=150.0,
innerConeAngle=np.pi / 10.0,
outerConeAngle=np.pi / 2.0)
light_pose = np.copy(camera_pose)
#light_pose[1, 3] = 2.0
light_pose[0, 3] = -1.0
light_pose[2, 3] = 5.0
light_pose2 = np.copy(camera_pose)
light_pose2[0, 3] = 0.5
light_pose2[1, 3] = 0.5
light_pose2[2, 3] = 5.0
# light_pose[1, ]
#self.scene.add(light, pose=light_pose)
#self.scene.add(light, pose=light_pose2)
else:
#self.viewer.render_lock.acquire()
# reset the human mesh
self.scene.remove_node(self.node_list_bed1[0])
self.scene.add(mesh_bed1)
for node in self.scene.get_nodes(obj=mesh_bed1):
self.node_list_bed1[0] = node
# reset the human mesh
#self.scene.remove_node(self.node_list[0])
#self.scene.add(mesh)
#for node in self.scene.get_nodes(obj=mesh):
# self.node_list[0] = node
self.scene.remove_node(self.node_list_2[0])
self.scene.add(mesh2)
for node in self.scene.get_nodes(obj=mesh2):
self.node_list_2[0] = node
self.scene.remove_node(self.node_list_3[0])
self.scene.add(mesh3)
for node in self.scene.get_nodes(obj=mesh3):
self.node_list_3[0] = node
#self.viewer.render_lock.release()
r = pyrender.OffscreenRenderer(2000, 2000)
# r.render(self.scene)
color, depth = r.render(self.scene)
# plt.subplot(1, 2, 1)
plt.axis('off')
plt.imshow(color)
# plt.subplot(1, 2, 2)
# plt.axis('off')
# plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show()
fig.set_size_inches(15., 15.)
fig.tight_layout()
save_name = 'f_hbh_' + '{:04}'.format(self.pic_num)
fig.savefig('/home/henry/Pictures/CVPR2020_study/' + save_name +
'.png',
dpi=300)
#plt.savefig('test2png.png', dpi=100)
self.pic_num += 1
#plt.show()
if self.pic_num == 20:
print "DONE"
time.sleep(1000000)
print "got here"
