def create_2d_point_mesh(width, height, pts_2d, point_size_px):
# create a triangle mesh
def _rot_mat(angle_rad):
return np.array(
[[np.cos(angle_rad), -np.sin(angle_rad)],
[np.sin(angle_rad), np.cos(angle_rad)]],
dtype=np.float32)
if isinstance(pts_2d, list):
pts_2d = np.array(pts_2d, dtype=np.float32)
vertices = np.zeros((3, 3), dtype=np.float32)
point_size_ndc = point_size_px / height
vec = np.array([0, point_size_ndc])
vertices[0, 0:2] = vec
vertices[1, 0:2] = _rot_mat(np.radians(120)) @ vec
vertices[2, 0:2] = _rot_mat(np.radians(-120)) @ vec
mat = MetallicRoughnessMaterial(doubleSided=True,
baseColorFactor=[1.0, 0, 0, 1])
instances = [np.eye(4) for i in range(pts_2d.shape[0])]
for i in range(pts_2d.shape[0]):
instances[i][0, 3] = (2 * pts_2d[i][0] - width) / height
instances[i][1, 3] = pts_2d[i][1] / height * 2 - 1
instances[i][2, 3] = -1
prim = Primitive(positions=vertices,
mode=GLTF.TRIANGLES,
material=mat,
poses=instances)
return Mesh(primitives=[prim])
def _reset_scene(self, scale_factor=1.0):
"""Resets the scene.
Parameters
----------
scale_factor : float
optional scale factor to apply to the image dimensions
"""
# delete scene
if self._scene is not None:
self._scene.clear()
del self._scene
# create scene
scene = Scene()
# setup camera
camera = IntrinsicsCamera(
self.camera.intrinsics.fx,
self.camera.intrinsics.fy,
self.camera.intrinsics.cx,
self.camera.intrinsics.cy,
)
pose_m = self.camera.pose.matrix.copy()
pose_m[:, 1:3] *= -1.0
scene.add(camera, pose=pose_m, name=self.camera.frame)
scene.main_camera_node = next(
iter(scene.get_nodes(name=self.camera.frame))
)
material = MetallicRoughnessMaterial(
baseColorFactor=np.array([1, 1, 1, 1.0]),
metallicFactor=0.2,
roughnessFactor=0.8,
)
# add workspace objects
for obj_key in self.state.workspace_keys:
obj_state = self.state[obj_key]
obj_mesh = Mesh.from_trimesh(obj_state.mesh, material=material)
T_obj_world = obj_state.pose.matrix
scene.add(obj_mesh, pose=T_obj_world, name=obj_key)
# add scene objects
for obj_key in self.state.obj_keys:
obj_state = self.state[obj_key]
obj_mesh = Mesh.from_trimesh(obj_state.mesh, material=material)
T_obj_world = obj_state.pose.matrix
scene.add(obj_mesh, pose=T_obj_world, name=obj_key)
# add light (for color rendering)
light = DirectionalLight(color=np.ones(3), intensity=1.0)
scene.add(light, pose=np.eye(4))
ray_light_nodes = self._create_raymond_lights()
[scene.add_node(rln) for rln in ray_light_nodes]
self._scene = scene
def extract_material(mesh):
pyrender_mat = None
if mesh.visual.kind == 'texture':
trimesh_mat = mesh.visual.material
if isinstance(trimesh_mat, trimesh.visual.texture.PBRMaterial):
pyrender_mat = MetallicRoughnessMaterial(
normalTexture=trimesh_mat.normalTexture,
occlusionTexture=trimesh_mat.occlusionTexture,
emissiveTexture=trimesh_mat.emissiveTexture,
emissiveFactor=trimesh_mat.emissiveFactor,
alphaMode='BLEND',
baseColorFactor=trimesh_mat.baseColorFactor,
baseColorTexture=trimesh_mat.baseColorTexture,
metallicFactor=trimesh_mat.metallicFactor,
metallicRoughnessTexture=trimesh_mat.metallicRoughnessTexture,
doubleSided=trimesh_mat.doubleSided,
alphaCutoff=trimesh_mat.alphaCutoff
)
elif isinstance(trimesh_mat, trimesh.visual.texture.SimpleMaterial):
pyrender_mat = MetallicRoughnessMaterial(
alphaMode='BLEND',
baseColorTexture=trimesh_mat.image
)
return pyrender_mat
def get_random_material(self):
roughness = random.choice(self.roughness_values)
metalness = random.uniform(0.0, 1.0)
if self.use_textures and random.random() < 0.9:
image = self.load_random_image(self.texture_paths)
# base_color = [1.0, 1.0, 1.0]
base_color = np.random.uniform(1.0, 2.0, 3)
else:
base_color = np.random.uniform(0.2, 1.0, 3)
image = None
return MetallicRoughnessMaterial(
alphaMode='BLEND',
roughnessFactor=roughness,
metallicFactor=metalness,
baseColorFactor=base_color,
baseColorTexture=image,
)
def _create_node_from_mesh(mesh,
name=None,
pose=None,
color=None,
material=None,
poses=None,
wireframe=False,
smooth=True):
"""Helper method that creates a pyrender.Node from a trimesh.Trimesh"""
# Create default pose
if pose is None:
pose = np.eye(4)
elif isinstance(pose, RigidTransform):
pose = pose.matrix
# Create vertex colors if needed
if color is not None:
color = np.asanyarray(color, dtype=np.float)
if color.ndim == 1 or len(color) != len(mesh.vertices):
color = np.repeat(color[np.newaxis, :],
len(mesh.vertices),
axis=0)
mesh.visual.vertex_colors = color
if material is None and mesh.visual.kind != 'texture':
if color is not None:
material = None
else:
material = MetallicRoughnessMaterial(baseColorFactor=np.array(
[1.0, 1.0, 1.0, 1.0]),
metallicFactor=0.2,
roughnessFactor=0.8)
m = Mesh.from_trimesh(mesh,
material=material,
poses=poses,
wireframe=wireframe,
smooth=smooth)
return Node(mesh=m, name=name, matrix=pose)
def __init__(self, args):
super(self.__class__, self).__init__()
self.save_params_path = args.save_model_params if args.save_model_params is not None\
else args.fit_result
# Result retrieve
with open(args.fit_result, 'rb') as f:
fitting_results = pickle.load(f, encoding='latin1')
self.cam_rot = fitting_results['camera_rotation'][0]
self.cam_trans = fitting_results['camera_translation'][0]
# Background image setup
self.base_img = load_image(
args.image_path) if args.image_path else None
if self.base_img is not None:
self.canvas_size = np.array(self.base_img.shape[:2][::-1],
dtype=np.float32)
if self.canvas_size[1] > MAX_CANVAS_HEIGHT:
self.canvas_size *= (MAX_CANVAS_HEIGHT /
float(self.canvas_size[1]))
if self.canvas_size[0] > MAX_CANVAS_WIDTH:
self.canvas_size *= (MAX_CANVAS_WIDTH /
float(self.canvas_size[0]))
self.canvas_size = tuple(self.canvas_size.astype(int))
else:
self.canvas_size = None
# Model setup
self.model = self._init_model(fitting_results, args.gender)
# Scene setup
self.scene = Scene(bg_color=[1.0, 1.0, 1.0])
self.material = MetallicRoughnessMaterial(metallicFactor=0.0,
alphaMode='BLEND',
baseColorFactor=(5.0, 5.0,
5.0, 1.0))
im_size = self.canvas_size if self.canvas_size is not None else (
MAX_CANVAS_WIDTH, MAX_CANVAS_HEIGHT)
# self.camera = PerspectiveCamera(yfov=np.pi/3.0,
# aspectRatio=float(self.canvas_size[0])/self.canvas_size[1])
self.camera = IntrinsicsCamera(fx=5000.,
fy=5000.,
cx=im_size[0] / 2,
cy=im_size[1] / 2)
# self.camera = CustomIntrinsicsCamera(fx=5000., fy=5000.,
# cx=im_size[0]/2, cy=im_size[1]/2)
self.camera_params = {
't': self.cam_trans,
'rt': cv2.Rodrigues(self.cam_rot)[0],
'c': np.array(im_size).astype(float) / 2,
'f': np.array((im_size[0], ) * 2) * 1.0,
'k': np.zeros(5),
'frustum': {
'near': self.camera.znear,
'far': self.camera.zfar,
'width': im_size[0],
'height': im_size[1]
}
}
camera_pos = np.eye(4)
# print(self.cam_trans)
# print(self.camera.get_projection_matrix(im_size[0], im_size[1]))
# print(self.camera.cx, self.camera.cy)
camera_pos[:3, :3] = self.cam_rot
self.cam_trans[0] *= -1
camera_pos[:3, 3] = self.cam_trans
slight = SpotLight(color=[1.0, 1.0, 1.0], intensity=10.0)
slight_pos = np.eye(4)
slight_pos[:3, 3] = np.array([0, 0, 5])
self.scene.add(self.camera, pose=camera_pos)
self.scene.add(slight, pose=slight_pos)
self.renderer = OffscreenRenderer(viewport_width=im_size[0],
viewport_height=im_size[1],
point_size=1.0)
# Rest setup
self.setupUi(self)
self._moving = False
self._rotating = False
self._mouse_begin_pos = None
self._loaded_gender = None
self._update_canvas = False
self.camera_widget = Ui_CameraWidget(self.camera,
self.camera_params['frustum'],
self.draw, self.camera_params)
self._bind_actions()
self._update_canvas = True