def render_segmentation_images(self):
"""Renders segmentation masks (modal and amodal) for each object in the state.
"""
full_depth = self.observation
modal_data = np.zeros((full_depth.shape[0], full_depth.shape[1], len(self.obj_keys)), dtype=np.uint8)
amodal_data = np.zeros((full_depth.shape[0], full_depth.shape[1], len(self.obj_keys)), dtype=np.uint8)
renderer = OffscreenRenderer(self.camera.width, self.camera.height)
flags = RenderFlags.DEPTH_ONLY
# Hide all meshes
obj_mesh_nodes = [next(iter(self._scene.get_nodes(name=k))) for k in self.obj_keys]
for mn in self._scene.mesh_nodes:
mn.mesh.is_visible = False
for i, node in enumerate(obj_mesh_nodes):
node.mesh.is_visible = True
depth = renderer.render(self._scene, flags=flags)
amodal_mask = depth > 0.0
modal_mask = np.logical_and(
(np.abs(depth - full_depth) < 1e-6), full_depth > 0.0
)
amodal_data[amodal_mask,i] = np.iinfo(np.uint8).max
modal_data[modal_mask,i] = np.iinfo(np.uint8).max
node.mesh.is_visible = False
renderer.delete()
# Show all meshes
for mn in self._scene.mesh_nodes:
mn.mesh.is_visible = True
return amodal_data, modal_data
def render_camera_image(self, color=True):
"""Render the camera image for the current scene."""
renderer = OffscreenRenderer(self.camera.width, self.camera.height)
flags = RenderFlags.NONE if color else RenderFlags.DEPTH_ONLY
image = renderer.render(self._scene, flags=flags)
renderer.delete()
return image
def render_big_gallery(results_dir,
nb=30,
pts_colors=[0.5, 0.5, 0.5],
draw_text=False):
'''
pts_colors: [0,0,0]
return np array of a big image
'''
cam = PerspectiveCamera(yfov=(YFOV))
cam_pose = CAM_POSE
point_l = PointLight(color=np.ones(3), intensity=POINT_LIGHT_INTENSITY)
scene = Scene(bg_color=np.array([1, 1, 1, 0]))
# cam and light
_ = scene.add(cam, pose=cam_pose)
_ = scene.add(point_l, pose=cam_pose)
input_ply_filenames = get_all_filnames(results_dir, nb)
r = OffscreenRenderer(viewport_width=640 * 2,
viewport_height=480 * 2,
point_size=POINT_SIZE)
pc_pose = PC_POSE
images = []
for _, input_pf in enumerate(input_ply_filenames):
input_pc = read_ply_xyz(input_pf)
colors = np.array(pts_colors)
colors = np.tile(colors, (input_pc.shape[0], 1))
input_pc_node = add_point_cloud_mesh_to_scene(input_pc, scene, pc_pose,
colors)
renderred_color, _ = r.render(scene)
scene.remove_node(input_pc_node)
if draw_text:
im_here = Image.fromarray(renderred_color)
d = ImageDraw.Draw(im_here)
fnt = ImageFont.truetype(
font='/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',
size=100)
d.text((0, 0),
input_pf.split('/')[-1],
fill=(0, 0, 0, 255),
font=fnt)
renderred_color = np.array(im_here)
images.append(renderred_color)
big_gallery = np.concatenate(images, axis=0)
r.delete()
return big_gallery
def __init__(self, render_scene=None, viewport_size=None) -> None:
viewport_width, viewport_height = viewport_size if viewport_size is not None else (
640, 480)
PyRenderBase.__init__(self, render_scene=render_scene)
OffscreenRenderer.__init__(self,
viewport_width=viewport_width,
viewport_height=viewport_height,
point_size=1.0)
def main():
# Initialize colors
color_candidates = []
for n in range(args.num_colors):
hue = n / args.num_colors
saturation = 1
lightness = 1
red, green, blue = colorsys.hsv_to_rgb(hue, saturation, lightness)
color_candidates.append((red, green, blue))
renderer = OffscreenRenderer(viewport_width=args.image_size,
viewport_height=args.image_size)
rad_step = math.pi / 18
total_frames = int(math.pi * 2 / rad_step)
camera_distance = 2
fig = plt.figure(figsize=(3, 3))
ims = []
for num_cubes in range(1, 8):
scene = build_scene(num_cubes, color_candidates)[0]
camera = OrthographicCamera(xmag=0.9, ymag=0.9)
camera_node = Node(camera=camera)
scene.add_node(camera_node)
current_rad = 0
for _ in range(total_frames):
camera_position = np.array(
(math.sin(current_rad), math.sin(math.pi / 6),
math.cos(current_rad)))
camera_position = camera_distance * camera_position / np.linalg.norm(
camera_position)
# Compute yaw and pitch
yaw, pitch = compute_yaw_and_pitch(camera_position)
camera_node.rotation = genearte_camera_quaternion(yaw, pitch)
camera_node.translation = camera_position
# Rendering
flags = RenderFlags.SHADOWS_DIRECTIONAL
if args.anti_aliasing:
flags |= RenderFlags.ANTI_ALIASING
image = renderer.render(scene, flags=flags)[0]
im = plt.imshow(image, interpolation="none", animated=True)
ims.append([im])
current_rad += rad_step
renderer.delete()
ani = animation.ArtistAnimation(fig,
ims,
interval=1 / 24,
blit=True,
repeat_delay=0)
ani.save("shepard_metzler.gif", writer="imagemagick")
class PixelMaskRenderer():
"""Render-ready scene composed of a single object and a single moving camera.
# Arguments
path_OBJ: String containing the path to an OBJ file.
viewport_size: List, specifying [H, W] of rendered image.
y_fov: Float indicating the vertical field of view in radians.
distance: List of floats indicating [max_distance, min_distance]
light: List of floats indicating [max_light, min_light]
top_only: Boolean. If True images are only take from the top.
roll: Float, to sample [-roll, roll] rolls of the Z OpenGL camera axis.
shift: Float, to sample [-shift, shift] to move in X, Y OpenGL axes.
"""
def __init__(self, path_OBJ, viewport_size=(128, 128), y_fov=3.14159 / 4.0,
distance=[0.3, 0.5], light=[0.5, 30], top_only=False,
roll=None, shift=None):
self.distance, self.roll, self.shift = distance, roll, shift
self.light_intensity, self.top_only = light, top_only
self._build_scene(path_OBJ, viewport_size, light, y_fov)
self.renderer = OffscreenRenderer(viewport_size[0], viewport_size[1])
self.flags_RGBA = RenderFlags.RGBA
self.flags_FLAT = RenderFlags.RGBA | RenderFlags.FLAT
self.epsilon = 0.01
def _build_scene(self, path, size, light, y_fov):
self.scene = Scene(bg_color=[0, 0, 0, 0])
self.light = self.scene.add(
DirectionalLight([1.0, 1.0, 1.0], np.mean(light)))
self.camera = self.scene.add(
PerspectiveCamera(y_fov, aspectRatio=np.divide(*size)))
self.pixel_mesh = self.scene.add(color_object(path))
self.mesh = self.scene.add(
Mesh.from_trimesh(trimesh.load(path), smooth=True))
self.world_origin = self.mesh.mesh.centroid
def _sample_parameters(self):
distance = sample_uniformly(self.distance)
camera_origin = sample_point_in_sphere(distance, self.top_only)
camera_origin = random_perturbation(camera_origin, self.epsilon)
light_intensity = sample_uniformly(self.light_intensity)
return camera_origin, light_intensity
def render(self):
camera_origin, intensity = self._sample_parameters()
camera_to_world, world_to_camera = compute_modelview_matrices(
camera_origin, self.world_origin, self.roll, self.shift)
self.light.light.intensity = intensity
self.scene.set_pose(self.camera, camera_to_world)
self.scene.set_pose(self.light, camera_to_world)
self.pixel_mesh.mesh.is_visible = False
image, depth = self.renderer.render(self.scene, self.flags_RGBA)
self.pixel_mesh.mesh.is_visible = True
image, alpha = split_alpha_channel(image)
self.mesh.mesh.is_visible = False
RGB_mask, _ = self.renderer.render(self.scene, self.flags_FLAT)
self.mesh.mesh.is_visible = True
return image, alpha, RGB_mask
def __init__(self, path_OBJ, viewport_size=(128, 128), y_fov=3.14159 / 4.0,
distance=[0.3, 0.5], light=[0.5, 30], top_only=False,
roll=None, shift=None):
self.distance, self.roll, self.shift = distance, roll, shift
self.light_intensity, self.top_only = light, top_only
self._build_scene(path_OBJ, viewport_size, light, y_fov)
self.renderer = OffscreenRenderer(viewport_size[0], viewport_size[1])
self.flags_RGBA = RenderFlags.RGBA
self.flags_FLAT = RenderFlags.RGBA | RenderFlags.FLAT
self.epsilon = 0.01
def render_big_gallery_overlay(dir_1,
dir_2,
pts_color_1=[0.5, 0.5, 0.5],
pts_color_2=[0.5, 0.5, 0.5],
nb=30):
'''
return np array of a big image
'''
cam = PerspectiveCamera(yfov=(YFOV))
cam_pose = CAM_POSE
point_l = PointLight(color=np.ones(3), intensity=POINT_LIGHT_INTENSITY)
scene = Scene(bg_color=np.array([1, 1, 1, 0]))
# cam and light
_ = scene.add(cam, pose=cam_pose)
_ = scene.add(point_l, pose=cam_pose)
input_ply_filenames_1 = get_all_filnames(dir_1, nb)
input_ply_filenames_2 = get_all_filnames(dir_2, nb)
r = OffscreenRenderer(viewport_width=640 * 2,
viewport_height=480 * 2,
point_size=POINT_SIZE)
pc_pose = PC_POSE
images = []
for idx, input_pf in enumerate(input_ply_filenames_1):
input_pc_1 = read_ply_xyz(input_pf)
input_pc_2 = read_ply_xyz(input_ply_filenames_2[idx])
color_1 = np.array(pts_color_1)
color_1 = np.tile(color_1, (input_pc_1.shape[0], 1))
color_2 = np.array(pts_color_2)
color_2 = np.tile(color_2, (input_pc_2.shape[0], 1))
input_pc_node_1 = add_point_cloud_mesh_to_scene(
input_pc_1, scene, pc_pose, color_1)
input_pc_node_2 = add_point_cloud_mesh_to_scene(
input_pc_2, scene, pc_pose, color_2)
renderred_color, _ = r.render(scene)
scene.remove_node(input_pc_node_1)
scene.remove_node(input_pc_node_2)
images.append(renderred_color)
big_gallery = np.concatenate(images, axis=0)
r.delete()
return big_gallery
def __init__(self,
filepath,
viewport_size=(128, 128),
y_fov=3.14159 / 4.0,
distance=0.3,
light=5.0,
top_only=True,
scale=10.0,
roll=None,
shift=None):
self._build_scene(filepath, viewport_size, light, y_fov)
self.distance, self.roll = distance, roll
self.top_only, self.shift, self.scale = top_only, shift, scale
self.renderer = OffscreenRenderer(*viewport_size)
self.RGBA = RenderFlags.RGBA
self.epsilon = 0.01
def render_sensor(
point_set,
render_sensor_path="/Users/macbookpro15/Documents/mujoco_hand_exps/data/sensor_render"
):
"""
pointset: it is collectiono of sensor points for all timesteps
"""
# first take one of the point, subtract the center from it which
# I know is the 0-th position out of the 220 points
# form the mesh from this
if not os.path.exists(render_sensor_path):
os.makedirs(render_sensor_path)
time_steps = len(point_set)
for t in range(time_steps):
sensor = trimesh.load_mesh(
f'../data/mesh_dir/mesh_{t}_out/mc_mesh_out.ply')
sensor_mesh = Mesh.from_trimesh(sensor)
# Light for the scene
direc_l = DirectionalLight(color=np.ones(3), intensity=1.0)
spot_l = SpotLight(color=np.ones(3),
intensity=10.0,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6)
point_l = PointLight(color=np.ones(3), intensity=10.0)
# add camera to the scene
cam = PerspectiveCamera(yfov=(np.pi / 3.0))
cam_pose = np.array([[0.0, -np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.5], [1.0, 0.0, 0.0, 0.0],
[0.0, np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.4], [0.0, 0.0, 0.0, 1.0]])
# create the scene
scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02, 1.0]))
point_mesh_node = scene.add(sensor_mesh)
direc_l_node = scene.add(direc_l, pose=cam_pose)
spot_l_node = scene.add(spot_l, pose=cam_pose)
cam_node = scene.add(cam, pose=cam_pose)
print('rendering the scene offline')
r = OffscreenRenderer(viewport_width=640, viewport_height=480)
color, depth = r.render(scene)
r.delete()
plt.figure()
plt.imshow(color)
plt.savefig(f'{render_sensor_path}/img_{t}.jpg')
def __init__(self, path_OBJ, camera_pose, min_corner, max_corner,
symmetric_transforms, viewport_size=(128, 128),
y_fov=3.14159 / 4.0, light_intensity=[0.5, 30]):
self.light_intensity = light_intensity
self.symmetric_transforms = symmetric_transforms
self.min_corner, self.max_corner = min_corner, max_corner
self.scene = Scene(bg_color=[0, 0, 0, 0])
self.light = self._build_light(light_intensity, camera_pose)
self.camera = self._build_camera(y_fov, viewport_size, camera_pose)
self.pixel_mesh = self.scene.add(color_object(path_OBJ))
self.mesh = self.scene.add(
Mesh.from_trimesh(trimesh.load(path_OBJ), smooth=True))
self.renderer = OffscreenRenderer(viewport_size[0], viewport_size[1])
self.flags_RGBA = RenderFlags.RGBA
self.flags_FLAT = RenderFlags.RGBA | RenderFlags.FLAT
def render_mesh(mesh, h=256, w=256):
"""https://pyrender.readthedocs.io/en/latest/examples/quickstart.html"""
mesh = pyrender.Mesh.from_trimesh(mesh.trimesh())
scene = Scene()
scene.add(mesh)
# z-axis away from the scene, x-axis right, y-axis up
pose = np.eye(4)
pose[2, 3] = 250
# add camera
camera = PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
scene.add(camera, pose=pose)
# add light
# light = DirectionalLight(color=np.ones(3), intensity=5.0)
light = PointLight(color=[1.0, 1.0, 1.0], intensity=2.0)
scene.add(light, pose=pose)
r = OffscreenRenderer(h, w)
color, depth = r.render(scene)
return color
def __init__(self,
filepath,
viewport_size=(128, 128),
y_fov=3.14159 / 4.,
distance=0.30,
top_only=False,
light=5.0,
theta_steps=10,
phi_steps=10):
self.scene = Scene(bg_color=[0, 0, 0])
self.camera = self.scene.add(
PerspectiveCamera(y_fov, aspectRatio=np.divide(*viewport_size)))
self.mesh = self.scene.add(
Mesh.from_trimesh(trimesh.load(filepath), smooth=True))
self.world_origin = self.mesh.mesh.centroid
self.light = self.scene.add(DirectionalLight([1.0, 1.0, 1.0], light))
self.distance = distance
# 0.1 values are to avoid gimbal lock
theta_max = np.pi / 2.0 if top_only else np.pi
self.thetas = np.linspace(0.1, theta_max - 0.1, theta_steps)
self.phis = np.linspace(0.1, 2 * np.pi - 0.1, phi_steps)
self.renderer = OffscreenRenderer(*viewport_size)
self.RGBA = RenderFlags.RGBA
spot_l_node = scene.add(spot_l, pose=cam_pose)
# ==============================================================================
# Using the viewer with a default camera
# ==============================================================================
# needs a screen
# v = Viewer(scene, shadows=True)
# ==============================================================================
# Using the viewer with a pre-specified camera
# ==============================================================================
# needs a screen
# cam_node = scene.add(cam, pose=cam_pose)
# v = Viewer(scene, central_node=drill_node)
# ==============================================================================
# Rendering offscreen from that camera
# ==============================================================================
cam_node = scene.add(cam, pose=cam_pose)
r = OffscreenRenderer(viewport_width=640 * 2, viewport_height=480 * 2)
color, depth = r.render(scene)
r.delete()
import matplotlib.pyplot as plt
plt.figure(figsize=(20, 20))
plt.imshow(color)
plt.show()
class CanonicalScene():
def __init__(self, path_OBJ, camera_pose, min_corner, max_corner,
symmetric_transforms, viewport_size=(128, 128),
y_fov=3.14159 / 4.0, light_intensity=[0.5, 30]):
self.light_intensity = light_intensity
self.symmetric_transforms = symmetric_transforms
self.min_corner, self.max_corner = min_corner, max_corner
self.scene = Scene(bg_color=[0, 0, 0, 0])
self.light = self._build_light(light_intensity, camera_pose)
self.camera = self._build_camera(y_fov, viewport_size, camera_pose)
self.pixel_mesh = self.scene.add(color_object(path_OBJ))
self.mesh = self.scene.add(
Mesh.from_trimesh(trimesh.load(path_OBJ), smooth=True))
self.renderer = OffscreenRenderer(viewport_size[0], viewport_size[1])
self.flags_RGBA = RenderFlags.RGBA
self.flags_FLAT = RenderFlags.RGBA | RenderFlags.FLAT
def _build_light(self, light, pose):
directional_light = DirectionalLight([1.0, 1.0, 1.0], np.mean(light))
directional_light = self.scene.add(directional_light, pose=pose)
return directional_light
def _build_camera(self, y_fov, viewport_size, pose):
aspect_ratio = np.divide(*viewport_size)
camera = PerspectiveCamera(y_fov, aspectRatio=aspect_ratio)
camera = self.scene.add(camera, pose=pose)
return camera
def _sample_parameters(self, min_corner, max_corner):
mesh_transform = sample_affine_transform(min_corner, max_corner)
light_intensity = sample_uniformly(self.light_intensity)
return mesh_transform, light_intensity
def render(self):
mesh_transform, light_intensity = self._sample_parameters(
self.min_corner, self.max_corner)
mesh_rotation = mesh_transform[0:3, 0:3]
canonical_rotation = calculate_canonical_rotation(
mesh_rotation, self.symmetric_transforms)
# mesh_rotation[0:3, 0:3] = canonical_rotation
canonical_rotation = np.dot(mesh_rotation, canonical_rotation)
mesh_rotation[0:3, 0:3] = canonical_rotation
self.scene.set_pose(self.mesh, mesh_transform)
self.scene.set_pose(self.pixel_mesh, mesh_transform)
self.light.light.intensity = light_intensity
self.pixel_mesh.mesh.is_visible = False
image, depth = self.renderer.render(self.scene, self.flags_RGBA)
self.pixel_mesh.mesh.is_visible = True
image, alpha = split_alpha_channel(image)
self.mesh.mesh.is_visible = False
RGB_mask, _ = self.renderer.render(self.scene, self.flags_FLAT)
self.mesh.mesh.is_visible = True
return image, alpha, RGB_mask
def render_symmetries(self):
images, alphas, RGB_masks = [], [], []
for rotation in self.symmetric_transforms:
symmetric_transform = to_affine_matrix(rotation, np.zeros(3))
self.scene.set_pose(self.mesh, symmetric_transform)
self.scene.set_pose(self.pixel_mesh, symmetric_transform)
self.pixel_mesh.mesh.is_visible = False
image, depth = self.renderer.render(self.scene, self.flags_RGBA)
self.pixel_mesh.mesh.is_visible = True
image, alpha = split_alpha_channel(image)
self.mesh.mesh.is_visible = False
RGB_mask, _ = self.renderer.render(self.scene, self.flags_FLAT)
self.mesh.mesh.is_visible = True
images.append(image)
alphas.append(alpha)
RGB_masks.append(RGB_mask[..., 0:3])
images = np.concatenate(images, axis=1)
RGB_masks = np.concatenate(RGB_masks, axis=1)
images = np.concatenate([images, RGB_masks], axis=0)
return images
class Ui_MainWindow(QtWidgets.QMainWindow, Ui_MainWindow_Base):
def __init__(self, args):
super(self.__class__, self).__init__()
self.save_params_path = args.save_model_params if args.save_model_params is not None\
else args.fit_result
# Result retrieve
with open(args.fit_result, 'rb') as f:
fitting_results = pickle.load(f, encoding='latin1')
self.cam_rot = fitting_results['camera_rotation'][0]
self.cam_trans = fitting_results['camera_translation'][0]
# Background image setup
self.base_img = load_image(
args.image_path) if args.image_path else None
if self.base_img is not None:
self.canvas_size = np.array(self.base_img.shape[:2][::-1],
dtype=np.float32)
if self.canvas_size[1] > MAX_CANVAS_HEIGHT:
self.canvas_size *= (MAX_CANVAS_HEIGHT /
float(self.canvas_size[1]))
if self.canvas_size[0] > MAX_CANVAS_WIDTH:
self.canvas_size *= (MAX_CANVAS_WIDTH /
float(self.canvas_size[0]))
self.canvas_size = tuple(self.canvas_size.astype(int))
else:
self.canvas_size = None
# Model setup
self.model = self._init_model(fitting_results, args.gender)
# Scene setup
self.scene = Scene(bg_color=[1.0, 1.0, 1.0])
self.material = MetallicRoughnessMaterial(metallicFactor=0.0,
alphaMode='BLEND',
baseColorFactor=(5.0, 5.0,
5.0, 1.0))
im_size = self.canvas_size if self.canvas_size is not None else (
MAX_CANVAS_WIDTH, MAX_CANVAS_HEIGHT)
# self.camera = PerspectiveCamera(yfov=np.pi/3.0,
# aspectRatio=float(self.canvas_size[0])/self.canvas_size[1])
self.camera = IntrinsicsCamera(fx=5000.,
fy=5000.,
cx=im_size[0] / 2,
cy=im_size[1] / 2)
# self.camera = CustomIntrinsicsCamera(fx=5000., fy=5000.,
# cx=im_size[0]/2, cy=im_size[1]/2)
self.camera_params = {
't': self.cam_trans,
'rt': cv2.Rodrigues(self.cam_rot)[0],
'c': np.array(im_size).astype(float) / 2,
'f': np.array((im_size[0], ) * 2) * 1.0,
'k': np.zeros(5),
'frustum': {
'near': self.camera.znear,
'far': self.camera.zfar,
'width': im_size[0],
'height': im_size[1]
}
}
camera_pos = np.eye(4)
# print(self.cam_trans)
# print(self.camera.get_projection_matrix(im_size[0], im_size[1]))
# print(self.camera.cx, self.camera.cy)
camera_pos[:3, :3] = self.cam_rot
self.cam_trans[0] *= -1
camera_pos[:3, 3] = self.cam_trans
slight = SpotLight(color=[1.0, 1.0, 1.0], intensity=10.0)
slight_pos = np.eye(4)
slight_pos[:3, 3] = np.array([0, 0, 5])
self.scene.add(self.camera, pose=camera_pos)
self.scene.add(slight, pose=slight_pos)
self.renderer = OffscreenRenderer(viewport_width=im_size[0],
viewport_height=im_size[1],
point_size=1.0)
# Rest setup
self.setupUi(self)
self._moving = False
self._rotating = False
self._mouse_begin_pos = None
self._loaded_gender = None
self._update_canvas = False
self.camera_widget = Ui_CameraWidget(self.camera,
self.camera_params['frustum'],
self.draw, self.camera_params)
self._bind_actions()
self._update_canvas = True
def _init_model(self, params, gender):
model = SMPLX(DATA_FOLDER,
gender=gender,
use_pca=False,
body_pose=params['body_pose'],
betas=params['betas'],
left_hand_pose=params['left_hand_pose'],
right_hand_pose=params['right_hand_pose'],
leye_pose=params['leye_pose'],
reye_pose=params['reye_pose'],
expression=params['expression'])
return model
def _bind_actions(self):
self.btn_camera.clicked.connect(lambda: self._show_camera_widget())
for i, val in enumerate(self.model.betas[0]):
self.__dict__['shape_{}'.format(i)].setProperty(
'value', int(10 * val + 50))
for key, shape in self._shapes():
shape.valueChanged[int].connect(
lambda val, k=key: self._update_shape(k, val))
self.set_body_layout()
self.pos_0.valueChanged[float].connect(
lambda val: self._update_position(0, val))
self.pos_1.valueChanged[float].connect(
lambda val: self._update_position(1, val))
self.pos_2.valueChanged[float].connect(
lambda val: self._update_position(2, val))
self.reset_pose.clicked.connect(self._reset_pose)
self.reset_shape.clicked.connect(self._reset_shape)
self.reset_postion.clicked.connect(self._reset_position)
self.to_body.clicked.connect(self.set_body_layout)
self.to_left_hand.clicked.connect(self.set_left_layout)
self.to_right_hand.clicked.connect(self.set_right_layout)
self.save_params.clicked.connect(self._save_params)
self.canvas.wheelEvent = self._zoom
self.canvas.mousePressEvent = self._mouse_begin
self.canvas.mouseMoveEvent = self._move
self.canvas.mouseReleaseEvent = self._mouse_end
self.view_joints.triggered.connect(self.draw)
self.view_joint_ids.triggered.connect(self.draw)
self.view_bones.triggered.connect(self.draw)
self.pos_0.setValue(self.model.transl[0][0])
self.pos_1.setValue(self.model.transl[0][1])
self.pos_2.setValue(self.model.transl[0][2])
def set_body_layout(self):
for key, pose in self._poses():
try:
pose.valueChanged[int].disconnect()
except Exception as e:
pass
for i, val in enumerate(self.model.global_orient.detach().numpy()[0]):
value = int(50 * val / np.pi + 50)
self.__dict__['pose_{}'.format(i)].setProperty('value', value)
for i, val in enumerate(self.model.body_pose.detach().numpy()[0]):
value = int(50 * val / np.pi + 50)
self.__dict__['pose_{}'.format(i + 3)].setProperty('value', value)
for i, val in enumerate(self.model.jaw_pose.detach().numpy()[0]):
value = int(50 * val / np.pi + 50)
self.__dict__['pose_{}'.format(i + 66)].setProperty('value', value)
for key, pose in self._poses():
pose.valueChanged[int].connect(
lambda val, k=key: self._update_pose_body(k, val))
self.label_3.setText(_translate("MainWindow", "Global Orientation"))
self.label_4.setText(_translate("MainWindow", "Hip L"))
self.label_5.setText(_translate("MainWindow", "Hip R"))
self.label_6.setText(_translate("MainWindow", "Lower Spine"))
self.label_7.setText(_translate("MainWindow", "Knee L"))
self.label_8.setText(_translate("MainWindow", "Knee R"))
self.label_9.setText(_translate("MainWindow", "Mid Spine"))
self.label_10.setText(_translate("MainWindow", "Ankle L"))
self.label_11.setText(_translate("MainWindow", "Ankle R"))
self.label_12.setText(_translate("MainWindow", "Upper Spine"))
self.label_13.setText(_translate("MainWindow", "Toes L"))
self.label_14.setText(_translate("MainWindow", "Toes R"))
self.label_15.setText(_translate("MainWindow", "Neck"))
self.label_16.setText(_translate("MainWindow", "Clavicle L"))
self.label_17.setText(_translate("MainWindow", "Clavicle R"))
self.label_18.setText(_translate("MainWindow", "Head"))
self.label_19.setText(_translate("MainWindow", "Shoulder L"))
self.label_20.setText(_translate("MainWindow", "Shoulder R"))
self.label_21.setText(_translate("MainWindow", "Elbow L"))
self.label_22.setText(_translate("MainWindow", "Elbow R"))
self.label_23.setText(_translate("MainWindow", "Wrist L"))
self.label_24.setText(_translate("MainWindow", "Wrist R"))
self.label_25.setText(_translate("MainWindow", "Jaw"))
self.label_26.setText(_translate("MainWindow", "-"))
def set_left_layout(self):
for key, pose in self._poses():
try:
pose.valueChanged[int].disconnect()
except Exception as e:
pass
for i, val in enumerate(self.model.left_hand_pose.detach().numpy()[0]):
value = int(50 * val / np.pi + 50)
self.__dict__['pose_{}'.format(i)].setProperty('value', value)
for i, val in enumerate(self.model.leye_pose.detach().numpy()[0]):
value = int(50 * val / np.pi + 50)
self.__dict__['pose_{}'.format(i + 45)].setProperty('value', value)
for key, pose in self._poses():
pose.valueChanged[int].connect(
lambda val, k=key: self._update_pose_left(k, val))
self.label_3.setText(_translate("MainWindow", "Index1 L"))
self.label_4.setText(_translate("MainWindow", "Index2 L"))
self.label_5.setText(_translate("MainWindow", "Index3 L"))
self.label_6.setText(_translate("MainWindow", "Middle1 L"))
self.label_7.setText(_translate("MainWindow", "Middle2 L"))
self.label_8.setText(_translate("MainWindow", "Middle3 L"))
self.label_9.setText(_translate("MainWindow", "Little1 L"))
self.label_10.setText(_translate("MainWindow", "Little2 L"))
self.label_11.setText(_translate("MainWindow", "Little3 L"))
self.label_12.setText(_translate("MainWindow", "Ring1 L"))
self.label_13.setText(_translate("MainWindow", "Ring2 L"))
self.label_14.setText(_translate("MainWindow", "Ring3 L"))
self.label_15.setText(_translate("MainWindow", "Thumb1 L"))
self.label_16.setText(_translate("MainWindow", "Thumb2 L"))
self.label_17.setText(_translate("MainWindow", "Thumb3 L"))
self.label_18.setText(_translate("MainWindow", "Eye Left"))
self.label_19.setText(_translate("MainWindow", "-"))
self.label_20.setText(_translate("MainWindow", "-"))
self.label_21.setText(_translate("MainWindow", "-"))
self.label_22.setText(_translate("MainWindow", "-"))
self.label_23.setText(_translate("MainWindow", "-"))
self.label_24.setText(_translate("MainWindow", "-"))
self.label_25.setText(_translate("MainWindow", "-"))
self.label_26.setText(_translate("MainWindow", "-"))
def set_right_layout(self):
for key, pose in self._poses():
try:
pose.valueChanged[int].disconnect()
except Exception as e:
pass
for i, val in enumerate(
self.model.right_hand_pose.detach().numpy()[0]):
value = int(50 * val / np.pi + 50)
self.__dict__['pose_{}'.format(i)].setProperty('value', value)
for i, val in enumerate(self.model.reye_pose.detach().numpy()[0]):
value = int(50 * val / np.pi + 50)
self.__dict__['pose_{}'.format(i + 45)].setProperty('value', value)
for key, pose in self._poses():
pose.valueChanged[int].connect(
lambda val, k=key: self._update_pose_right(k, val))
self.label_3.setText(_translate("MainWindow", "Index1 R"))
self.label_4.setText(_translate("MainWindow", "Index2 R"))
self.label_5.setText(_translate("MainWindow", "Index3 R"))
self.label_6.setText(_translate("MainWindow", "Middle1 R"))
self.label_7.setText(_translate("MainWindow", "Middle2 R"))
self.label_8.setText(_translate("MainWindow", "Middle3 R"))
self.label_9.setText(_translate("MainWindow", "Little1 R"))
self.label_10.setText(_translate("MainWindow", "Little2 R"))
self.label_11.setText(_translate("MainWindow", "Little3 R"))
self.label_12.setText(_translate("MainWindow", "Ring1 R"))
self.label_13.setText(_translate("MainWindow", "Ring2 R"))
self.label_14.setText(_translate("MainWindow", "Ring3 R"))
self.label_15.setText(_translate("MainWindow", "Thumb1 R"))
self.label_16.setText(_translate("MainWindow", "Thumb2 R"))
self.label_17.setText(_translate("MainWindow", "Thumb3 R"))
self.label_18.setText(_translate("MainWindow", "Eye Left"))
self.label_19.setText(_translate("MainWindow", "-"))
self.label_20.setText(_translate("MainWindow", "-"))
self.label_21.setText(_translate("MainWindow", "-"))
self.label_22.setText(_translate("MainWindow", "-"))
self.label_23.setText(_translate("MainWindow", "-"))
self.label_24.setText(_translate("MainWindow", "-"))
self.label_25.setText(_translate("MainWindow", "-"))
self.label_26.setText(_translate("MainWindow", "-"))
def _save_params(self):
params = {
'camera_rotation': np.expand_dims(self.cam_rot, axis=0),
'camera_translation': np.expand_dims(self.cam_trans, axis=0),
'body_pose': self.model.body_pose.detach().numpy(),
'left_hand_pose': self.model.left_hand_pose.detach().numpy(),
'right_hand_pose': self.model.right_hand_pose.detach().numpy(),
'jaw_pose': self.model.jaw_pose.detach().numpy(),
'reye_pose': self.model.reye_pose.detach().numpy(),
'leye_pose': self.model.leye_pose.detach().numpy(),
'expression': self.model.expression.detach().numpy(),
'global_orient': self.model.global_orient.detach().numpy(),
'betas': self.model.betas.detach().numpy()
}
with open(self.save_params_path, 'wb') as f:
pickle.dump(params, f, protocol=2)
def showEvent(self, event):
self._init_camera()
super(self.__class__, self).showEvent(event)
def resizeEvent(self, event):
self._init_camera()
super(self.__class__, self).resizeEvent(event)
def closeEvent(self, event):
self.camera_widget.close()
super(self.__class__, self).closeEvent(event)
def draw(self):
if self._update_canvas:
img = self._overlay_image()
if self.canvas_size is not None:
img = cv2.resize(img, self.canvas_size)
self.canvas.setScaledContents(False)
self.canvas.setPixmap(self._to_pixmap(img))
def _init_camera(self):
self.draw()
def _overlay_image(self):
for node in self.scene.get_nodes():
if node.name == 'body_mesh':
self.scene.remove_node(node)
break
rectified_orient = self.model.global_orient.detach().clone()
rectified_orient[0][0] += np.pi
model_obj = self.model(global_orient=rectified_orient)
out_mesh = Trimesh(model_obj.vertices.detach().numpy().squeeze(),
self.model.faces.squeeze())
rot = trimesh.transformations.rotation_matrix(np.radians(180),
[1, 0, 0])
out_mesh.apply_transform(rot)
mesh = Mesh.from_trimesh(out_mesh, material=self.material)
self.scene.add(mesh, name='body_mesh')
rendered, _ = self.renderer.render(self.scene)
if self.base_img is not None:
img = self.base_img.copy()
else:
img = np.uint8(rendered)
img_mask = (rendered != 255.0).sum(axis=2) == 3
img[img_mask] = rendered[img_mask].astype(np.uint8)
img = np.concatenate(
(img, img_mask[:, :, np.newaxis].astype(np.uint8)), axis=2)
return img
def _zoom(self, event):
delta = -event.angleDelta().y() / 1200.0
self.camera_widget.pos_2.setValue(self.camera_widget.pos_2.value() +
delta)
def _mouse_begin(self, event):
if event.button() == 1:
self._moving = True
elif event.button() == 2:
self._rotating = True
self._mouse_begin_pos = event.pos()
def _mouse_end(self, event):
self._moving = False
self._rotating = False
def _move(self, event):
if self._moving:
delta = event.pos() - self._mouse_begin_pos
self.camera_widget.pos_0.setValue(
self.camera_widget.pos_0.value() + delta.x() / 500.)
self.camera_widget.pos_1.setValue(
self.camera_widget.pos_1.value() + delta.y() / 500.)
self._mouse_begin_pos = event.pos()
elif self._rotating:
delta = event.pos() - self._mouse_begin_pos
self.camera_widget.rot_0.setValue(
self.camera_widget.rot_0.value() + delta.y() / 300.)
self.camera_widget.rot_1.setValue(
self.camera_widget.rot_1.value() - delta.x() / 300.)
self._mouse_begin_pos = event.pos()
def _show_camera_widget(self):
self.camera_widget.show()
self.camera_widget.raise_()
def _update_shape(self, id, val):
val = (val - 50) / 50.0 * 5.0
self.model.betas[0][id] = val
self.draw()
def _reset_shape(self):
self._update_canvas = False
for key, shape in self._shapes():
# value = self.model_params['betas'][key] / 5.0 * 50.0 + 50
shape.setValue(50)
self._update_canvas = True
self.draw()
def _update_pose_body(self, id, val):
val = (val - 50) / 50.0 * np.pi
if id < 3:
self.model.global_orient[0, id] = val
elif id > 65:
if id < 69:
self.model.jaw_pose[0, id - 66] = val
else:
self.model.body_pose[0, id - 3] = val
self.draw()
def _update_pose_left(self, id, val):
val = (val - 50) / 50.0 * np.pi
if id < 45:
self.model.left_hand_pose[0][id] = val
elif id > 44 and id < 48:
self.model.leye_pose[0][id - 45] = val
self.draw()
def _update_pose_right(self, id, val):
val = (val - 50) / 50.0 * np.pi
if id < 45:
self.model.right_hand_pose[0][id] = val
elif id > 44 and id < 48:
self.model.reye_pose[0][id - 45] = val
self.draw()
def _reset_pose(self):
self._update_canvas = False
for key, pose in self._poses():
pose.setValue(50)
self._update_canvas = True
self.draw()
def _update_position(self, id, val):
self.model.transl[0][id] = val
self.draw()
def _reset_position(self):
self._update_canvas = False
self.pos_0.setValue(self.model.transl[0][0])
self.pos_1.setValue(self.model.transl[0][1])
self.pos_2.setValue(self.model.transl[0][2])
self._update_canvas = True
self.draw()
def _poses(self):
return enumerate(
[self.__dict__['pose_{}'.format(i)] for i in range(72)])
def _shapes(self):
return enumerate(
[self.__dict__['shape_{}'.format(i)] for i in range(10)])
@staticmethod
def _to_pixmap(im):
if im.dtype == np.float32 or im.dtype == np.float64:
im = np.uint8(im * 255)
if len(im.shape) < 3 or im.shape[-1] == 1:
im = cv2.cvtColor(im, cv2.COLOR_GRAY2RGB)
else:
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
qimg = QtGui.QImage(im, im.shape[1], im.shape[0], im.strides[0],
QtGui.QImage.Format_RGB888)
return QtGui.QPixmap(qimg)
def __init__(self, args):
super(self.__class__, self).__init__()
self.save_params_path = args.save_model_params if args.save_model_params is not None\
else args.fit_result
# Result retrieve
with open(args.fit_result, 'rb') as f:
fitting_results = pickle.load(f, encoding='latin1')
self.cam_rot = fitting_results['camera_rotation'][0]
self.cam_trans = fitting_results['camera_translation'][0]
# Background image setup
self.base_img = load_image(
args.image_path) if args.image_path else None
if self.base_img is not None:
self.canvas_size = np.array(self.base_img.shape[:2][::-1],
dtype=np.float32)
if self.canvas_size[1] > MAX_CANVAS_HEIGHT:
self.canvas_size *= (MAX_CANVAS_HEIGHT /
float(self.canvas_size[1]))
if self.canvas_size[0] > MAX_CANVAS_WIDTH:
self.canvas_size *= (MAX_CANVAS_WIDTH /
float(self.canvas_size[0]))
self.canvas_size = tuple(self.canvas_size.astype(int))
else:
self.canvas_size = None
# Model setup
self.model = self._init_model(fitting_results, args.gender)
# Scene setup
self.scene = Scene(bg_color=[1.0, 1.0, 1.0])
self.material = MetallicRoughnessMaterial(metallicFactor=0.0,
alphaMode='BLEND',
baseColorFactor=(5.0, 5.0,
5.0, 1.0))
im_size = self.canvas_size if self.canvas_size is not None else (
MAX_CANVAS_WIDTH, MAX_CANVAS_HEIGHT)
# self.camera = PerspectiveCamera(yfov=np.pi/3.0,
# aspectRatio=float(self.canvas_size[0])/self.canvas_size[1])
self.camera = IntrinsicsCamera(fx=5000.,
fy=5000.,
cx=im_size[0] / 2,
cy=im_size[1] / 2)
# self.camera = CustomIntrinsicsCamera(fx=5000., fy=5000.,
# cx=im_size[0]/2, cy=im_size[1]/2)
self.camera_params = {
't': self.cam_trans,
'rt': cv2.Rodrigues(self.cam_rot)[0],
'c': np.array(im_size).astype(float) / 2,
'f': np.array((im_size[0], ) * 2) * 1.0,
'k': np.zeros(5),
'frustum': {
'near': self.camera.znear,
'far': self.camera.zfar,
'width': im_size[0],
'height': im_size[1]
}
}
camera_pos = np.eye(4)
# print(self.cam_trans)
# print(self.camera.get_projection_matrix(im_size[0], im_size[1]))
# print(self.camera.cx, self.camera.cy)
camera_pos[:3, :3] = self.cam_rot
self.cam_trans[0] *= -1
camera_pos[:3, 3] = self.cam_trans
slight = SpotLight(color=[1.0, 1.0, 1.0], intensity=10.0)
slight_pos = np.eye(4)
slight_pos[:3, 3] = np.array([0, 0, 5])
self.scene.add(self.camera, pose=camera_pos)
self.scene.add(slight, pose=slight_pos)
self.renderer = OffscreenRenderer(viewport_width=im_size[0],
viewport_height=im_size[1],
point_size=1.0)
# Rest setup
self.setupUi(self)
self._moving = False
self._rotating = False
self._mouse_begin_pos = None
self._loaded_gender = None
self._update_canvas = False
self.camera_widget = Ui_CameraWidget(self.camera,
self.camera_params['frustum'],
self.draw, self.camera_params)
self._bind_actions()
self._update_canvas = True
def main():
try:
os.makedirs(args.output_directory)
except:
pass
last_file_number = args.initial_file_number + args.total_scenes // args.num_scenes_per_file - 1
initial_file_number = args.initial_file_number
if os.path.isdir(args.output_directory):
files = os.listdir(args.output_directory)
for name in files:
number = int(name.replace(".h5", ""))
if number > last_file_number:
continue
if number < args.initial_file_number:
continue
if number < initial_file_number:
continue
initial_file_number = number + 1
total_scenes_to_render = args.total_scenes - args.num_scenes_per_file * (
initial_file_number - args.initial_file_number)
assert args.num_scenes_per_file <= total_scenes_to_render
# Colors
colors = []
for n in range(args.num_colors):
hue = n / args.num_colors
saturation = 1
lightness = 1
red, green, blue = colorsys.hsv_to_rgb(hue, saturation, lightness)
colors.append(np.array((red, green, blue, 1)))
renderer = OffscreenRenderer(viewport_width=args.image_size,
viewport_height=args.image_size)
archiver = Archiver(
directory=args.output_directory,
num_scenes_per_file=args.num_scenes_per_file,
image_size=(args.image_size, args.image_size),
num_observations_per_scene=args.num_observations_per_scene,
initial_file_number=initial_file_number)
for scene_index in tqdm(range(total_scenes_to_render)):
scene = build_scene(floor_textures,
wall_textures,
fix_light_position=args.fix_light_position)
place_objects(scene,
colors,
objects,
max_num_objects=args.max_num_objects,
discrete_position=args.discrete_position,
rotate_object=args.rotate_object)
camera_distance = 4.5
camera = PerspectiveCamera(yfov=math.pi / 4)
camera_node = Node(camera=camera, translation=np.array([0, 1, 1]))
scene.add_node(camera_node)
scene_data = SceneData((args.image_size, args.image_size),
args.num_observations_per_scene)
for observation_index in range(args.num_observations_per_scene):
rand_position_xz = np.random.normal(size=2)
rand_position_xz = camera_distance * rand_position_xz / np.linalg.norm(
rand_position_xz)
# Compute yaw and pitch
camera_direction = np.array(
[rand_position_xz[0], 0, rand_position_xz[1]])
yaw, pitch = compute_yaw_and_pitch(camera_direction)
camera_node.rotation = genearte_camera_quaternion(yaw, pitch)
camera_position = np.array(
[rand_position_xz[0], 1, rand_position_xz[1]])
camera_node.translation = camera_position
# Rendering
flags = RenderFlags.SHADOWS_DIRECTIONAL
if args.anti_aliasing:
flags |= RenderFlags.ANTI_ALIASING
image = renderer.render(scene, flags=flags)[0]
scene_data.add(image, camera_position, math.cos(yaw),
math.sin(yaw), math.cos(pitch), math.sin(pitch))
if args.visualize:
plt.clf()
plt.imshow(image)
plt.pause(1e-10)
archiver.add(scene_data)
renderer.delete()
# Using the viewer with a default camera
# ==============================================================================
v = Viewer(scene, shadows=True)
# ==============================================================================
# Using the viewer with a pre-specified camera
# ==============================================================================
cam_node = scene.add(cam, pose=cam_pose)
v = Viewer(scene, central_node=drill_node)
# ==============================================================================
# Rendering offscreen from that camera
# ==============================================================================
r = OffscreenRenderer(viewport_width=640 * 2, viewport_height=480 * 2)
color, depth = r.render(scene)
import matplotlib.pyplot as plt
plt.figure()
plt.imshow(color)
plt.show()
# ==============================================================================
# Segmask rendering
# ==============================================================================
nm = {node: 20 * (i + 1) for i, node in enumerate(scene.mesh_nodes)}
seg = r.render(scene, RenderFlags.SEG, nm)[0]
plt.figure()
def main():
# Load MNIST images
mnist_images = load_mnist_images()
renderer = OffscreenRenderer(viewport_width=args.image_size,
viewport_height=args.image_size)
plt.tight_layout()
fig = plt.figure(figsize=(6, 3))
axis_perspective = fig.add_subplot(1, 2, 1)
axis_orthogonal = fig.add_subplot(1, 2, 2)
ims = []
scene = build_scene(floor_textures,
wall_textures,
fix_light_position=args.fix_light_position)
place_dice(scene,
mnist_images,
discrete_position=args.discrete_position,
rotate_dice=args.rotate_dice)
camera_distance = 5
perspective_camera = PerspectiveCamera(yfov=math.pi / 4)
perspective_camera_node = Node(camera=perspective_camera,
translation=np.array([0, 1, 1]))
orthographic_camera = OrthographicCamera(xmag=3, ymag=3)
orthographic_camera_node = Node(camera=orthographic_camera)
rad_step = math.pi / 36
total_frames = int(math.pi * 2 / rad_step)
current_rad = 0
for _ in range(total_frames):
scene.add_node(perspective_camera_node)
# Perspective camera
camera_xz = camera_distance * np.array(
(math.sin(current_rad), math.cos(current_rad)))
# Compute yaw and pitch
camera_direction = np.array([camera_xz[0], 0, camera_xz[1]])
yaw, pitch = compute_yaw_and_pitch(camera_direction)
perspective_camera_node.rotation = genearte_camera_quaternion(
yaw, pitch)
camera_position = np.array([camera_xz[0], 1, camera_xz[1]])
perspective_camera_node.translation = camera_position
# Rendering
flags = RenderFlags.SHADOWS_DIRECTIONAL
if args.anti_aliasing:
flags |= RenderFlags.ANTI_ALIASING
image = renderer.render(scene, flags=flags)[0]
im1 = axis_perspective.imshow(image,
interpolation="none",
animated=True)
scene.remove_node(perspective_camera_node)
# Orthographic camera
scene.add_node(orthographic_camera_node)
camera_direction = camera_distance * np.array(
(math.sin(current_rad), math.sin(
math.pi / 6), math.cos(current_rad)))
yaw, pitch = compute_yaw_and_pitch(camera_direction)
orthographic_camera_node.rotation = genearte_camera_quaternion(
yaw, pitch)
orthographic_camera_node.translation = np.array(
[camera_direction[0], 4, camera_direction[2]])
image = renderer.render(scene, flags=flags)[0]
im2 = axis_orthogonal.imshow(image,
interpolation="none",
animated=True)
ims.append([im1, im2])
plt.pause(1e-8)
current_rad += rad_step
scene.remove_node(orthographic_camera_node)
ani = animation.ArtistAnimation(fig,
ims,
interval=1 / 24,
blit=True,
repeat_delay=0)
filename = "mnist_dice"
if args.discrete_position:
filename += "_discrete_position"
if args.rotate_dice:
filename += "_rotate_dice"
if args.fix_light_position:
filename += "_fix_light_position"
filename += ".gif"
ani.save(filename, writer="imagemagick")
def main():
try:
os.makedirs(args.output_directory)
except:
pass
# Colors
colors = []
for n in range(args.num_colors):
hue = n / args.num_colors
saturation = 1
lightness = 1
red, green, blue = colorsys.hsv_to_rgb(hue, saturation, lightness)
colors.append(np.array((red, green, blue, 1)))
renderer = OffscreenRenderer(viewport_width=args.image_size,
viewport_height=args.image_size)
archiver = Archiver(
directory=args.output_directory,
total_scenes=args.total_scenes,
num_scenes_per_file=min(args.num_scenes_per_file, args.total_scenes),
image_size=(args.image_size, args.image_size),
num_observations_per_scene=args.num_observations_per_scene,
initial_file_number=args.initial_file_number)
for scene_index in tqdm(range(args.total_scenes)):
scene = build_scene(floor_textures,
wall_textures,
fix_light_position=args.fix_light_position)
place_objects(scene,
colors,
objects,
max_num_objects=args.max_num_objects,
discrete_position=args.discrete_position,
rotate_object=args.rotate_object)
camera_distance = 3
camera = PerspectiveCamera(yfov=math.pi / 4)
camera_node = Node(camera=camera, translation=np.array([0, 1, 1]))
scene.add_node(camera_node)
scene_data = SceneData((args.image_size, args.image_size),
args.num_observations_per_scene)
for observation_index in range(args.num_observations_per_scene):
# Sample camera position
rand_position_xz = np.random.uniform(-3, 3, size=2)
rand_lookat_xz = np.random.uniform(-6, 6, size=2)
camera_position = np.array(
[rand_position_xz[0], 1, rand_position_xz[1]])
# Compute yaw and pitch
camera_direction = rand_position_xz - rand_lookat_xz
camera_direction = np.array(
[camera_direction[0], 0, camera_direction[1]])
yaw, pitch = compute_yaw_and_pitch(camera_direction)
camera_node.rotation = genearte_camera_quaternion(yaw, pitch)
camera_node.translation = camera_position
# Rendering
flags = RenderFlags.SHADOWS_DIRECTIONAL
if args.anti_aliasing:
flags |= RenderFlags.ANTI_ALIASING
image = renderer.render(scene, flags=flags)[0]
scene_data.add(image, camera_position, math.cos(yaw),
math.sin(yaw), math.cos(pitch), math.sin(pitch))
if args.visualize:
plt.clf()
plt.imshow(image)
plt.pause(1e-10)
archiver.add(scene_data)
renderer.delete()
def main():
try:
os.makedirs(args.output_directory)
except:
pass
last_file_number = args.initial_file_number + args.total_scenes // args.num_scenes_per_file - 1
initial_file_number = args.initial_file_number
if os.path.isdir(args.output_directory):
files = os.listdir(args.output_directory)
for name in files:
number = int(name.replace(".h5", ""))
if number > last_file_number:
continue
if number < args.initial_file_number:
continue
if number < initial_file_number:
continue
initial_file_number = number + 1
total_scenes_to_render = args.total_scenes - args.num_scenes_per_file * (
initial_file_number - args.initial_file_number)
assert args.num_scenes_per_file <= total_scenes_to_render
# Initialize colors
color_candidates = []
for n in range(args.num_colors):
hue = n / args.num_colors
saturation = 1
lightness = 1
red, green, blue = colorsys.hsv_to_rgb(hue, saturation, lightness)
color_candidates.append((red, green, blue))
scene, cube_nodes = build_scene(args.num_cubes, color_candidates)
camera = OrthographicCamera(xmag=0.9, ymag=0.9)
camera_node = Node(camera=camera)
scene.add_node(camera_node)
renderer = OffscreenRenderer(
viewport_width=args.image_size, viewport_height=args.image_size)
archiver = Archiver(
directory=args.output_directory,
num_scenes_per_file=args.num_scenes_per_file,
image_size=(args.image_size, args.image_size),
num_observations_per_scene=args.num_observations_per_scene,
initial_file_number=initial_file_number)
for scene_index in tqdm(range(total_scenes_to_render)):
camera_distance = 2
scene_data = SceneData((args.image_size, args.image_size),
args.num_observations_per_scene)
for observation_index in range(args.num_observations_per_scene):
# Generate random point on a sphere
camera_position = np.random.normal(size=3)
camera_position = camera_distance * camera_position / np.linalg.norm(
camera_position)
# Compute yaw and pitch
yaw, pitch = compute_yaw_and_pitch(camera_position)
camera_node.rotation = genearte_camera_quaternion(yaw, pitch)
camera_node.translation = camera_position
# Rendering
flags = RenderFlags.SHADOWS_DIRECTIONAL
if args.anti_aliasing:
flags |= RenderFlags.ANTI_ALIASING
image = renderer.render(scene, flags=flags)[0]
scene_data.add(image, camera_position, math.cos(yaw),
math.sin(yaw), math.cos(pitch), math.sin(pitch))
if args.visualize:
plt.clf()
plt.imshow(image)
plt.pause(1e-10)
archiver.add(scene_data)
# Change cube color and position
update_cube_color_and_position(cube_nodes, color_candidates)
# Transfer changes to the vertex buffer on gpu
udpate_vertex_buffer(cube_nodes)
renderer.delete()
if inside(e):
if not inside(s):
outputList.append(computeIntersection())
outputList.append(e)
elif inside(s):
outputList.append(computeIntersection())
s = e
cp1 = cp2
return (outputList)
# View the scene
view_render_flags = {'cull_faces': False}
v = Viewer(scene,
viewport_size=(width, height),
render_flags=view_render_flags)
r = OffscreenRenderer(viewport_width=width, viewport_height=height)
color, depth = r.render(scene, flags=RenderFlags.SKIP_CULL_FACES)
# color = cvtColor(color, COLOR_RGB2BGR)
def draw_point(img, imgpts):
for point in imgpts:
img = circle(img, tuple(point), 4, (255, 0, 0), 2)
return img
# draw the computed (x,y) coords on the image
color = draw_point(color, points)
plt.imshow(color)
plt.show()
#==============================================================================
# Attempt to convert 3D to 2D position
#==============================================================================
# View the scene
view_render_flags = {'cull_faces': False}
v = Viewer(scene, render_flags=view_render_flags)
#==============================================================================
# Rendering offscreen from that camera
#==============================================================================
dimensions = 600
# dimensions = 1280
r = OffscreenRenderer(viewport_width=dimensions, viewport_height=dimensions)
# Generate dataset ?
question_string = "Generate chess board dataset? (select no because it's already there) [y/n]: "
choice = input(question_string)
accepted_inputs = ['y', 'n']
while (not (choice in accepted_inputs)):
choice = input(question_string)
# Parameters
# alpha = 0
# beta = np.pi/8
iterations = 20
# distance = 0.7
# force_height = 0.0
mode = CameraPose.SIDE
proj = cam.get_projection_matrix()
mvp = proj @ np.linalg.inv(cam_pose)
inv_mvp = np.linalg.inv(mvp)
#==============================================================================
# Scene creation
#==============================================================================
scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02, 1.0]))
cam_node = scene.add(cam, pose=cam_pose)
scene.main_camera_node = cam_node
#==============================================================================
drill_node = scene.add(drill_mesh)
r = OffscreenRenderer(viewport_width=640, viewport_height=480)
rf = pyrender.RenderFlags.NONE
rf |= pyrender.RenderFlags.USE_RAW_DEPTH
color, raw_depth = r.render(scene, flags=rf)
r.delete()
# unproject to get point cloud
pcd = unproj(inv_mvp, raw_depth)
#==============================================================================
#------------------------------------------------------------------------------
# Creating meshes from point clouds
#------------------------------------------------------------------------------
points_mesh = Mesh.from_points(pcd, colors=np.array((0.0, 1.0, 0.0, 1.0)))
pcd_node = scene.add(points_mesh)
def render_camera_image(self):
""" Render the camera image for the current scene. """
renderer = OffscreenRenderer(self.camera.width, self.camera.height)
image = renderer.render(self._scene, flags=RenderFlags.DEPTH_ONLY)
renderer.delete()
return image
class DictionaryView():
"""Render-ready scene composed of a single object and a single moving camera.
# Arguments
filepath: String containing the path to an OBJ file.
viewport_size: List, specifying [H, W] of rendered image.
y_fov: Float indicating the vertical field of view in radians.
distance: List of floats indicating [max_distance, min_distance]
top_only: Boolean. If True images are only take from the top.
light: List of floats indicating [max_light, min_light]
"""
def __init__(self,
filepath,
viewport_size=(128, 128),
y_fov=3.14159 / 4.,
distance=0.30,
top_only=False,
light=5.0,
theta_steps=10,
phi_steps=10):
self.scene = Scene(bg_color=[0, 0, 0])
self.camera = self.scene.add(
PerspectiveCamera(y_fov, aspectRatio=np.divide(*viewport_size)))
self.mesh = self.scene.add(
Mesh.from_trimesh(trimesh.load(filepath), smooth=True))
self.world_origin = self.mesh.mesh.centroid
self.light = self.scene.add(DirectionalLight([1.0, 1.0, 1.0], light))
self.distance = distance
# 0.1 values are to avoid gimbal lock
theta_max = np.pi / 2.0 if top_only else np.pi
self.thetas = np.linspace(0.1, theta_max - 0.1, theta_steps)
self.phis = np.linspace(0.1, 2 * np.pi - 0.1, phi_steps)
self.renderer = OffscreenRenderer(*viewport_size)
self.RGBA = RenderFlags.RGBA
def render(self):
dictionary_data = []
for theta_arg, theta in enumerate(self.thetas):
for phi_arg, phi in enumerate(self.phis):
x = self.distance * np.sin(theta) * np.cos(phi)
y = self.distance * np.sin(theta) * np.sin(phi)
z = self.distance * np.cos(theta)
matrices = compute_modelview_matrices(np.array([x, z, y]),
self.world_origin)
camera_to_world, world_to_camera = matrices
self.scene.set_pose(self.camera, camera_to_world)
self.scene.set_pose(self.light, camera_to_world)
camera_to_world = camera_to_world.flatten()
world_to_camera = world_to_camera.flatten()
image, depth = self.renderer.render(self.scene,
flags=self.RGBA)
image, alpha = split_alpha_channel(image)
matrices = np.vstack([world_to_camera, camera_to_world])
sample = {
'image': image,
'alpha': alpha,
'depth': depth,
'matrices': matrices
}
dictionary_data.append(sample)
return dictionary_data
def test_offscreen_renderer(tmpdir):
# Fuze trimesh
fuze_trimesh = trimesh.load('examples/models/fuze.obj')
fuze_mesh = Mesh.from_trimesh(fuze_trimesh)
# Drill trimesh
drill_trimesh = trimesh.load('examples/models/drill.obj')
drill_mesh = Mesh.from_trimesh(drill_trimesh)
drill_pose = np.eye(4)
drill_pose[0, 3] = 0.1
drill_pose[2, 3] = -np.min(drill_trimesh.vertices[:, 2])
# Wood trimesh
wood_trimesh = trimesh.load('examples/models/wood.obj')
wood_mesh = Mesh.from_trimesh(wood_trimesh)
# Water bottle trimesh
bottle_gltf = trimesh.load('examples/models/WaterBottle.glb')
bottle_trimesh = bottle_gltf.geometry[list(bottle_gltf.geometry.keys())[0]]
bottle_mesh = Mesh.from_trimesh(bottle_trimesh)
bottle_pose = np.array([
[1.0, 0.0, 0.0, 0.1],
[0.0, 0.0, -1.0, -0.16],
[0.0, 1.0, 0.0, 0.13],
[0.0, 0.0, 0.0, 1.0],
])
boxv_trimesh = trimesh.creation.box(extents=0.1 * np.ones(3))
boxv_vertex_colors = np.random.uniform(size=(boxv_trimesh.vertices.shape))
boxv_trimesh.visual.vertex_colors = boxv_vertex_colors
boxv_mesh = Mesh.from_trimesh(boxv_trimesh, smooth=False)
boxf_trimesh = trimesh.creation.box(extents=0.1 * np.ones(3))
boxf_face_colors = np.random.uniform(size=boxf_trimesh.faces.shape)
boxf_trimesh.visual.face_colors = boxf_face_colors
# Instanced
poses = np.tile(np.eye(4), (2, 1, 1))
poses[0, :3, 3] = np.array([-0.1, -0.10, 0.05])
poses[1, :3, 3] = np.array([-0.15, -0.10, 0.05])
boxf_mesh = Mesh.from_trimesh(boxf_trimesh, poses=poses, smooth=False)
points = trimesh.creation.icosphere(radius=0.05).vertices
point_colors = np.random.uniform(size=points.shape)
points_mesh = Mesh.from_points(points, colors=point_colors)
direc_l = DirectionalLight(color=np.ones(3), intensity=1.0)
spot_l = SpotLight(color=np.ones(3),
intensity=10.0,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6)
cam = PerspectiveCamera(yfov=(np.pi / 3.0))
cam_pose = np.array([[0.0, -np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.5], [1.0, 0.0, 0.0, 0.0],
[0.0, np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.4], [0.0, 0.0, 0.0, 1.0]])
scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02]))
fuze_node = Node(mesh=fuze_mesh,
translation=np.array(
[0.1, 0.15, -np.min(fuze_trimesh.vertices[:, 2])]))
scene.add_node(fuze_node)
boxv_node = Node(mesh=boxv_mesh, translation=np.array([-0.1, 0.10, 0.05]))
scene.add_node(boxv_node)
boxf_node = Node(mesh=boxf_mesh)
scene.add_node(boxf_node)
_ = scene.add(drill_mesh, pose=drill_pose)
_ = scene.add(bottle_mesh, pose=bottle_pose)
_ = scene.add(wood_mesh)
_ = scene.add(direc_l, pose=cam_pose)
_ = scene.add(spot_l, pose=cam_pose)
_ = scene.add(points_mesh)
_ = scene.add(cam, pose=cam_pose)
r = OffscreenRenderer(viewport_width=640, viewport_height=480)
color, depth = r.render(scene)
assert color.shape == (480, 640, 3)
assert depth.shape == (480, 640)
assert np.max(depth.data) > 0.05
assert np.count_nonzero(depth.data) > (0.2 * depth.size)
r.delete()
# point_l_node = scene.add(point_l_cam, pose=new_campose)
#==============================================================================
# Use viewer to display scene
#==============================================================================
# View the scene
view_render_flags = {'cull_faces': False, 'vertex_normals': False}
v = Viewer(scene, render_flags=view_render_flags, viewport_size=(IMAGE_WIDTH, IMAGE_HEIGHT))
#==============================================================================
# Prepare for dataset generation
#==============================================================================
flags = RenderFlags.SKIP_CULL_FACES
r = OffscreenRenderer(viewport_width=IMAGE_WIDTH, viewport_height=IMAGE_HEIGHT)
# color, _ = r.render(scene, flags=flags)
# Generate dataset ?
choice = input("Generate dataset? [y/n]: ")
accepted_inputs = ['y','n']
while (not (choice in accepted_inputs)):
choice = input("Generate dataset? [y/n]: ")
iterations = 5
mode = CameraPose.NONE
csvfile = './outputs/flowers_dataset.csv'
csvMode = 'w' # 'a' => append / 'w' => write (overwrites existing csv file)
meshes = [(flower_mesh, 'flower'), (stem_mesh, 'stem'), (center_mesh, 'center')]
skip_default_view = False
salt = '001'
def main():
# Colors
colors = []
for n in range(args.num_colors):
hue = n / args.num_colors
saturation = 1
lightness = 1
red, green, blue = colorsys.hsv_to_rgb(hue, saturation, lightness)
colors.append(np.array((red, green, blue, 1)))
floor_textures = [
"../textures/lg_floor_d.tga",
"../textures/lg_style_01_floor_blue_d.tga",
"../textures/lg_style_01_floor_orange_bright_d.tga",
]
wall_textures = [
"../textures/lg_style_01_wall_cerise_d.tga",
"../textures/lg_style_01_wall_green_bright_d.tga",
"../textures/lg_style_01_wall_red_bright_d.tga",
"../textures/lg_style_02_wall_yellow_d.tga",
"../textures/lg_style_03_wall_orange_bright_d.tga",
]
objects = [
pyrender.objects.Capsule,
pyrender.objects.Cylinder,
pyrender.objects.Icosahedron,
pyrender.objects.Box,
pyrender.objects.Sphere,
]
renderer = OffscreenRenderer(viewport_width=args.image_size,
viewport_height=args.image_size)
plt.tight_layout()
fig = plt.figure(figsize=(6, 3))
axis_perspective = fig.add_subplot(1, 2, 1)
axis_orthogonal = fig.add_subplot(1, 2, 2)
ims = []
scene = build_scene(floor_textures,
wall_textures,
fix_light_position=args.fix_light_position)
place_objects(scene,
colors,
objects,
min_num_objects=args.num_objects,
max_num_objects=args.num_objects,
discrete_position=args.discrete_position,
rotate_object=args.rotate_object)
camera_distance = 5
perspective_camera = PerspectiveCamera(yfov=math.pi / 4)
perspective_camera_node = Node(camera=perspective_camera,
translation=np.array([0, 1, 1]))
orthographic_camera = OrthographicCamera(xmag=3, ymag=3)
orthographic_camera_node = Node(camera=orthographic_camera)
rad_step = math.pi / 36
total_frames = int(math.pi * 2 / rad_step)
current_rad = 0
for _ in range(total_frames):
scene.add_node(perspective_camera_node)
# Perspective camera
camera_xz = camera_distance * np.array(
(math.sin(current_rad), math.cos(current_rad)))
# Compute yaw and pitch
camera_direction = np.array([camera_xz[0], 0, camera_xz[1]])
yaw, pitch = compute_yaw_and_pitch(camera_direction)
perspective_camera_node.rotation = genearte_camera_quaternion(
yaw, pitch)
camera_position = np.array([camera_xz[0], 1, camera_xz[1]])
perspective_camera_node.translation = camera_position
# Rendering
flags = RenderFlags.SHADOWS_DIRECTIONAL
if args.anti_aliasing:
flags |= RenderFlags.ANTI_ALIASING
image = renderer.render(scene, flags=flags)[0]
im1 = axis_perspective.imshow(image,
interpolation="none",
animated=True)
scene.remove_node(perspective_camera_node)
# Orthographic camera
scene.add_node(orthographic_camera_node)
camera_direction = camera_distance * np.array(
(math.sin(current_rad), math.sin(
math.pi / 6), math.cos(current_rad)))
yaw, pitch = compute_yaw_and_pitch(camera_direction)
orthographic_camera_node.rotation = genearte_camera_quaternion(
yaw, pitch)
orthographic_camera_node.translation = np.array(
[camera_direction[0], 4, camera_direction[2]])
image = renderer.render(scene, flags=flags)[0]
im2 = axis_orthogonal.imshow(image,
interpolation="none",
animated=True)
ims.append([im1, im2])
plt.pause(1e-8)
current_rad += rad_step
scene.remove_node(orthographic_camera_node)
ani = animation.ArtistAnimation(fig,
ims,
interval=1 / 24,
blit=True,
repeat_delay=0)
filename = "rooms"
if args.discrete_position:
filename += "_discrete_position"
if args.rotate_object:
filename += "_rotate_object"
if args.fix_light_position:
filename += "_fix_light_position"
filename += ".gif"
ani.save(filename, writer="imagemagick")