def render_model_exemplars(pbar,
pair: ExemplarShapePair,
render_shape=config.SHAPE_REND_SHAPE):
camera = cameras.spherical_coord_to_cam(pair.fov, pair.azimuth,
pair.elevation)
pbar.set_description(f'[{pair.id}] Loading shape')
mesh, materials = pair.shape.load()
camera.near, camera.far = compute_tight_clipping_planes(
mesh, camera.view_mat())
segment_im = render_segments(mesh, camera)
fg_bbox = mask_bbox(segment_im > -1)
pbar.set_description('Rendering preview')
phong_im = np.clip(
render_preview(mesh,
camera,
config.SHAPE_REND_RADMAP_PATH,
gamma=2.2,
ssaa=2), 0, 1)
phong_im = crop_tight_fg(phong_im, render_shape, bbox=fg_bbox)
vis.image(phong_im.transpose((2, 0, 1)), win='shape-preview')
pbar.set_description(f'[{pair.id}] Saving data')
with warnings.catch_warnings():
warnings.simplefilter('ignore')
pair.save_data(config.SHAPE_REND_PREVIEW_NAME,
skimage.img_as_uint(phong_im))
def main():
with session_scope() as sess:
pairs = (sess.query(ExemplarShapePair)
.filter(ExemplarShapePair.distance < config.ALIGN_DIST_THRES)
.all())
print(f'Fetched {len(pairs)} pairs. '
f'align_dist_thres = {config.ALIGN_DIST_THRES}',
len(pairs), config.ALIGN_DIST_THRES)
pbar = tqdm(pairs)
for pair in pbar:
pbar.set_description(f'[Pair {pair.id}]')
mesh, materials = pair.shape.load()
camera = cameras.spherical_coord_to_cam(
pair.fov, pair.azimuth, pair.elevation)
camera.near, camera.far = \
compute_tight_clipping_planes(mesh, camera.view_mat())
segment_im = render_segments(mesh, camera)
fg_bbox = mask_bbox(segment_im > -1)
pair.params = {
'camera': camera.tojsd(),
'crop_bbox': fg_bbox,
}
sess.commit()
def render_segment_map(pbar, d, params):
with session_scope() as sess:
pair = sess.query(models.ExemplarShapePair).get(params['pair_id'])
rk_mesh, _ = pair.shape.load()
shape_id = params['shape_id']
cam_fov = params['camera']['fov']
cam_azimuth = params['camera']['azimuth']
cam_elevation = params['camera']['elevation']
cam_dist = params['camera']['distance']
rk_mesh.resize(1)
# Set random camera.
rk_camera = cameras.spherical_coord_to_cam(cam_fov,
cam_azimuth,
cam_elevation,
cam_dist=cam_dist,
max_len=500)
seg_map = shortcuts.render_segments(rk_mesh, rk_camera)
seg_vis = visualize_map(seg_map)[:, :, :3]
path = d['seg_vis_path']
pbar.set_description(f'{path}')
imsave(d['seg_vis_path'], seg_vis)
imsave(d['seg_map_path'], (seg_map + 1).astype(np.uint8))
vis.image(visualize_map(seg_map).transpose((2, 0, 1)))
def render_model_exemplars(pbar,
pair: ExemplarShapePair,
render_shape=config.SHAPE_REND_SHAPE):
warnings.simplefilter('ignore')
camera = cameras.spherical_coord_to_cam(pair.fov, pair.azimuth,
pair.elevation)
pbar.set_description(f'[{pair.id}] Loading shape')
mesh, materials = pair.shape.load()
camera.near, camera.far = compute_tight_clipping_planes(
mesh, camera.view_mat())
pbar.set_description(f'[{pair.id}] Rendering segments')
if not pair.data_exists(config.PAIR_FG_BBOX_NAME):
segment_im = render_segments(mesh, camera)
fg_mask = segment_im > -1
fg_bbox = mask_bbox(fg_mask)
segment_im = crop_tight_fg(segment_im,
render_shape,
bbox=fg_bbox,
fill=-1,
order=0)
pair.save_data(config.SHAPE_REND_SEGMENT_VIS_NAME,
skimage.img_as_uint(visualize_map(segment_im)))
pair.save_data(config.SHAPE_REND_SEGMENT_MAP_NAME,
(segment_im + 1).astype(np.uint8))
tqdm.write(f" * Saving {pair.get_data_path(config.PAIR_FG_BBOX_NAME)}")
pair.save_data(config.PAIR_RAW_SEGMENT_MAP_NAME,
(segment_im + 1).astype(np.uint8))
pair.save_data(config.PAIR_FG_BBOX_NAME,
fg_mask.astype(np.uint8) * 255)
else:
fg_mask = pair.load_data(config.PAIR_FG_BBOX_NAME)
fg_bbox = mask_bbox(fg_mask)
if not pair.data_exists(config.SHAPE_REND_PHONG_NAME):
pbar.set_description('Rendering phong')
phong_im = np.clip(
render_wavefront_mtl(mesh,
camera,
materials,
config.SHAPE_REND_RADMAP_PATH,
gamma=2.2,
ssaa=3,
tonemap='reinhard'), 0, 1)
phong_im = crop_tight_fg(phong_im, render_shape, bbox=fg_bbox)
pbar.set_description(f'[{pair.id}] Saving data')
pair.save_data(config.SHAPE_REND_PHONG_NAME,
skimage.img_as_uint(phong_im))
def do_render(scene, pair, mesh, envmaps_by_split, cam_angles, rk_mesh,
seg_substances, mats_by_subst, bmats):
time_begin = time.time()
# Jitter camera params.
cam_azimuth, cam_elevation = random.choice(cam_angles)
cam_azimuth += random.uniform(-math.pi / 12, -math.pi / 12)
cam_elevation += random.uniform(-math.pi / 24, -math.pi / 24)
cam_dist = random.uniform(1.3, 1.75)
cam_fov = random.uniform(FOV_MIN, FOV_MAX)
rk_camera = cameras.spherical_coord_to_cam(cam_fov,
cam_azimuth,
cam_elevation,
cam_dist=cam_dist,
max_len=_REND_SHAPE[0] / 2)
# Jitter envmap params.
# Set envmap rotation so that the camera points at the "back"
# but also allow some wiggle room.
envmap_scale = random.uniform(0.9, 1.2)
envmap = random.choice(envmaps_by_split[pair.shape.split_set])
envmap_rotation = (0, 0, (envmap.azimuth + pi / 2 + cam_azimuth +
random.uniform(-pi / 24, pi / 24)))
scene.set_envmap(envmap.get_data_path('hdr.exr'),
scale=envmap_scale,
rotation=envmap_rotation)
if scene.camera is None:
camera = brender.CalibratedCamera(scene, rk_camera.cam_to_world(),
cam_fov)
scene.set_active_camera(camera)
else:
scene.camera.set_params(rk_camera.cam_to_world(), cam_fov)
segment_material_ids = {}
segment_uv_ref_scales = {}
segment_uv_rotations = {}
segment_uv_translations = {}
segment_mean_roughness = {}
logger.info('Setting materials...')
brdf_names = set()
substances = set()
for seg_name in rk_mesh.materials:
for bobj in bpy.data.materials:
if bobj.name == seg_name:
if seg_name not in seg_substances:
logger.warning('Substance unknown for %s', seg_name)
return
substance = seg_substances[seg_name]
substances.add(substance)
materials = mats_by_subst[substance]
if len(materials) == 0:
logger.warning('No materials for substance %s', substance)
return
material: models.Material = random.choice(materials)
brdf_names.add(material.name)
# Jitter UV map.
uv_translation = (random.uniform(0, 1), random.uniform(0, 1))
uv_rotation = random.uniform(0, 2 * math.pi)
uv_ref_scale = (2**(material.default_scale - 3.0 +
random.uniform(-1.0, 0.5)))
segment_uv_ref_scales[seg_name] = uv_ref_scale
segment_material_ids[seg_name] = material.id
segment_uv_rotations[seg_name] = uv_rotation
segment_uv_translations[seg_name] = uv_translation
bmat: NodesMaterial = loader.material_to_brender(
material,
bobj=bobj,
uv_ref_scale=uv_ref_scale,
uv_translation=uv_translation,
uv_rotation=uv_rotation)
segment_mean_roughness[seg_name] = \
float(bmat.mean_roughness())
bmats.append(bmat)
# This needs to come after the materials are initialized.
logger.info('Computing UV density...')
mesh.compute_uv_density()
logger.info('Rendering...')
with suppress_stdout():
rend = scene.render_to_array(format='exr')
caption = f'{envmap.name}, {str(substances)}, {str(brdf_names)}'
rend_srgb = toolbox.images.linear_to_srgb(rend)
time_elapsed = time.time() - time_begin
logger.info('Rendered one in %fs', time_elapsed)
seg_map = shortcuts.render_segments(rk_mesh, rk_camera)
seg_vis = visualize_map(seg_map)[:, :, :3]
normal_map = shortcuts.render_mesh_normals(rk_mesh, rk_camera)
normal_map_blender = normal_map.copy()
normal_map_blender[:, :, :3] += 1.0
normal_map_blender[:, :, :3] /= 2.0
normal_map_blender = np.round(255.0 * normal_map_blender).astype(np.uint8)
figure = toolbox.images.to_8bit(np.hstack((
seg_vis,
rend_srgb[:, :, :3],
)))
segment_ids = {name: i for i, name in enumerate(rk_mesh.materials)}
params = {
'split_set': pair.shape.split_set,
'pair_id': pair.id,
'shape_id': pair.shape_id,
'exemplar_id': pair.exemplar_id,
'camera': {
'fov': cam_fov,
'azimuth': cam_azimuth,
'elevation': cam_elevation,
'distance': cam_dist,
},
'envmap': {
'id': envmap.id,
'name': envmap.name,
'source': envmap.source,
'scale': envmap_scale,
'rotation': envmap_rotation,
},
'segment': {
'segment_ids': segment_ids,
'materials': segment_material_ids,
'uv_ref_scales': segment_uv_ref_scales,
'uv_translations': segment_uv_translations,
'uv_rotations': segment_uv_rotations,
'mean_roughness': segment_mean_roughness,
},
'time_elapsed': time_elapsed,
}
return {
'ldr': toolbox.images.to_8bit(rend_srgb),
'hdr': rend,
'seg_map': (seg_map + 1).astype(np.uint8),
'seg_vis': toolbox.images.to_8bit(seg_vis),
'normal_image': normal_map_blender,
'params': params,
}
def render_pair(app: brender.Brender, pair: ExemplarShapePair,
materials_by_substance, base_out_dir):
# Load shapenet mesh and resize to 1.0 to match Blender size.
rk_mesh, _ = pair.shape.load()
rk_mesh.resize(1)
with open(_TMP_MESH_PATH, 'w') as f:
wavefront.save_obj_file(f, rk_mesh)
scene = brender.Scene(app, shape=_REND_SHAPE, aa_samples=32)
envmap_rotation = (0, 0, (math.pi + math.pi/2 + pair.azimuth))
scene.set_envmap(_ENVMAP_PATH, scale=5, rotation=envmap_rotation)
with suppress_stdout():
mesh = Mesh.from_obj(scene, _TMP_MESH_PATH)
mat_substances = utils.compute_segment_substances(pair)
# Get exemplar camera parameters.
rk_camera = cameras.spherical_coord_to_cam(
pair.fov, pair.azimuth, pair.elevation, cam_dist=2.0,
max_len=_REND_SHAPE[0]/2)
segment_im = render_segments(rk_mesh, rk_camera)
camera = brender.CalibratedCamera(
scene, rk_camera.cam_to_world(), pair.fov)
scene.set_active_camera(camera)
bmats = []
segment_pbar = tqdm(rk_mesh.materials)
for segment_name in segment_pbar:
segment_pbar.set_description(f'Segment {segment_name}')
try:
mat_subst = mat_substances[segment_name]
materials = materials_by_substance[mat_subst]
except KeyError:
continue
out_dir = Path(base_out_dir, str(pair.id), str(segment_name))
out_dir.mkdir(parents=True, exist_ok=True)
material_pbar = tqdm(materials)
for material in material_pbar:
material_pbar.set_description(f'Material {material.id}')
out_path = Path(out_dir, f'{material.id}.png')
if out_path.exists():
material_pbar.set_description(
f'Material {material.id} already rendered')
continue
# Activate only current material.
bobj = None
for bobj in bpy.data.materials:
if bobj.name == segment_name:
break
bobj_matches = [o for o in bpy.data.materials
if o.name == segment_name]
if len(bobj_matches) == 0:
bmat = InvisibleMaterial(bobj=bobj)
else:
bmat = loader.material_to_brender(material, bobj=bobj)
bmats.append(bmat)
with suppress_stdout():
rend_im = scene.render_to_array()
vis.image(rend_im.transpose((2, 0, 1)), win='rend-im')
rend_im[segment_im != rk_mesh.materials.index(segment_name)] = 0
fg_bbox = mask_bbox(segment_im > -1)
rend_im = crop_tight_fg(rend_im, _FINAL_SHAPE, bbox=fg_bbox,
fill=0, use_pil=True)
with warnings.catch_warnings():
warnings.simplefilter('ignore', UserWarning)
skio.imsave(str(out_path), rend_im)
while len(bmats) > 0:
bmat = bmats.pop()
bmat.bobj.name = bmat.bobj.name
del bmat