def run(ycb_path, ibl_path, plane_texture_path):
mesh_path = pathlib.Path(ycb_path) / 'models'
if ibl_path:
ibl_path = pathlib.Path(ibl_path)
sl.init() # use sl.init_cuda() for CUDA interop
# Load all meshes
meshes = sl.Mesh.load_threaded(
[mesh_path / c / 'textured.obj' for c in CLASSES[1:]])
# Setup class IDs
for i, mesh in enumerate(meshes):
mesh.class_index = i + 1
# Create a scene with matching intrinsics
scene = sl.Scene(RESOLUTION)
scene.set_camera_intrinsics(*INTRINSICS)
for mesh in random.sample(meshes, 10):
obj = sl.Object(mesh)
# Override the metallic/roughness parameters so that it gets interesting
obj.metallic = random.random()
obj.roughness = random.random()
scene.add_object(obj)
# Let them fall in a heap
scene.simulate_tabletop_scene()
# Setup lighting
if ibl_path:
scene.light_map = sl.LightMap(ibl_path)
else:
scene.choose_random_light_position()
# Display a plane & set background color
scene.background_plane_size = torch.tensor([3.0, 3.0])
scene.background_color = torch.tensor([0.1, 0.1, 0.1, 1.0])
if plane_texture_path:
scene.background_plane_texture = sl.Texture2D(plane_texture_path)
# Display interactive viewer
sl.view(scene)
# Render a frame
renderer = sl.RenderPass()
result = renderer.render(scene)
# Save as JPEG
Image.fromarray(result.rgb()[:, :, :3].cpu().numpy()).save('rgb.jpeg')
def test_render(self):
scene = sl.Scene((640, 480))
with Timer('Mesh load'):
mesh = sl.Mesh(
os.path.join(TESTS_PATH, 'stanford_bunny', 'scene.gltf'))
mesh.center_bbox()
mesh.scale_to_bbox_diagonal(0.5)
object = sl.Object(mesh)
scene.add_object(object)
pose = torch.eye(4)
pose[2, 3] = 0.5
object.set_pose(pose)
with Timer('render'):
renderer = sl.RenderPass()
result = renderer.render(scene)
with Timer('retrieve'):
rgb = result.rgb()
print("First pixel:", rgb[0, 0])
print("min: {}, max: {}".format(rgb.min(), rgb.max()))
rgb_np = rgb.cpu().numpy()
img = Image.fromarray(rgb_np, mode='RGBA')
img.save('/tmp/stillleben.png')
dbg = sl.render_debug_image(scene)
dbg_np = dbg.cpu().numpy()
dbg_img = Image.fromarray(dbg_np, mode='RGBA')
dbg_img.save('/tmp/stillleben_debug.png')
rgb_noise = process_image(rgb.permute(2, 0, 1)[:3].float() / 255.0)
rgb_noise_np = (rgb_noise * 255).byte().permute(
1, 2, 0).contiguous().cpu().numpy()
noise_img = Image.fromarray(rgb_noise_np, mode='RGB')
noise_img.save('/tmp/stillleben_noise.png')
for i in range(20):
obj = sl.Object(mesh)
# Override the metallic/roughness parameters so that it gets interesting
obj.metallic = random.random()
obj.roughness = random.random()
scene.add_object(obj)
# Let them fall in a heap
scene.simulate_tabletop_scene()
# Setup lighting
scene.light_map = sl.LightMap(SL_PATH / 'examples' / 'Circus_Backstage' /
'Circus_Backstage.ibl')
# Display a plane & set background color
scene.background_plane_size = torch.tensor([3.0, 3.0])
scene.background_color = torch.tensor([0.1, 0.1, 0.1, 1.0])
# Display interactive viewer
sl.view(scene)
# Render a frame
renderer = sl.RenderPass()
result = renderer.render(scene)
print('Resulting RGB frame:', result.rgb().shape)
print('Resulting segmentation frame:', result.instance_index().shape)
# Save as JPEG
Image.fromarray(result.rgb()[:, :, :3].cpu().numpy()).save('rgb.jpeg')
def test_gradient(self):
for mesh in self.meshes:
object = sl.Object(mesh)
self.scene.add_object(object)
pose = th.tensor([[0.0596, 0.8315, -0.5523, -0.0651],
[0.4715, 0.4642, 0.7498, -0.06036],
[0.8798, -0.3051, -0.3644, 0.80551],
[0.0000, 0.0000, 0.0000, 1.0000]])
U, S, V = th.svd(pose[:3, :3])
pose[:3, :3] = th.matmul(U, V.t())
object.set_pose(pose)
renderer = sl.RenderPass()
result = renderer.render(self.scene)
gt_mask = (result.instance_index() != 0).float()
rgb = result.rgb().detach()
gt_rgb = rgb[:, :, :3]
gt_rgb_np = gt_rgb.cpu().numpy()
gt_pose = 0
if VIS:
import visdom
from torchnet.logger import VisdomLogger
env_name = 'stillleben'
vis = visdom.Visdom(port=8097, env=env_name)
vis.close(env=env_name)
gt_logger = VisdomLogger('image',
env=env_name,
port=8097,
opts=dict(title='gt'))
img_logger = VisdomLogger('image',
env=env_name,
port=8097,
opts=dict(title='rgb'))
for param in range(6):
for obj in self.scene.objects:
new_pose = obj.pose().clone()
gt_pose = obj.pose().clone()
GT_DELTA = th.zeros(6)
GT_DELTA[param] = 0.01
new_pose = sl.diff.apply_pose_delta(gt_pose, GT_DELTA)
obj.set_pose(new_pose)
rendered_result = renderer.render(self.scene)
rendered_rgb = rendered_result.rgb()
rendered_rgb = rendered_rgb[:, :, :3]
rgb_np = rendered_rgb.cpu().numpy()
if VIS:
img_logger.log(rgb_np.transpose(2, 0, 1))
gt_logger.log(gt_rgb_np.transpose(2, 0, 1))
gt_mask = th.ones_like(gt_mask)
grad_wrt_img, l = get_gaussian_pyramid_comparison(
gt_rgb.float() / 255.0,
rendered_rgb.float() / 255.0, gt_mask.squeeze())
delta = sl.diff.backpropagate_gradient_to_poses(
self.scene, rendered_result, grad_wrt_img)
print('GT delta', GT_DELTA)
print('Delta', delta)
self.assertGreater(delta[0][param], 0)
pose[2, 3] = scene.min_dist_for_object_diameter(mesh.bbox.diagonal)
object.set_pose(pose)
elif args.placement == 'random':
if True:
if not scene.find_noncolliding_pose(object,
sampler='random',
max_iterations=50,
viewpoint=torch.tensor(
[1.0, 0.0, 0.0])):
print('WARNING: Could not find non-colliding pose')
elif True:
pose = scene.place_object_randomly(mesh.bbox.diagonal)
object.set_pose(pose)
renderer = sl.RenderPass(shading=args.shading)
if args.placement == 'tabletop':
if args.tabletop_video:
import subprocess
command = [
'/usr/bin/ffmpeg',
'-y', # (optional) overwrite output file if it exists
'-f',
'rawvideo',
'-vcodec',
'rawvideo',
'-s',
'640x480', # size of one frame