def run(ycb_path, ibl_path, plane_texture_path): mesh_path = pathlib.Path(ycb_path) / 'models' if ibl_path: ibl_path = pathlib.Path(ibl_path) sl.init() # use sl.init_cuda() for CUDA interop # Load all meshes meshes = sl.Mesh.load_threaded( [mesh_path / c / 'textured.obj' for c in CLASSES[1:]]) # Setup class IDs for i, mesh in enumerate(meshes): mesh.class_index = i + 1 # Create a scene with matching intrinsics scene = sl.Scene(RESOLUTION) scene.set_camera_intrinsics(*INTRINSICS) for mesh in random.sample(meshes, 10): obj = sl.Object(mesh) # Override the metallic/roughness parameters so that it gets interesting obj.metallic = random.random() obj.roughness = random.random() scene.add_object(obj) # Let them fall in a heap scene.simulate_tabletop_scene() # Setup lighting if ibl_path: scene.light_map = sl.LightMap(ibl_path) else: scene.choose_random_light_position() # Display a plane & set background color scene.background_plane_size = torch.tensor([3.0, 3.0]) scene.background_color = torch.tensor([0.1, 0.1, 0.1, 1.0]) if plane_texture_path: scene.background_plane_texture = sl.Texture2D(plane_texture_path) # Display interactive viewer sl.view(scene) # Render a frame renderer = sl.RenderPass() result = renderer.render(scene) # Save as JPEG Image.fromarray(result.rgb()[:, :, :3].cpu().numpy()).save('rgb.jpeg')
def test_render(self): scene = sl.Scene((640, 480)) with Timer('Mesh load'): mesh = sl.Mesh( os.path.join(TESTS_PATH, 'stanford_bunny', 'scene.gltf')) mesh.center_bbox() mesh.scale_to_bbox_diagonal(0.5) object = sl.Object(mesh) scene.add_object(object) pose = torch.eye(4) pose[2, 3] = 0.5 object.set_pose(pose) with Timer('render'): renderer = sl.RenderPass() result = renderer.render(scene) with Timer('retrieve'): rgb = result.rgb() print("First pixel:", rgb[0, 0]) print("min: {}, max: {}".format(rgb.min(), rgb.max())) rgb_np = rgb.cpu().numpy() img = Image.fromarray(rgb_np, mode='RGBA') img.save('/tmp/stillleben.png') dbg = sl.render_debug_image(scene) dbg_np = dbg.cpu().numpy() dbg_img = Image.fromarray(dbg_np, mode='RGBA') dbg_img.save('/tmp/stillleben_debug.png') rgb_noise = process_image(rgb.permute(2, 0, 1)[:3].float() / 255.0) rgb_noise_np = (rgb_noise * 255).byte().permute( 1, 2, 0).contiguous().cpu().numpy() noise_img = Image.fromarray(rgb_noise_np, mode='RGB') noise_img.save('/tmp/stillleben_noise.png')
import random from PIL import Image sl.init() # use sl.init_cuda() for CUDA interop # Load a mesh (the constructor accepts pathlib paths) mesh = sl.Mesh(SL_PATH / 'tests' / 'stanford_bunny' / 'scene.gltf') # Meshes can come in strange dimensions - rescale to something reasonable mesh.scale_to_bbox_diagonal(0.5) # Create a scene with a few bunnies scene = sl.Scene((1920, 1080)) for i in range(20): obj = sl.Object(mesh) # Override the metallic/roughness parameters so that it gets interesting obj.metallic = random.random() obj.roughness = random.random() scene.add_object(obj) # Let them fall in a heap scene.simulate_tabletop_scene() # Setup lighting scene.light_map = sl.LightMap(SL_PATH / 'examples' / 'Circus_Backstage' / 'Circus_Backstage.ibl') # Display a plane & set background color scene.background_plane_size = torch.tensor([3.0, 3.0])
def test_gradient(self): for mesh in self.meshes: object = sl.Object(mesh) self.scene.add_object(object) pose = th.tensor([[0.0596, 0.8315, -0.5523, -0.0651], [0.4715, 0.4642, 0.7498, -0.06036], [0.8798, -0.3051, -0.3644, 0.80551], [0.0000, 0.0000, 0.0000, 1.0000]]) U, S, V = th.svd(pose[:3, :3]) pose[:3, :3] = th.matmul(U, V.t()) object.set_pose(pose) renderer = sl.RenderPass() result = renderer.render(self.scene) gt_mask = (result.instance_index() != 0).float() rgb = result.rgb().detach() gt_rgb = rgb[:, :, :3] gt_rgb_np = gt_rgb.cpu().numpy() gt_pose = 0 if VIS: import visdom from torchnet.logger import VisdomLogger env_name = 'stillleben' vis = visdom.Visdom(port=8097, env=env_name) vis.close(env=env_name) gt_logger = VisdomLogger('image', env=env_name, port=8097, opts=dict(title='gt')) img_logger = VisdomLogger('image', env=env_name, port=8097, opts=dict(title='rgb')) for param in range(6): for obj in self.scene.objects: new_pose = obj.pose().clone() gt_pose = obj.pose().clone() GT_DELTA = th.zeros(6) GT_DELTA[param] = 0.01 new_pose = sl.diff.apply_pose_delta(gt_pose, GT_DELTA) obj.set_pose(new_pose) rendered_result = renderer.render(self.scene) rendered_rgb = rendered_result.rgb() rendered_rgb = rendered_rgb[:, :, :3] rgb_np = rendered_rgb.cpu().numpy() if VIS: img_logger.log(rgb_np.transpose(2, 0, 1)) gt_logger.log(gt_rgb_np.transpose(2, 0, 1)) gt_mask = th.ones_like(gt_mask) grad_wrt_img, l = get_gaussian_pyramid_comparison( gt_rgb.float() / 255.0, rendered_rgb.float() / 255.0, gt_mask.squeeze()) delta = sl.diff.backpropagate_gradient_to_poses( self.scene, rendered_result, grad_wrt_img) print('GT delta', GT_DELTA) print('Delta', delta) self.assertGreater(delta[0][param], 0)
if args.ambient: scene.ambient_light = torch.tensor( [float(a) for a in args.ambient.split(',')]) if args.light_map: scene.light_map = sl.LightMap(args.light_map) scene.background_plane_size = torch.tensor( [float(a) for a in args.background_plane_size.split(',')]) if args.background_plane_texture: scene.background_plane_texture = sl.Texture2D( args.background_plane_texture) for mesh in meshes: object = sl.Object(mesh, options=opts) object.shininess = args.shininess if args.metalness: object.metalness = args.metalness if args.roughness: object.roughness = args.roughness if args.specular_color: object.specular_color = torch.tensor( [float(a) for a in args.specular_color.split(',')]) object.sticker_range = torch.tensor( [float(a) for a in args.sticker_range.split(',')]) if args.sticker_texture: