def _cam2world_matrix_from_cam_extrinsics(self, config: Config) -> np.ndarray: """ Determines camera extrinsics by using the given config and returns them in form of a cam to world frame transformation matrix. :param config: The configuration object. :return: The 4x4 cam to world transformation matrix. """ if not config.has_param("cam2world_matrix"): # Print warning if local_frame_change is used with other attributes than cam2world_matrix if self.local_frame_change != ["X", "Y", "Z"]: print( "Warning: The local_frame_change parameter is at the moment only supported when setting the cam2world_matrix attribute." ) position = change_coordinate_frame_of_point( config.get_vector3d("location", [0, 0, 0]), self.world_frame_change) # Rotation rotation_format = config.get_string("rotation/format", "euler") value = config.get_vector3d("rotation/value", [0, 0, 0]) # Transform to blender coord frame value = change_coordinate_frame_of_point(value, self.world_frame_change) if rotation_format == "euler": # Rotation, specified as euler angles rotation_matrix = Euler(value, 'XYZ').to_matrix() elif rotation_format == "forward_vec": # Convert forward vector to euler angle (Assume Up = Z) rotation_matrix = CameraUtility.rotation_from_forward_vec( value) elif rotation_format == "look_at": # Convert forward vector to euler angle (Assume Up = Z) rotation_matrix = CameraUtility.rotation_from_forward_vec( value - position) else: raise Exception("No such rotation format:" + str(rotation_format)) if rotation_format == "look_at" or rotation_format == "forward_vec": inplane_rot = config.get_float("rotation/inplane_rot", 0.0) rotation_matrix = np.matmul( rotation_matrix, Euler((0.0, 0.0, inplane_rot)).to_matrix()) cam2world_matrix = build_transformation_mat( position, rotation_matrix) else: cam2world_matrix = np.array( config.get_list("cam2world_matrix")).reshape(4, 4).astype( np.float32) cam2world_matrix = change_source_coordinate_frame_of_transformation_matrix( cam2world_matrix, self.local_frame_change) cam2world_matrix = change_target_coordinate_frame_of_transformation_matrix( cam2world_matrix, self.world_frame_change) return cam2world_matrix
def _set_cam_extrinsics(self, config, frame=None): """ Sets camera extrinsics according to the config. :param frame: Optional, the frame to set the camera pose to. :param config: A configuration object with cam extrinsics. """ if config.has_param("frame"): frame = config.get_int("frame") cam2world_matrix = self._cam2world_matrix_from_cam_extrinsics(config) CameraUtility.add_camera_pose(cam2world_matrix, frame)
def stereo_global_matching( color_images: List[np.ndarray], depth_max: Optional[float] = None, window_size: int = 7, num_disparities: int = 32, min_disparity: int = 0, disparity_filter: bool = True, depth_completion: bool = True ) -> Tuple[List[np.ndarray], List[np.ndarray]]: """ Does the stereo global matching in the following steps: 1. Collect camera object and its state, 2. For each frame, load left and right images and call the `sgm()` methode. 3. Write the results to a numpy file. :param color_images: A list of stereo images, where each entry has the shape [2, height, width, 3]. :param depth_max: The maximum depth value for clipping the resulting depth values. If None, distance_start + distance_range that were configured for distance rendering are used. :param window_size: Semi-global matching kernel size. Should be an odd number. :param num_disparities: Semi-global matching number of disparities. Should be > 0 and divisible by 16. :param min_disparity: Semi-global matching minimum disparity. :param disparity_filter: Applies post-processing of the generated disparity map using WLS filter. :param depth_completion: Applies basic depth completion using image processing techniques. :return: Returns the computed depth and disparity images for all given frames. """ # Collect camera and camera object cam_ob = bpy.context.scene.camera cam = cam_ob.data baseline = cam.stereo.interocular_distance if not baseline: raise Exception( "Stereo parameters are not set. Make sure to enable RGB stereo rendering before this module." ) if depth_max is None: depth_max = bpy.context.scene.world.mist_settings.start + bpy.context.scene.world.mist_settings.depth baseline = cam.stereo.interocular_distance if not baseline: raise Exception( "Stereo parameters are not set. Make sure to enable RGB stereo rendering before this module." ) focal_length = CameraUtility.get_intrinsics_as_K_matrix()[0, 0] depth_frames = [] disparity_frames = [] for frame, color_image in enumerate(color_images): depth, disparity = StereoGlobalMatching._sgm( color_image[0], color_image[1], baseline, depth_max, focal_length, window_size, num_disparities, min_disparity, disparity_filter, depth_completion) depth_frames.append(depth) disparity_frames.append(disparity) return depth_frames, disparity_frames
def set_default_parameters(): """ Loads and sets default parameters defined in DefaultConfig.py """ # Set default intrinsics CameraUtility.set_intrinsics_from_blender_params( DefaultConfig.fov, DefaultConfig.resolution_x, DefaultConfig.resolution_y, DefaultConfig.clip_start, DefaultConfig.clip_end, DefaultConfig.pixel_aspect_x, DefaultConfig.pixel_aspect_y, DefaultConfig.shift_x, DefaultConfig.shift_y, DefaultConfig.lens_unit) CameraUtility.set_stereo_parameters( DefaultConfig.stereo_convergence_mode, DefaultConfig.stereo_convergence_distance, DefaultConfig.stereo_interocular_distance) # Init renderer RendererUtility._render_init() RendererUtility.set_samples(DefaultConfig.samples) addon_utils.enable("render_auto_tile_size") RendererUtility.toggle_auto_tile_size(True) # Set number of cpu cores used for rendering (1 thread is always used for coordination => 1 # cpu thread means GPU-only rendering) RendererUtility.set_cpu_threads(0) RendererUtility.set_denoiser(DefaultConfig.denoiser) RendererUtility.set_simplify_subdivision_render( DefaultConfig.simplify_subdivision_render) RendererUtility.set_light_bounces( DefaultConfig.diffuse_bounces, DefaultConfig.glossy_bounces, DefaultConfig.ao_bounces_render, DefaultConfig.max_bounces, DefaultConfig.transmission_bounces, DefaultConfig.transparency_bounces, DefaultConfig.volume_bounces) RendererUtility.set_output_format(DefaultConfig.file_format, DefaultConfig.color_depth, DefaultConfig.enable_transparency, DefaultConfig.jpg_quality)
def sample_and_validate_cam_pose(self, config: Config, existing_poses: List[np.ndarray]) -> bool: """ Samples a new camera pose, sets the parameters of the given camera object accordingly and validates it. :param config: The config object describing how to sample :param existing_poses: A list of already sampled valid poses. :return: True, if the sampled pose was valid """ # Sample camera extrinsics (we do not set them yet for performance reasons) cam2world_matrix = self._sample_pose(config) if self._is_pose_valid(cam2world_matrix, existing_poses): # Set camera extrinsics as the pose is valid frame = CameraUtility.add_camera_pose(cam2world_matrix) # Optional callback self._on_new_pose_added(cam2world_matrix, frame) # Add to the list of added cam poses existing_poses.append(cam2world_matrix) return True else: return False
def enable_normals_output(output_dir: Optional[str] = None, file_prefix: str = "normals_", output_key: str = "normals"): """ Enables writing normal images. Normal images will be written in the form of .exr files during the next rendering. :param output_dir: The directory to write files to, if this is None the temporary directory is used. :param file_prefix: The prefix to use for writing the files. :param output_key: The key to use for registering the normal output. """ if output_dir is None: output_dir = Utility.get_temporary_directory() bpy.context.scene.render.use_compositing = True bpy.context.scene.use_nodes = True tree = bpy.context.scene.node_tree links = tree.links # Use existing render layer render_layer_node = Utility.get_the_one_node_with_type(tree.nodes, 'CompositorNodeRLayers') separate_rgba = tree.nodes.new("CompositorNodeSepRGBA") space_between_nodes_x = 200 space_between_nodes_y = -300 separate_rgba.location.x = space_between_nodes_x separate_rgba.location.y = space_between_nodes_y links.new(render_layer_node.outputs["Normal"], separate_rgba.inputs["Image"]) combine_rgba = tree.nodes.new("CompositorNodeCombRGBA") combine_rgba.location.x = space_between_nodes_x * 14 c_channels = ["R", "G", "B"] offset = space_between_nodes_x * 2 multiplication_values: List[List[bpy.types.Node]] = [[], [], []] channel_results = {} for row_index, channel in enumerate(c_channels): # matrix multiplication mulitpliers = [] for column in range(3): multiply = tree.nodes.new("CompositorNodeMath") multiply.operation = "MULTIPLY" multiply.inputs[1].default_value = 0 # setting at the end for all frames multiply.location.x = column * space_between_nodes_x + offset multiply.location.y = row_index * space_between_nodes_y links.new(separate_rgba.outputs[c_channels[column]], multiply.inputs[0]) mulitpliers.append(multiply) multiplication_values[row_index].append(multiply) first_add = tree.nodes.new("CompositorNodeMath") first_add.operation = "ADD" first_add.location.x = space_between_nodes_x * 5 + offset first_add.location.y = row_index * space_between_nodes_y links.new(mulitpliers[0].outputs["Value"], first_add.inputs[0]) links.new(mulitpliers[1].outputs["Value"], first_add.inputs[1]) second_add = tree.nodes.new("CompositorNodeMath") second_add.operation = "ADD" second_add.location.x = space_between_nodes_x * 6 + offset second_add.location.y = row_index * space_between_nodes_y links.new(first_add.outputs["Value"], second_add.inputs[0]) links.new(mulitpliers[2].outputs["Value"], second_add.inputs[1]) channel_results[channel] = second_add # set the matrix accordingly rot_around_x_axis = mathutils.Matrix.Rotation(math.radians(-90.0), 4, 'X') for frame in range(bpy.context.scene.frame_start, bpy.context.scene.frame_end): used_rotation_matrix = CameraUtility.get_camera_pose(frame) @ rot_around_x_axis for row_index in range(3): for column_index in range(3): current_multiply = multiplication_values[row_index][column_index] current_multiply.inputs[1].default_value = used_rotation_matrix[column_index][row_index] current_multiply.inputs[1].keyframe_insert(data_path='default_value', frame=frame) offset = 8 * space_between_nodes_x for index, channel in enumerate(c_channels): multiply = tree.nodes.new("CompositorNodeMath") multiply.operation = "MULTIPLY" multiply.location.x = space_between_nodes_x * 2 + offset multiply.location.y = index * space_between_nodes_y links.new(channel_results[channel].outputs["Value"], multiply.inputs[0]) if channel == "G": multiply.inputs[1].default_value = -0.5 else: multiply.inputs[1].default_value = 0.5 add = tree.nodes.new("CompositorNodeMath") add.operation = "ADD" add.location.x = space_between_nodes_x * 3 + offset add.location.y = index * space_between_nodes_y links.new(multiply.outputs["Value"], add.inputs[0]) add.inputs[1].default_value = 0.5 output_channel = channel if channel == "G": output_channel = "B" elif channel == "B": output_channel = "G" links.new(add.outputs["Value"], combine_rgba.inputs[output_channel]) output_file = tree.nodes.new("CompositorNodeOutputFile") output_file.base_path = output_dir output_file.format.file_format = "OPEN_EXR" output_file.file_slots.values()[0].path = file_prefix output_file.location.x = space_between_nodes_x * 15 links.new(combine_rgba.outputs["Image"], output_file.inputs["Image"]) Utility.add_output_entry({ "key": output_key, "path": os.path.join(output_dir, file_prefix) + "%04d" + ".exr", "version": "2.0.0" })
def load_bop(bop_dataset_path: str, sys_paths: Union[List[str], str], temp_dir: str = None, model_type: str = "", cam_type: str = "", split: str = "test", scene_id: int = -1, obj_ids: list = [], sample_objects: bool = False, num_of_objs_to_sample: int = None, obj_instances_limit: int = -1, move_origin_to_x_y_plane: bool = False, source_frame: list = ["X", "-Y", "-Z"], mm2m: bool = False) -> List[MeshObject]: """ Loads the 3D models of any BOP dataset and allows replicating BOP scenes - Interfaces with the bob_toolkit, allows loading of train, val and test splits - Relative camera poses are loaded/computed with respect to a reference model - Sets real camera intrinsics :param bop_dataset_path: Full path to a specific bop dataset e.g. /home/user/bop/tless. :param sys_paths: System paths to append. Can be a string or a list of strings. :param temp_dir: A temp directory which is used for writing the temporary .ply file. :param model_type: Optionally, specify type of BOP model. Available: [reconst, cad or eval]. :param cam_type: Camera type. If not defined, dataset-specific default camera type is used. :param split: Optionally, test or val split depending on BOP dataset. :param scene_id: Optionally, specify BOP dataset scene to synthetically replicate. Default: -1 (no scene is replicated, only BOP Objects are loaded). :param obj_ids: List of object ids to load. Default: [] (all objects from the given BOP dataset if scene_id is not specified). :param sample_objects: Toggles object sampling from the specified dataset. :param num_of_objs_to_sample: Amount of objects to sample from the specified dataset. If this amount is bigger than the dataset actually contains, then all objects will be loaded. :param obj_instances_limit: Limits the amount of object copies when sampling. Default: -1 (no limit). :param move_origin_to_x_y_plane: Move center of the object to the lower side of the object, this will not work when used in combination with pose estimation tasks! This is designed for the use-case where BOP objects are used as filler objects in the background. :param source_frame: Can be used if the given positions and rotations are specified in frames different from the blender frame. Has to be a list of three strings. Example: ['X', '-Z', 'Y']: Point (1,2,3) will be transformed to (1, -3, 2). Available: ['X', 'Y', 'Z', '-X', '-Y', '-Z']. :param mm2m: Specify whether to convert poses and models to meters. :return: The list of loaded mesh objects. """ # Make sure sys_paths is a list if not isinstance(sys_paths, list): sys_paths = [sys_paths] for sys_path in sys_paths: if 'bop_toolkit' in sys_path: sys.path.append(sys_path) if temp_dir is None: temp_dir = Utility.get_temporary_directory() scale = 0.001 if mm2m else 1 bop_dataset_name = os.path.basename(bop_dataset_path) has_external_texture = bop_dataset_name in ["ycbv", "ruapc"] if obj_ids or sample_objects: allow_duplication = True else: allow_duplication = False datasets_path = os.path.dirname(bop_dataset_path) dataset = os.path.basename(bop_dataset_path) print("bob: {}, dataset_path: {}".format(bop_dataset_path, datasets_path)) print("dataset: {}".format(dataset)) try: from bop_toolkit_lib import dataset_params, inout except ImportError as error: print('ERROR: Please download the bop_toolkit package and add it to sys_paths in config!') print('https://github.com/thodan/bop_toolkit') raise error model_p = dataset_params.get_model_params(datasets_path, dataset, model_type=model_type if model_type else None) cam_p = dataset_params.get_camera_params(datasets_path, dataset, cam_type=cam_type if cam_type else None) try: split_p = dataset_params.get_split_params(datasets_path, dataset, split=split) except ValueError: raise Exception("Wrong path or {} split does not exist in {}.".format(split, dataset)) bpy.context.scene.render.resolution_x = cam_p['im_size'][0] bpy.context.scene.render.resolution_y = cam_p['im_size'][1] loaded_objects = [] # only load all/selected objects here, use other modules for setting poses # e.g. camera.CameraSampler / object.ObjectPoseSampler if scene_id == -1: # TLESS exception because images are cropped if bop_dataset_name in ['tless']: cam_p['K'][0, 2] = split_p['im_size'][0] / 2 cam_p['K'][1, 2] = split_p['im_size'][1] / 2 # set camera intrinsics CameraUtility.set_intrinsics_from_K_matrix(cam_p['K'], split_p['im_size'][0], split_p['im_size'][1]) obj_ids = obj_ids if obj_ids else model_p['obj_ids'] # if sampling is enabled if sample_objects: loaded_ids = {} loaded_amount = 0 if obj_instances_limit != -1 and len(obj_ids) * obj_instances_limit < num_of_objs_to_sample: raise RuntimeError("{}'s {} split contains {} objects, {} object where requested to sample with " "an instances limit of {}. Raise the limit amount or decrease the requested " "amount of objects.".format(bop_dataset_path, split, len(obj_ids), num_of_objs_to_sample, obj_instances_limit)) while loaded_amount != num_of_objs_to_sample: random_id = choice(obj_ids) if random_id not in loaded_ids.keys(): loaded_ids.update({random_id: 0}) # if there is no limit or if there is one, but it is not reached for this particular object if obj_instances_limit == -1 or loaded_ids[random_id] < obj_instances_limit: cur_obj = BopLoader._load_mesh(random_id, model_p, bop_dataset_name, has_external_texture, temp_dir, allow_duplication, scale) loaded_ids[random_id] += 1 loaded_amount += 1 loaded_objects.append(cur_obj) else: print("ID {} was loaded {} times with limit of {}. Total loaded amount {} while {} are " "being requested".format(random_id, loaded_ids[random_id], obj_instances_limit, loaded_amount, num_of_objs_to_sample)) else: for obj_id in obj_ids: cur_obj = BopLoader._load_mesh(obj_id, model_p, bop_dataset_name, has_external_texture, temp_dir, allow_duplication, scale) loaded_objects.append(cur_obj) # replicate scene: load scene objects, object poses, camera intrinsics and camera poses else: sc_gt = inout.load_scene_gt(split_p['scene_gt_tpath'].format(**{'scene_id': scene_id})) sc_camera = inout.load_json(split_p['scene_camera_tpath'].format(**{'scene_id': scene_id})) for i, (cam_id, insts) in enumerate(sc_gt.items()): cam_K, cam_H_m2c_ref = BopLoader._get_ref_cam_extrinsics_intrinsics(sc_camera, cam_id, insts, scale) if i == 0: # define world = first camera cam_H_m2w_ref = cam_H_m2c_ref.copy() cur_objs = [] # load scene objects and set their poses for inst in insts: cur_objs.append(BopLoader._load_mesh(inst['obj_id'], model_p, bop_dataset_name, has_external_texture, temp_dir, allow_duplication, scale)) BopLoader.set_object_pose(cur_objs[-1], inst, scale) cam_H_c2w = BopLoader._compute_camera_to_world_trafo(cam_H_m2w_ref, cam_H_m2c_ref, source_frame) # set camera intrinsics CameraUtility.set_intrinsics_from_K_matrix(cam_K, split_p['im_size'][0], split_p['im_size'][1]) # set camera extrinsics as next frame frame_id = CameraUtility.add_camera_pose(cam_H_c2w) # Add key frame for camera shift, as it changes from frame to frame in the tless replication cam = bpy.context.scene.camera.data cam.keyframe_insert(data_path='shift_x', frame=frame_id) cam.keyframe_insert(data_path='shift_y', frame=frame_id) # Copy object poses to key frame (to be sure) for cur_obj in cur_objs: BopLoader._insert_key_frames(cur_obj, frame_id) # move the origin of the object to the world origin and on top of the X-Y plane # makes it easier to place them later on, this does not change the `.location` # This is only useful if the BOP objects are not used in a pose estimation scenario. if move_origin_to_x_y_plane: for obj in loaded_objects: obj.move_origin_to_bottom_mean_point() return loaded_objects
def _set_cam_intrinsics(self, cam, config): """ Sets camera intrinsics from a source with following priority 1. from config function parameter if defined 2. from custom properties of cam if set in Loader 3. default config: resolution_x/y: 512 pixel_aspect_x: 1 clip_start: : 0.1 clip_end : 1000 fov : 0.691111 :param cam: The camera which contains only camera specific attributes. :param config: A configuration object with cam intrinsics. """ if config.is_empty(): return width = config.get_int("resolution_x", bpy.context.scene.render.resolution_x) height = config.get_int("resolution_y", bpy.context.scene.render.resolution_y) # Clipping clip_start = config.get_float("clip_start", cam.clip_start) clip_end = config.get_float("clip_end", cam.clip_end) if config.has_param("cam_K"): if config.has_param("fov"): print( 'WARNING: FOV defined in config is ignored. Mutually exclusive with cam_K' ) if config.has_param("pixel_aspect_x"): print( 'WARNING: pixel_aspect_x defined in config is ignored. Mutually exclusive with cam_K' ) cam_K = np.array(config.get_list("cam_K")).reshape(3, 3).astype( np.float32) CameraUtility.set_intrinsics_from_K_matrix(cam_K, width, height, clip_start, clip_end) else: # Set FOV fov = config.get_float("fov", cam.angle) # Set Pixel Aspect Ratio pixel_aspect_x = config.get_float( "pixel_aspect_x", bpy.context.scene.render.pixel_aspect_x) pixel_aspect_y = config.get_float( "pixel_aspect_y", bpy.context.scene.render.pixel_aspect_y) # Set camera shift shift_x = config.get_float("shift_x", cam.shift_x) shift_y = config.get_float("shift_y", cam.shift_y) CameraUtility.set_intrinsics_from_blender_params(fov, width, height, clip_start, clip_end, pixel_aspect_x, pixel_aspect_y, shift_x, shift_y, lens_unit="FOV") CameraUtility.set_stereo_parameters( config.get_string("stereo_convergence_mode", cam.stereo.convergence_mode), config.get_float("convergence_distance", cam.stereo.convergence_distance), config.get_float("interocular_distance", cam.stereo.interocular_distance)) if config.has_param("depth_of_field"): depth_of_field_config = Config( config.get_raw_dict("depth_of_field")) fstop_value = depth_of_field_config.get_float("fstop", 2.4) aperture_blades = depth_of_field_config.get_int( "aperture_blades", 0) aperture_ratio = depth_of_field_config.get_float( "aperture_ratio", 1.0) aperture_rotation = depth_of_field_config.get_float( "aperture_rotation_in_rad", 0.0) if depth_of_field_config.has_param( "depth_of_field_dist") and depth_of_field_config.has_param( "focal_object"): raise RuntimeError( "You can only use either depth_of_field_dist or a focal_object but not both!" ) if depth_of_field_config.has_param("depth_of_field_dist"): depth_of_field_dist = depth_of_field_config.get_float( "depth_of_field_dist") CameraUtility.add_depth_of_field(cam, None, fstop_value, aperture_blades, aperture_rotation, aperture_ratio, depth_of_field_dist) elif depth_of_field_config.has_param("focal_object"): focal_object = depth_of_field_config.get_list("focal_object") if len(focal_object) != 1: raise RuntimeError( f"There has to be exactly one focal object, use 'random_samples: 1' or change " f"the selector. Found {len(focal_object)}.") CameraUtility.add_depth_of_field(Entity(focal_object[0]), fstop_value, aperture_blades, aperture_rotation, aperture_ratio) else: raise RuntimeError( "The depth_of_field dict must contain either a focal_object definition or " "a depth_of_field_dist")