def run(self):
""" Performs physics simulation in the scene. """
# locations of all soon to be active objects before we shift their origin points
locations_before_origin_shift = {}
for obj in get_all_blender_mesh_objects():
if obj["physics"]:
locations_before_origin_shift.update({obj.name: obj.location.copy()})
# enable rigid body and shift origin point for active objects
locations_after_origin_shift = self._add_rigidbody()
# compute origin point shift for all active objects
origin_shift = {}
for obj in locations_after_origin_shift:
shift = locations_before_origin_shift[obj] - locations_after_origin_shift[obj]
origin_shift.update({obj: shift})
bpy.context.scene.rigidbody_world.substeps_per_frame = self.substeps_per_frame
bpy.context.scene.rigidbody_world.solver_iterations = self.solver_iters
obj_poses_before_sim = self._get_pose()
# perform simulation
obj_poses_after_sim = self._do_simulation()
# reset origin point of all active objects to the total shift location of the 3D cursor
for obj in get_all_blender_mesh_objects():
if obj.rigid_body.type == "ACTIVE":
bpy.context.view_layer.objects.active = obj
obj.select_set(True)
# compute relative object rotation before and after simulation
R_obj_before_sim = mathutils.Euler(obj_poses_before_sim[obj.name]['rotation']).to_matrix()
R_obj_after = mathutils.Euler(obj_poses_after_sim[obj.name]['rotation']).to_matrix()
R_obj_rel = R_obj_before_sim @ R_obj_after.transposed()
# compute origin shift in object coordinates
origin_shift[obj.name] = R_obj_rel.transposed() @ origin_shift[obj.name]
# set 3d cursor location to the total shift of the object
bpy.context.scene.cursor.location = origin_shift[obj.name] + obj_poses_after_sim[obj.name]['location']
bpy.ops.object.origin_set(type='ORIGIN_CURSOR', center='MEDIAN')
obj.select_set(False)
# reset 3D cursor location
bpy.context.scene.cursor.location = mathutils.Vector([0, 0, 0])
# get current poses
curr_pose = self._get_pose()
# displace for the origin shift
final_poses = {}
for obj in curr_pose:
final_poses.update({obj: {'location': curr_pose[obj]['location'] + origin_shift[obj], 'rotation': curr_pose[obj]['rotation']}})
self._set_pose(final_poses)
self._remove_rigidbody()
def run(self):
""" Stores frames and annotations for objects from the specified dataset.
"""
all_mesh_objects = get_all_blender_mesh_objects()
# Select objects from the specified dataset.
if self.dataset:
self.dataset_objects = []
for obj in all_mesh_objects:
if "bop_dataset_name" in obj:
if obj["bop_dataset_name"] == self.dataset:
self.dataset_objects.append(obj)
else:
self.dataset_objects = all_mesh_objects
# Check if there is any object from the specified dataset.
if not self.dataset_objects:
raise Exception(
"The scene does not contain any object from the "
"specified dataset: {}. Either remove the dataset parameter "
"or assign custom property 'bop_dataset_name' to selected objects"
.format(self.dataset))
# Get the camera.
cam_ob = bpy.context.scene.camera
self.cam = cam_ob.data
self.cam_pose = (self.cam, cam_ob)
# Save the data.
self._write_camera()
self._write_frames()
def _add_rigidbody(self):
""" Adds a rigidbody element to all mesh objects and sets their type depending on the custom property "physics".
:return: Object locations after origin point shift. Type: dict.
"""
locations_after_origin_shift = {}
for obj in get_all_blender_mesh_objects():
bpy.context.view_layer.objects.active = obj
bpy.ops.rigidbody.object_add()
if "physics" not in obj:
raise Exception("The obj: '{}' has no physics attribute, each object needs one.".format(obj.name))
obj.rigid_body.type = "ACTIVE" if obj["physics"] else "PASSIVE"
obj.select_set(True)
if obj in self.config.get_list("objs_with_box_collision_shape", []):
obj.rigid_body.collision_shape = "BOX"
else:
obj.rigid_body.collision_shape = self.collision_shape
obj.rigid_body.collision_margin = self.collision_margin
obj.rigid_body.use_margin = True
obj.rigid_body.mesh_source = self.collision_mesh_source
obj.rigid_body.friction = self.friction
obj.rigid_body.angular_damping = self.angular_damping
obj.rigid_body.linear_damping = self.linear_damping
if obj.rigid_body.type == "ACTIVE":
bpy.ops.object.origin_set(type='ORIGIN_CENTER_OF_VOLUME', center='MEDIAN')
bpy.ops.object.transform_apply(location=False, rotation=False, scale=True)
locations_after_origin_shift.update({obj.name: obj.location.copy()})
if self.mass_scaling:
obj.rigid_body.mass = get_bound_volume(obj) * self.mass_factor
obj.select_set(False)
return locations_after_origin_shift
def run(self):
""" Collect all mesh objects and writes their id, name and pose."""
objects = []
for object in get_all_blender_mesh_objects():
objects.append(object)
self.write_attributes_to_file(self.object_writer, objects, "object_states_", "object_states",
["id", "name", "location", "rotation_euler", "matrix_world"])
def sample(objects_to_sample: [MeshObject], sample_pose_func: Callable[[MeshObject], None], objects_to_check_collisions: [MeshObject] = None, max_tries: int = 1000):
""" Samples positions and rotations of selected object inside the sampling volume while performing mesh and bounding box collision checks.
:param objects_to_sample: A list of mesh objects whose poses are sampled based on the given function.
:param sample_pose_func: The function to use for sampling the pose of a given object.
:param objects_to_check_collisions: A list of mesh objects who should not be considered when checking for collisions.
:param max_tries: Amount of tries before giving up on an object and moving to the next one.
"""
# After this many tries we give up on current object and continue with the rest
if objects_to_check_collisions is None:
objects_to_check_collisions = MeshObject.convert_to_meshes(get_all_blender_mesh_objects())
# Among objects_to_sample only check collisions against already placed objects
cur_objects_to_check_collisions = list(set(objects_to_check_collisions) - set(objects_to_sample))
if max_tries <= 0:
raise ValueError("The value of max_tries must be greater than zero: {}".format(max_tries))
if not objects_to_sample:
raise Exception("The list of objects_to_sample can not be empty!")
# cache to fasten collision detection
bvh_cache = {}
# for every selected object
for obj in objects_to_sample:
no_collision = True
amount_of_tries_done = -1
# Try max_iter amount of times
for i in range(max_tries):
# Put the top object in queue at the sampled point in space
sample_pose_func(obj)
bpy.context.view_layer.update()
# Remove bvh cache, as object has changed
if obj.get_name() in bvh_cache:
del bvh_cache[obj.get_name()]
no_collision = CollisionUtility.check_intersections(obj, bvh_cache, cur_objects_to_check_collisions, [])
# If no collision then keep the position
if no_collision:
amount_of_tries_done = i
break
# After placing an object, we will check collisions with it
cur_objects_to_check_collisions.append(obj)
if amount_of_tries_done == -1:
amount_of_tries_done = max_tries
if not no_collision:
print("Could not place " + obj.get_name() + " without a collision.")
else:
print("It took " + str(amount_of_tries_done + 1) + " tries to place " + obj.get_name())
def run(self):
"""
Samples positions and rotations of selected object inside the sampling volume while performing mesh and
bounding box collision checks in the following steps:
1. While we have objects remaining and have not run out of tries - sample a point.
2. If no collisions are found keep the point.
"""
# While we have objects remaining and have not run out of tries - sample a point
# List of successfully placed objects
placed = []
# After this many tries we give up on current object and continue with the rest
max_tries = self.config.get_int("max_iterations", 1000)
objects = self.config.get_list("objects_to_sample",
get_all_blender_mesh_objects())
if max_tries <= 0:
raise ValueError(
"The value of max_tries must be greater than zero: {}".format(
max_tries))
if not objects:
raise Exception("The list of objects can not be empty!")
# cache to fasten collision detection
bvh_cache = {}
# for every selected object
for obj in objects:
if obj.type == "MESH":
no_collision = True
amount_of_tries_done = -1
# Try max_iter amount of times
for i in range(max_tries):
# Put the top object in queue at the sampled point in space
position = self.config.get_vector3d("pos_sampler")
rotation = self.config.get_vector3d("rot_sampler")
no_collision = ObjectPoseSampler.check_pose_for_object(
obj, position, rotation, bvh_cache, placed, [])
# If no collision then keep the position
if no_collision:
amount_of_tries_done = i
break
if amount_of_tries_done == -1:
amount_of_tries_done = max_tries
placed.append(obj)
if not no_collision:
print("Could not place " + obj.name +
" without a collision.")
else:
print("It took " + str(amount_of_tries_done + 1) +
" tries to place " + obj.name)
def simulate(min_simulation_time: float = 4.0,
max_simulation_time: float = 40.0,
check_object_interval: float = 2.0,
object_stopped_location_threshold: float = 0.01,
object_stopped_rotation_threshold: float = 0.1,
substeps_per_frame: int = 10,
solver_iters: int = 10) -> dict:
""" Simulates the current scene.
The simulation is run for at least `min_simulation_time` seconds and at a maximum `max_simulation_time` seconds.
Every `check_object_interval` seconds, it is checked if the maximum object movement in the last second is below a given threshold.
If that is the case, the simulation is stopped.
The origin of all objects is set to their center of mass in this function which is necessary to achieve a realistic simulation in blender (see https://blender.stackexchange.com/questions/167488/physics-not-working-as-expected)
Also the scale of each participating object is persisted as scale != 1 can make the simulation unstable.
:param min_simulation_time: The minimum number of seconds to simulate.
:param max_simulation_time: The maximum number of seconds to simulate.
:param check_object_interval: The interval in seconds at which all objects should be checked if they are still moving. If all objects
have stopped moving, than the simulation will be stopped.
:param object_stopped_location_threshold: The maximum difference per second and per coordinate in the rotation Euler vector that is allowed. such
that an object is still recognized as 'stopped moving'.
:param object_stopped_rotation_threshold: The maximum difference per second and per coordinate in the rotation Euler vector that is allowed. such
that an object is still recognized as 'stopped moving'.
:param substeps_per_frame: Number of simulation steps taken per frame.
:param solver_iters: Number of constraint solver iterations made per simulation step.
:return: A dict containing for every active object the shift that was added to their origins.
"""
# Shift the origin of all objects to their center of mass to make the simulation more realistic
origin_shift = {}
objects_with_physics = [
MeshObject(obj) for obj in get_all_blender_mesh_objects()
if obj.rigid_body is not None
]
for obj in objects_with_physics:
prev_origin = obj.get_origin()
new_origin = obj.set_origin(mode="CENTER_OF_VOLUME")
origin_shift[obj.get_name()] = new_origin - prev_origin
# Persist mesh scaling as having a scale != 1 can make the simulation unstable
obj.persist_transformation_into_mesh(location=False,
rotation=False,
scale=True)
# Configure simulator
bpy.context.scene.rigidbody_world.substeps_per_frame = substeps_per_frame
bpy.context.scene.rigidbody_world.solver_iterations = solver_iters
# Perform simulation
PhysicsSimulation._do_simulation(min_simulation_time,
max_simulation_time,
check_object_interval,
object_stopped_location_threshold,
object_stopped_rotation_threshold)
return origin_shift
def run(self):
"""
:return: Point of interest in the scene. Type: mathutils.Vector.
"""
# For every selected object in the scene
selected_objects = MeshObject.convert_to_meshes(
self.config.get_list("selector", get_all_blender_mesh_objects()))
if len(selected_objects) == 0:
raise Exception("No objects were selected!")
return MeshObject.compute_poi(selected_objects)
def run(self):
""" Collect ShapeNet attributes and write them to a file."""
shapenet_objects = [
obj for obj in get_all_blender_mesh_objects()
if "used_synset_id" in obj
]
self.write_attributes_to_file(self.object_writer, shapenet_objects,
"shapenet_", "shapenet",
["used_synset_id", "used_source_id"])
def _get_pose(self):
"""Returns position and rotation values of all objects in the scene with ACTIVE rigid_body type.
:return: Dict of form {obj_name:{'location':[x, y, z], 'rotation':[x_rot, y_rot, z_rot]}}.
"""
objects_poses = {}
for obj in get_all_blender_mesh_objects():
if obj.rigid_body.type == 'ACTIVE':
location = bpy.context.scene.objects[obj.name].matrix_world.translation.copy()
rotation = mathutils.Vector(bpy.context.scene.objects[obj.name].matrix_world.to_euler())
objects_poses.update({obj.name: {'location': location, 'rotation': rotation}})
return objects_poses
def run(self):
"""
Samples positions and rotations of selected object inside the sampling volume while performing mesh and
bounding box collision checks in the following steps:
1. While we have objects remaining and have not run out of tries - sample a point.
2. If no collisions are found keep the point.
"""
objects_to_sample = self.config.get_list(
"objects_to_sample", get_all_blender_mesh_objects())
objects_to_check_collisions = self.config.get_list(
"objects_to_check_collisions", get_all_blender_mesh_objects())
max_tries = self.config.get_int("max_iterations", 1000)
def sample_pose(obj: MeshObject):
obj.set_location(self.config.get_vector3d("pos_sampler"))
obj.set_rotation_euler(self.config.get_vector3d("rot_sampler"))
ObjectPoseSampler.sample(
objects_to_sample=MeshObject.convert_to_meshes(objects_to_sample),
sample_pose_func=sample_pose,
objects_to_check_collisions=MeshObject.convert_to_meshes(
objects_to_check_collisions),
max_tries=max_tries)
def run(self):
total_noof_cams = self.config.get_int("total_noof_cams", 10)
noof_cams_per_scene = self.config.get_int("noof_cams_per_scene", 5)
for i in range(total_noof_cams):
if i % noof_cams_per_scene == 0:
# sample new object poses
self._object_pose_sampler.run()
# get current keyframe id
frame_id = bpy.context.scene.frame_end
# TODO: Use Getter for selecting objects
for obj in get_all_blender_mesh_objects():
# insert keyframes for current object poses
self.insert_key_frames(obj, frame_id)
# sample new camera poses
self._camera_pose_sampler.run()
def _add_rigidbody(self):
""" Adds a rigidbody element to all mesh objects and sets their physics attributes depending on their custom properties """
# Temporary function which returns either the value set in the custom properties (if set) or the fallback value.
def get_physics_attribute(obj, cp_name, default_value):
if cp_name in obj:
return obj[cp_name]
else:
return default_value
# Go over all mesh objects and set their physics attributes based on the custom properties or (if not set) based on the module config
for obj in get_all_blender_mesh_objects():
mesh_obj = MeshObject(obj)
# Skip if the object has already an active rigid body component
if mesh_obj.get_rigidbody() is None:
if "physics" not in obj:
raise Exception("The obj: '{}' has no physics attribute, each object needs one.".format(obj.name))
# Collect physics attributes
collision_shape = get_physics_attribute(obj, "physics_collision_shape", self.collision_shape)
collision_margin = get_physics_attribute(obj, "physics_collision_margin", self.collision_margin)
mass = get_physics_attribute(obj, "physics_mass", None if self.mass_scaling else 1)
collision_mesh_source = get_physics_attribute(obj, "physics_collision_mesh_source", self.collision_mesh_source)
friction = get_physics_attribute(obj, "physics_friction", self.friction)
angular_damping = get_physics_attribute(obj, "physics_angular_damping", self.angular_damping)
linear_damping = get_physics_attribute(obj, "physics_linear_damping", self.linear_damping)
# Set physics attributes
mesh_obj.enable_rigidbody(
active=obj["physics"],
collision_shape="COMPOUND" if collision_shape == "CONVEX_DECOMPOSITION" else collision_shape,
collision_margin=collision_margin,
mass=mass,
mass_factor=self.mass_factor,
collision_mesh_source=collision_mesh_source,
friction=friction,
angular_damping=angular_damping,
linear_damping=linear_damping
)
# Check if object needs decomposition
if collision_shape == "CONVEX_DECOMPOSITION":
mesh_obj.build_convex_decomposition_collision_shape(self._temp_dir, self.convex_decomposition_cache_path)
def run(self):
""" Collect all mesh objects and writes their id, name and pose."""
if ('selector' in self.config.data.keys()):
sel_objs = {}
sel_objs['selector'] = self.config.data['selector']
# create Config objects
sel_conf = Config(sel_objs)
# invoke a Getter, get a list of entities to manipulate
objects = sel_conf.get_list("selector")
else:
objects = []
for object in get_all_blender_mesh_objects():
objects.append(object)
self.write_attributes_to_file(
self.object_writer, objects, "object_states_", "object_states",
["name", "location", "rotation_euler", "matrix_world"])
def render(output_dir: Union[str, None] = None, file_prefix: str = "rgb_", output_key: str = "colors",
load_keys: Set = None, return_data: bool = True) -> Dict[str, List[np.ndarray]]:
""" Render all frames.
This will go through all frames from scene.frame_start to scene.frame_end and render each of them.
:param output_dir: The directory to write files to, if this is None the temporary directory is used. \
The temporary directory is usually in the shared memory (only true for linux).
:param file_prefix: The prefix to use for writing the images.
:param output_key: The key to use for registering the output.
:param load_keys: Set of output keys to load when available
:param return_data: Whether to load and return generated data. Backwards compatibility to config-based pipeline.
:return: dict of lists of raw renderer output. Keys can be 'distance', 'colors', 'normals'
"""
if output_dir is None:
output_dir = Utility.get_temporary_directory()
if load_keys is None:
load_keys = {'colors', 'distance', 'normals'}
if output_key is not None:
Utility.add_output_entry({
"key": output_key,
"path": os.path.join(output_dir, file_prefix) + "%04d" +
RendererUtility.map_file_format_to_file_ending(bpy.context.scene.render.image_settings.file_format),
"version": "2.0.0"
})
load_keys.add(output_key)
bpy.context.scene.render.filepath = os.path.join(output_dir, file_prefix)
# Skip if there is nothing to render
if bpy.context.scene.frame_end != bpy.context.scene.frame_start:
if len(get_all_blender_mesh_objects()) == 0:
raise Exception("There are no mesh-objects to render, "
"please load an object before invoking the renderer.")
# As frame_end is pointing to the next free frame, decrease it by one, as
# blender will render all frames in [frame_start, frame_ned]
bpy.context.scene.frame_end -= 1
bpy.ops.render.render(animation=True, write_still=True)
# Revert changes
bpy.context.scene.frame_end += 1
return WriterUtility.load_registered_outputs(load_keys) if return_data else {}
def run(self):
all_objects = get_all_blender_mesh_objects()
front_3D_objs = [obj for obj in all_objects if "is_3D_future" in obj and obj["is_3D_future"]]
floor_objs = [obj for obj in front_3D_objs if obj.name.lower().startswith("floor")]
# count objects per floor -> room
floor_obj_counters = {obj.name: 0 for obj in floor_objs}
counter = 0
for obj in front_3D_objs:
name = obj.name.lower()
if "wall" in name or "ceiling" in name:
continue
counter += 1
location = obj.location
for floor_obj in floor_objs:
is_above = self._position_is_above_object(location, floor_obj)
if is_above:
floor_obj_counters[floor_obj.name] += 1
amount_of_objects_needed_per_room = self.config.get_int("amount_of_objects_needed_per_room", 2)
self.used_floors = [obj for obj in floor_objs if floor_obj_counters[obj.name] > amount_of_objects_needed_per_room]
super().run()
def _init_bvh_tree(self):
""" Creates a bvh tree which contains all mesh objects in the scene.
Such a tree is later used for fast raycasting.
"""
# Create bmesh which will contain the meshes of all objects
bm = bmesh.new()
# Go through all mesh objects
for obj in get_all_blender_mesh_objects():
if obj in self.excluded_objects_in_proximity_check:
continue
# Add object mesh to bmesh (the newly added vertices will be automatically selected)
bm.from_mesh(obj.data)
# Apply world matrix to all selected vertices
bm.transform(obj.matrix_world, filter={"SELECT"})
# Deselect all vertices
for v in bm.verts:
v.select = False
# Create tree from bmesh
self.bvh_tree = mathutils.bvhtree.BVHTree.FromBMesh(bm)
self._is_bvh_tree_inited = True
def render(output_dir, file_prefix="rgb_", output_key="colors"):
""" Render all frames.
This will go through all frames from scene.frame_start to scene.frame_end and render each of them.
:param output_dir: The directory to write images to.
:param file_prefix: The prefix to use for writing the images.
:param output_key: The key to use for registering the output.
"""
if output_key is not None:
Utility.add_output_entry({
"key":
output_key,
"path":
os.path.join(output_dir, file_prefix) + "%04d" +
RendererUtility.map_file_format_to_file_ending(
bpy.context.scene.render.image_settings.file_format),
"version":
"2.0.0"
})
bpy.context.scene.render.filepath = os.path.join(
output_dir, file_prefix)
# Skip if there is nothing to render
if bpy.context.scene.frame_end != bpy.context.scene.frame_start:
if len(get_all_blender_mesh_objects()) == 0:
raise Exception(
"There are no mesh-objects to render, "
"please load an object before invoking the renderer.")
# As frame_end is pointing to the next free frame, decrease it by one, as
# blender will render all frames in [frame_start, frame_ned]
bpy.context.scene.frame_end -= 1
bpy.ops.render.render(animation=True, write_still=True)
# Revert changes
bpy.context.scene.frame_end += 1
def run(self):
"""
:return: Point of interest in the scene. Type: mathutils.Vector.
"""
# Init matrix for all points of all bounding boxes
mean_bb_points = []
# For every selected object in the scene
selected_objects = self.config.get_list("selector",
get_all_blender_mesh_objects())
if len(selected_objects) == 0:
raise Exception("No objects were selected!")
for obj in selected_objects:
# Get bounding box corners
bb_points = get_bounds(obj)
# Compute mean coords of bounding box
mean_bb_points.append(np.mean(bb_points, axis=0))
# Query point - mean of means
mean_bb_point = np.mean(mean_bb_points, axis=0)
# Closest point (from means) to query point (mean of means)
poi = mathutils.Vector(mean_bb_points[np.argmin(
np.linalg.norm(mean_bb_points - mean_bb_point, axis=1))])
return poi
def get_all_meshes_with_name(name: str):
return MeshObject.convert_to_meshes([obj for obj in get_all_blender_mesh_objects() if obj.name == name])
def run(self):
self.point_sampler = Front3DPointInRoomSampler(
MeshObject.convert_to_meshes(get_all_blender_mesh_objects()))
super().run()
def get_all_mesh_objects():
return MeshObject.convert_to_meshes(get_all_blender_mesh_objects())
def _sample_cam_poses(self, config):
""" Samples camera poses according to the given config
:param config: The config object
"""
cam_ob = bpy.context.scene.camera
cam = cam_ob.data
# Set global parameters
self.sqrt_number_of_rays = config.get_int("sqrt_number_of_rays", 10)
self.max_tries = config.get_int("max_tries", 10000)
self.proximity_checks = config.get_raw_dict("proximity_checks", {})
self.excluded_objects_in_proximity_check = config.get_list(
"excluded_objs_in_proximity_check", [])
self.min_interest_score = config.get_float("min_interest_score", 0.0)
self.interest_score_range = config.get_float("interest_score_range",
self.min_interest_score)
self.interest_score_step = config.get_float("interest_score_step", 0.1)
self.special_objects = config.get_list("special_objects", [])
self.special_objects_weight = config.get_float(
"special_objects_weight", 2)
self._above_objects = MeshObject.convert_to_meshes(
config.get_list("check_if_pose_above_object_list", []))
self.check_visible_objects = MeshObject.convert_to_meshes(
config.get_list("check_if_objects_visible", []))
# Set camera intrinsics
self._set_cam_intrinsics(
cam, Config(self.config.get_raw_dict("intrinsics", {})))
if self.proximity_checks:
# needs to build an bvh tree
mesh_objects = [
MeshObject(obj) for obj in get_all_blender_mesh_objects()
if obj not in self.excluded_objects_in_proximity_check
]
self.bvh_tree = MeshObject.create_bvh_tree_multi_objects(
mesh_objects)
if self.interest_score_step <= 0.0:
raise Exception(
"Must have an interest score step size bigger than 0")
# Determine the number of camera poses to sample
number_of_poses = config.get_int("number_of_samples", 1)
print("Sampling " + str(number_of_poses) + " cam poses")
# Start with max interest score
self.interest_score = self.interest_score_range
# Init
all_tries = 0
tries = 0
existing_poses = []
for i in range(number_of_poses):
# Do until a valid pose has been found or the max number of tries has been reached
while tries < self.max_tries:
tries += 1
all_tries += 1
# Sample a new cam pose and check if its valid
if self.sample_and_validate_cam_pose(config, existing_poses):
break
# If max tries has been reached
if tries >= self.max_tries:
# Decrease interest score and try again, if we have not yet reached minimum
continue_trying, self.interest_score = CameraValidation.decrease_interest_score(
self.interest_score, self.min_interest_score,
self.interest_score_step)
if continue_trying:
tries = 0
print(str(all_tries) + " tries were necessary")
def render(output_dir,
temp_dir,
used_attributes,
used_default_values={},
file_prefix="segmap_",
output_key="segmap",
segcolormap_output_file_prefix="class_inst_col_map",
segcolormap_output_key="segcolormap",
use_alpha_channel=False,
render_colorspace_size_per_dimension=2048):
""" Renders segmentation maps for all frames.
:param output_dir: The directory to write images to.
:param temp_dir: The directory to write intermediate data to.
:param used_attributes: The attributes to be used for color mapping.
:param used_default_values: The default values used for the keys used in used_attributes.
:param file_prefix: The prefix to use for writing the images.
:param output_key: The key to use for registering the output.
:param segcolormap_output_file_prefix: The prefix to use for writing the segmation-color map csv.
:param segcolormap_output_key: The key to use for registering the segmation-color map output.
:param use_alpha_channel: If true, the alpha channel stored in .png textures is used.
:param render_colorspace_size_per_dimension: As we use float16 for storing the rendering, the interval of integers which can be precisely stored is [-2048, 2048]. As blender does not allow negative values for colors, we use [0, 2048] ** 3 as our color space which allows ~8 billion different colors/objects. This should be enough.
"""
with Utility.UndoAfterExecution():
RendererUtility.init()
RendererUtility.set_samples(1)
RendererUtility.set_adaptive_sampling(0)
RendererUtility.set_denoiser(None)
RendererUtility.set_light_bounces(1, 0, 0, 1, 0, 8, 0)
# Get objects with meshes (i.e. not lights or cameras)
objs_with_mats = get_all_blender_mesh_objects()
colors, num_splits_per_dimension, used_objects = SegMapRendererUtility._colorize_objects_for_instance_segmentation(
objs_with_mats, use_alpha_channel,
render_colorspace_size_per_dimension)
bpy.context.scene.cycles.filter_width = 0.0
if use_alpha_channel:
MaterialLoaderUtility.add_alpha_channel_to_textures(
blurry_edges=False)
# Determine path for temporary and for final output
temporary_segmentation_file_path = os.path.join(temp_dir, "seg_")
final_segmentation_file_path = os.path.join(
output_dir, file_prefix)
RendererUtility.set_output_format("OPEN_EXR", 16)
RendererUtility.render(temp_dir, "seg_", None)
# Find optimal dtype of output based on max index
for dtype in [np.uint8, np.uint16, np.uint32]:
optimal_dtype = dtype
if np.iinfo(optimal_dtype).max >= len(colors) - 1:
break
if 'class' in used_default_values:
used_default_values['cp_category_id'] = used_default_values[
'class']
if isinstance(used_attributes, str):
# only one result is requested
result_channels = 1
used_attributes = [used_attributes]
elif isinstance(used_attributes, list):
result_channels = len(used_attributes)
else:
raise Exception(
"The type of this is not supported here: {}".format(
used_attributes))
save_in_csv_attributes = {}
# define them for the avoid rendering case
there_was_an_instance_rendering = False
list_of_used_attributes = []
# Check if stereo is enabled
if bpy.context.scene.render.use_multiview:
suffixes = ["_L", "_R"]
else:
suffixes = [""]
# After rendering
for frame in range(
bpy.context.scene.frame_start,
bpy.context.scene.frame_end): # for each rendered frame
for suffix in suffixes:
file_path = temporary_segmentation_file_path + (
"%04d" % frame) + suffix + ".exr"
segmentation = load_image(file_path)
print(file_path, segmentation.shape)
segmap = Utility.map_back_from_equally_spaced_equidistant_values(
segmentation, num_splits_per_dimension,
render_colorspace_size_per_dimension)
segmap = segmap.astype(optimal_dtype)
used_object_ids = np.unique(segmap)
max_id = np.max(used_object_ids)
if max_id >= len(used_objects):
raise Exception(
"There are more object colors than there are objects"
)
combined_result_map = []
there_was_an_instance_rendering = False
list_of_used_attributes = []
used_channels = []
for channel_id in range(result_channels):
resulting_map = np.empty(
(segmap.shape[0], segmap.shape[1]))
was_used = False
current_attribute = used_attributes[channel_id]
org_attribute = current_attribute
# if the class is used the category_id attribute is evaluated
if current_attribute == "class":
current_attribute = "cp_category_id"
# in the instance case the resulting ids are directly used
if current_attribute == "instance":
there_was_an_instance_rendering = True
resulting_map = segmap
was_used = True
# a non default value was also used
non_default_value_was_used = True
else:
if current_attribute != "cp_category_id":
list_of_used_attributes.append(
current_attribute)
# for the current attribute remove cp_ and _csv, if present
used_attribute = current_attribute
if used_attribute.startswith("cp_"):
used_attribute = used_attribute[len("cp_"):]
# check if a default value was specified
default_value_set = False
if current_attribute in used_default_values or used_attribute in used_default_values:
default_value_set = True
if current_attribute in used_default_values:
default_value = used_default_values[
current_attribute]
elif used_attribute in used_default_values:
default_value = used_default_values[
used_attribute]
last_state_save_in_csv = None
# this avoids that for certain attributes only the default value is written
non_default_value_was_used = False
# iterate over all object ids
for object_id in used_object_ids:
is_default_value = False
# get the corresponding object via the id
current_obj = used_objects[object_id]
# if the current obj has a attribute with that name -> get it
if hasattr(current_obj, used_attribute):
used_value = getattr(
current_obj, used_attribute)
# if the current object has a custom property with that name -> get it
elif current_attribute.startswith(
"cp_"
) and used_attribute in current_obj:
used_value = current_obj[used_attribute]
elif current_attribute.startswith("cf_"):
if current_attribute == "cf_basename":
used_value = current_obj.name
if "." in used_value:
used_value = used_value[:used_value
.rfind("."
)]
elif default_value_set:
# if none of the above applies use the default value
used_value = default_value
is_default_value = True
else:
# if the requested current_attribute is not a custom property or a attribute
# or there is a default value stored
# it throws an exception
raise Exception(
"The obj: {} does not have the "
"attribute: {}, striped: {}. Maybe try a default "
"value.".format(
current_obj.name,
current_attribute, used_attribute))
# check if the value should be saved as an image or in the csv file
save_in_csv = False
try:
resulting_map[segmap ==
object_id] = used_value
was_used = True
if not is_default_value:
non_default_value_was_used = True
# save everything which is not instance also in the .csv
if current_attribute != "instance":
save_in_csv = True
except ValueError:
save_in_csv = True
if last_state_save_in_csv is not None and last_state_save_in_csv != save_in_csv:
raise Exception(
"During creating the mapping, the saving to an image or a csv file "
"switched, this might indicated that the used default value, does "
"not have the same type as the returned value, "
"for: {}".format(current_attribute))
last_state_save_in_csv = save_in_csv
if save_in_csv:
if object_id in save_in_csv_attributes:
save_in_csv_attributes[object_id][
used_attribute] = used_value
else:
save_in_csv_attributes[object_id] = {
used_attribute: used_value
}
if was_used and non_default_value_was_used:
used_channels.append(org_attribute)
combined_result_map.append(resulting_map)
fname = final_segmentation_file_path + ("%04d" %
frame) + suffix
# combine all resulting images to one image
resulting_map = np.stack(combined_result_map, axis=2)
# remove the unneeded third dimension
if resulting_map.shape[2] == 1:
resulting_map = resulting_map[:, :, 0]
np.save(fname, resulting_map)
if not there_was_an_instance_rendering:
if len(list_of_used_attributes) > 0:
raise Exception(
"There were attributes specified in the may_by, which could not be saved as "
"there was no \"instance\" may_by key used. This is true for this/these "
"keys: {}".format(", ".join(list_of_used_attributes)))
# if there was no instance rendering no .csv file is generated!
# delete all saved infos about .csv
save_in_csv_attributes = {}
# write color mappings to file
if save_in_csv_attributes:
csv_file_path = os.path.join(
output_dir, segcolormap_output_file_prefix + ".csv")
with open(csv_file_path, 'w', newline='') as csvfile:
# get from the first element the used field names
fieldnames = ["idx"]
# get all used object element keys
for object_element in save_in_csv_attributes.values():
fieldnames.extend(list(object_element.keys()))
break
for channel_name in used_channels:
fieldnames.append("channel_{}".format(channel_name))
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
# save for each object all values in one row
for obj_idx, object_element in save_in_csv_attributes.items(
):
object_element["idx"] = obj_idx
for i, channel_name in enumerate(used_channels):
object_element["channel_{}".format(
channel_name)] = i
writer.writerow(object_element)
Utility.register_output(output_dir, file_prefix, output_key, ".npy",
"2.0.0")
if save_in_csv_attributes:
Utility.register_output(output_dir,
segcolormap_output_file_prefix,
segcolormap_output_key,
".csv",
"2.0.0",
unique_for_camposes=False)
def simulate_and_fix_final_poses(
min_simulation_time: float = 4.0,
max_simulation_time: float = 40.0,
check_object_interval: float = 2.0,
object_stopped_location_threshold: float = 0.01,
object_stopped_rotation_threshold: float = 0.1,
substeps_per_frame: int = 10,
solver_iters: int = 10):
""" Simulates the current scene and in the end fixes the final poses of all active objects.
The simulation is run for at least `min_simulation_time` seconds and at a maximum `max_simulation_time` seconds.
Every `check_object_interval` seconds, it is checked if the maximum object movement in the last second is below a given threshold.
If that is the case, the simulation is stopped.
After performing the simulation, the simulation cache is removed, the rigid body components are disabled and the pose of the active objects is set to their final pose in the simulation.
:param min_simulation_time: The minimum number of seconds to simulate.
:param max_simulation_time: The maximum number of seconds to simulate.
:param check_object_interval: The interval in seconds at which all objects should be checked if they are still moving. If all objects
have stopped moving, than the simulation will be stopped.
:param object_stopped_location_threshold: The maximum difference per second and per coordinate in the rotation Euler vector that is allowed. such
that an object is still recognized as 'stopped moving'.
:param object_stopped_rotation_threshold: The maximum difference per second and per coordinate in the rotation Euler vector that is allowed. such
that an object is still recognized as 'stopped moving'.
:param substeps_per_frame: Number of simulation steps taken per frame.
:param solver_iters: Number of constraint solver iterations made per simulation step.
"""
# Undo changes made in the simulation like origin adjustment and persisting the object's scale
with Utility.UndoAfterExecution():
# Run simulation and remember poses before and after
obj_poses_before_sim = PhysicsSimulation._get_pose()
origin_shifts = PhysicsSimulation.simulate(
min_simulation_time, max_simulation_time,
check_object_interval, object_stopped_location_threshold,
object_stopped_rotation_threshold, substeps_per_frame,
solver_iters)
obj_poses_after_sim = PhysicsSimulation._get_pose()
# Make sure to remove the simulation cache as we are only interested in the final poses
bpy.ops.ptcache.free_bake(
{"point_cache": bpy.context.scene.rigidbody_world.point_cache})
# Fix the pose of all objects to their pose at the and of the simulation (also revert origin shift)
objects_with_physics = [
MeshObject(obj) for obj in get_all_blender_mesh_objects()
if obj.rigid_body is not None
]
for obj in objects_with_physics:
# Skip objects that have parents with compound rigid body component
has_compound_parent = obj.get_parent(
) is not None and obj.get_parent().get_rigidbody(
) is not None and obj.get_parent().get_rigidbody(
).collision_shape == "COMPOUND"
if obj.get_rigidbody(
).type == "ACTIVE" and not has_compound_parent:
# compute relative object rotation before and after simulation
R_obj_before_sim = mathutils.Euler(obj_poses_before_sim[
obj.get_name()]['rotation']).to_matrix()
R_obj_after = mathutils.Euler(obj_poses_after_sim[
obj.get_name()]['rotation']).to_matrix()
R_obj_rel = R_obj_before_sim @ R_obj_after.transposed()
# Apply relative rotation to origin shift
origin_shift = R_obj_rel.transposed() @ mathutils.Vector(
origin_shifts[obj.get_name()])
# Fix pose of object to the one it had at the end of the simulation
obj.set_location(
obj_poses_after_sim[obj.get_name()]['location'] -
origin_shift)
obj.set_rotation_euler(
obj_poses_after_sim[obj.get_name()]['rotation'])
for obj in objects_with_physics:
# Disable the rigidbody element of the object
obj.disable_rigidbody()
def write(output_dir: str,
dataset: str = "",
append_to_existing_output: bool = False,
depth_scale: float = 1.0,
save_world2cam: bool = True,
ignore_dist_thres: float = 100.,
m2mm: bool = True,
frames_per_chunk: int = 1000):
"""Write the BOP data
:param output_dir: Path to the output directory.
:param dataset: Only save annotations for objects of the specified bop dataset. Saves all object poses if undefined.
:param append_to_existing_output: If true, the new frames will be appended to the existing ones.
:param depth_scale: Multiply the uint16 output depth image with this factor to get depth in mm. Used to trade-off between depth accuracy
and maximum depth value. Default corresponds to 65.54m maximum depth and 1mm accuracy.
:param save_world2cam: If true, camera to world transformations "cam_R_w2c", "cam_t_w2c" are saved in scene_camera.json
:param ignore_dist_thres: Distance between camera and object after which object is ignored. Mostly due to failed physics.
:param m2mm: Original bop annotations and models are in mm. If true, we convert the gt annotations to mm here. This
is needed if BopLoader option mm2m is used.
:param frames_per_chunk: Number of frames saved in each chunk (called scene in BOP)
"""
# Output paths.
dataset_dir = os.path.join(output_dir, 'bop_data', dataset)
chunks_dir = os.path.join(dataset_dir, 'train_pbr')
camera_path = os.path.join(dataset_dir, 'camera.json')
# Create the output directory structure.
if not os.path.exists(dataset_dir):
os.makedirs(dataset_dir)
os.makedirs(chunks_dir)
elif not append_to_existing_output:
raise Exception(
"The output folder already exists: {}.".format(dataset_dir))
all_mesh_objects = get_all_blender_mesh_objects()
# Select objects from the specified dataset.
if dataset:
dataset_objects = []
for obj in all_mesh_objects:
if "bop_dataset_name" in obj:
if obj["bop_dataset_name"] == dataset:
dataset_objects.append(obj)
else:
dataset_objects = all_mesh_objects
# Check if there is any object from the specified dataset.
if not dataset_objects:
raise Exception(
"The scene does not contain any object from the "
"specified dataset: {}. Either remove the dataset parameter "
"or assign custom property 'bop_dataset_name' to selected objects"
.format(dataset))
# Save the data.
BopWriterUtility._write_camera(camera_path, depth_scale=depth_scale)
BopWriterUtility._write_frames(chunks_dir,
dataset_objects=dataset_objects,
frames_per_chunk=frames_per_chunk,
m2mm=m2mm,
ignore_dist_thres=ignore_dist_thres,
save_world2cam=save_world2cam)
def _remove_rigidbody(self):
""" Removes the rigidbody element from all mesh objects. """
for obj in get_all_blender_mesh_objects():
bpy.context.view_layer.objects.active = obj
bpy.ops.rigidbody.object_remove()
