def run(self):
""" Performs physics simulation in the scene. """
# locations of all soon to be active objects before we shift their origin points
locations_before_origin_shift = {}
for obj in get_all_mesh_objects():
if obj["physics"]:
locations_before_origin_shift.update(
{obj.name: obj.location.copy()})
# enable rigid body and shift origin point for active objects
locations_after_origin_shift = self._add_rigidbody()
# compute origin point shift for all active objects
origin_shift = {}
for obj in locations_after_origin_shift:
shift = locations_before_origin_shift[
obj] - locations_after_origin_shift[obj]
origin_shift.update({obj: shift})
bpy.context.scene.rigidbody_world.steps_per_second = self.steps_per_sec
bpy.context.scene.rigidbody_world.solver_iterations = self.solver_iters
# perform simulation
obj_poses = self._do_simulation()
# reset origin point of all active objects to the total shift location of the 3D cursor
for obj in get_all_mesh_objects():
if obj.rigid_body.type == "ACTIVE":
bpy.context.view_layer.objects.active = obj
obj.select_set(True)
# set 3d cursor location to the total shift of the object
bpy.context.scene.cursor.location = origin_shift[
obj.name] + obj_poses[obj.name]['location']
bpy.ops.object.origin_set(type='ORIGIN_CURSOR',
center='MEDIAN')
obj.select_set(False)
# reset 3D cursor location
bpy.context.scene.cursor.location = mathutils.Vector([0, 0, 0])
# get current poses
curr_pose = self._get_pose()
# displace for the origin shift
final_poses = {}
for obj in curr_pose:
final_poses.update({
obj: {
'location': curr_pose[obj]['location'] + origin_shift[obj],
'rotation': curr_pose[obj]['rotation']
}
})
self._set_pose(final_poses)
self._remove_rigidbody()
def run(self):
""" Randomizes materials for selected objects.
1. For each object assign a randomly chosen material from the pool.
"""
self.randomization_level = self.config.get_float(
"randomization_level", 0.2)
self._objects_to_manipulate = self.config.get_list(
'manipulated_objects', get_all_mesh_objects())
self._materials_to_replace_with = self.config.get_list(
"materials_to_replace_with", get_all_materials())
op_mode = self.config.get_string("mode", "once_for_each")
# if there were no materials selected throw an exception
if not self._materials_to_replace_with:
raise Exception("There were no materials selected!")
if op_mode == "once_for_all":
random_material = np.random.choice(self._materials_to_replace_with)
# walk over all objects
for obj in self._objects_to_manipulate:
if hasattr(obj, 'material_slots'):
# walk over all materials
for material in obj.material_slots:
if np.random.uniform(0, 1) <= self.randomization_level:
if op_mode == "once_for_each":
random_material = np.random.choice(
self._materials_to_replace_with)
# select a random material to replace the old one with
material.material = random_material
def _render(self, default_prefix, custom_file_path=None):
""" Renders each registered keypoint.
:param default_prefix: The default prefix of the output files.
"""
if self.config.get_bool("render_depth", False):
self._write_depth_to_file()
if self.config.get_bool("render_normals", False):
self._write_normal_to_file()
if custom_file_path is None:
bpy.context.scene.render.filepath = os.path.join(
self._determine_output_dir(),
self.config.get_string("output_file_prefix", default_prefix))
else:
bpy.context.scene.render.filepath = custom_file_path
# Skip if there is nothing to render
if bpy.context.scene.frame_end != bpy.context.scene.frame_start:
if len(get_all_mesh_objects()) == 0:
raise Exception(
"There are no mesh-objects to render, "
"please load an object before invoking the renderer.")
# As frame_end is pointing to the next free frame, decrease it by one, as blender will render all frames in [frame_start, frame_ned]
bpy.context.scene.frame_end -= 1
if not self._avoid_rendering:
bpy.ops.render.render(animation=True, write_still=True)
# Revert changes
bpy.context.scene.frame_end += 1
def run(self):
all_objects = get_all_mesh_objects()
front_3D_objs = [
obj for obj in all_objects
if "is_3D_future" in obj and obj["is_3D_future"]
]
floor_objs = [
obj for obj in front_3D_objs
if obj.name.lower().startswith("floor")
]
# count objects per floor -> room
floor_obj_counters = {obj.name: 0 for obj in floor_objs}
counter = 0
for obj in front_3D_objs:
name = obj.name.lower()
if "wall" in name or "ceiling" in name:
continue
counter += 1
location = obj.location
for floor_obj in floor_objs:
is_above = self._position_is_above_object(location, floor_obj)
if is_above:
floor_obj_counters[floor_obj.name] += 1
amount_of_objects_needed_per_room = self.config.get_int(
"amount_of_objects_needed_per_room", 2)
self.used_floors = [
obj for obj in floor_objs
if floor_obj_counters[obj.name] > amount_of_objects_needed_per_room
]
super().run()
def _add_rigidbody(self):
""" Adds a rigidbody element to all mesh objects and sets their type depending on the custom property "physics".
:return: Object locations after origin point shift. Type: dict.
"""
locations_after_origin_shift = {}
for obj in get_all_mesh_objects():
bpy.context.view_layer.objects.active = obj
bpy.ops.rigidbody.object_add()
if "physics" not in obj:
raise Exception("The obj: '{}' has no physics attribute, each object needs one.".format(obj.name))
obj.rigid_body.type = "ACTIVE" if obj["physics"] else "PASSIVE"
obj.select_set(True)
if obj in self.config.get_list("objs_with_box_collision_shape", []):
obj.rigid_body.collision_shape = "BOX"
else:
obj.rigid_body.collision_shape = self.collision_shape
obj.rigid_body.collision_margin = self.collision_margin
obj.rigid_body.use_margin = True
obj.rigid_body.mesh_source = self.collision_mesh_source
obj.rigid_body.friction = self.friction
obj.rigid_body.angular_damping = self.angular_damping
obj.rigid_body.linear_damping = self.linear_damping
if obj.rigid_body.type == "ACTIVE":
bpy.ops.object.origin_set(type='ORIGIN_CENTER_OF_VOLUME', center='MEDIAN')
bpy.ops.object.transform_apply(location=False, rotation=False, scale=True)
locations_after_origin_shift.update({obj.name: obj.location.copy()})
if self.mass_scaling:
obj.rigid_body.mass = get_bound_volume(obj) * self.mass_factor
obj.select_set(False)
return locations_after_origin_shift
def run(self):
""" Stores frames and annotations for objects from the specified dataset.
"""
all_mesh_objects = get_all_mesh_objects()
# Select objects from the specified dataset.
if self.dataset:
self.dataset_objects = []
for obj in all_mesh_objects:
if "bop_dataset_name" in obj:
if obj["bop_dataset_name"] == self.dataset:
self.dataset_objects.append(obj)
else:
self.dataset_objects = all_mesh_objects
# Check if there is any object from the specified dataset.
if not self.dataset_objects:
raise Exception(
"The scene does not contain any object from the "
"specified dataset: {}. Either remove the dataset parameter "
"or assign custom property 'bop_dataset_name' to selected objects"
.format(self.dataset))
# Get the camera.
cam_ob = bpy.context.scene.camera
self.cam = cam_ob.data
self.cam_pose = (self.cam, cam_ob)
# Save the data.
self._write_camera()
self._write_frames()
def run(self):
""" Collect all mesh objects and writes their id, name and pose."""
objects = []
for object in get_all_mesh_objects():
objects.append(object)
self.write_attributes_to_file(self.object_writer, objects, "object_states_", "object_states",
["id", "name", "location", "rotation_euler"])
def _add_rigidbody(self):
""" Adds a rigidbody element to all mesh objects and sets their type depending on the custom property "physics". """
for obj in get_all_mesh_objects():
bpy.context.view_layer.objects.active = obj
bpy.ops.rigidbody.object_add()
obj.rigid_body.type = "ACTIVE" if obj["physics"] else "PASSIVE"
obj.rigid_body.collision_shape = "MESH"
obj.rigid_body.collision_margin = self.collision_margin
def run(self):
# Collect all mesh objects
objects = []
for object in get_all_mesh_objects():
objects.append(object)
self.write_attributes_to_file(self.object_writer, objects,
"object_states_", "object_states",
["id", "location", "rotation_euler"])
def run(self):
"""
Samples positions and rotations of selected object inside the sampling volume while performing mesh and
bounding box collision checks in the following steps:
1. While we have objects remaining and have not run out of tries - sample a point.
2. If no collisions are found keep the point.
"""
# While we have objects remaining and have not run out of tries - sample a point
# List of successfully placed objects
placed = []
# After this many tries we give up on current object and continue with the rest
max_tries = self.config.get_int("max_iterations", 1000)
objects = self.config.get_list("objects_to_sample",
get_all_mesh_objects())
if max_tries <= 0:
raise ValueError(
"The value of max_tries must be greater than zero: {}".format(
max_tries))
if not objects:
raise Exception("The list of objects can not be empty!")
# cache to fasten collision detection
bvh_cache = {}
# for every selected object
for obj in objects:
if obj.type == "MESH":
no_collision = True
amount_of_tries_done = -1
# Try max_iter amount of times
for i in range(max_tries):
# Put the top object in queue at the sampled point in space
position = self.config.get_vector3d("pos_sampler")
rotation = self.config.get_vector3d("rot_sampler")
no_collision = ObjectPoseSampler.check_pose_for_object(
obj, position, rotation, bvh_cache, placed, [])
# If no collision then keep the position
if no_collision:
amount_of_tries_done = i
break
if amount_of_tries_done == -1:
amount_of_tries_done = max_tries
placed.append(obj)
if not no_collision:
print("Could not place " + obj.name +
" without a collision.")
else:
print("It took " + str(amount_of_tries_done + 1) +
" tries to place " + obj.name)
def run(self):
""" Collect ShapeNet attributes and write them to a file."""
shapenet_objects = [
obj for obj in get_all_mesh_objects() if "used_synset_id" in obj
]
self.write_attributes_to_file(self.object_writer, shapenet_objects,
"shapenet_", "shapenet",
["used_synset_id", "used_source_id"])
def _get_pose(self):
"""Returns position and rotation values of all objects in the scene with ACTIVE rigid_body type.
:return: Dict of form {obj_name:{'location':[x, y, z], 'rotation':[x_rot, y_rot, z_rot]}}.
"""
objects_poses = {}
for obj in get_all_mesh_objects():
if obj.rigid_body.type == 'ACTIVE':
location = bpy.context.scene.objects[obj.name].matrix_world.translation.copy()
rotation = mathutils.Vector(bpy.context.scene.objects[obj.name].matrix_world.to_euler())
objects_poses.update({obj.name: {'location': location, 'rotation': rotation}})
return objects_poses
def _replace_and_edit(self, obj_to_remove, obj_to_add, scale=True):
"""
Scale, translate, rotate obj_to_add to match obj_to_remove and check if there is a bounding box collision
returns a boolean.
:param obj_to_remove: object to remove from the scene. Type: blender object.
:param obj_to_add: object to put in the scene instead of obj_to_remove. Type: blender object.
:param scale: Scales obj_to_add to match obj_to_remove dimensions. Type: bool.
"""
def _bb_ratio(bb1, bb2):
"""
Rough estimation of the ratios between two bounding boxes sides, not axis aligned
:param bb1: bounding box 1. Type: float multi-dimensional array of 8 * 3.
:param bb2: bounding box 2. Type: float multi-dimensional array of 8 * 3.
returns the ratio between each side of the bounding box. Type: a list of floats.
"""
def _two_points_distance(point1, point2):
"""
Eclidian distance between two points
:param point1: Point 1 as a list of three floats. Type: list.
:param point2: Point 2 as a list of three floats. Type: list.
returns a float.
"""
return np.linalg.norm(np.array(point1) - np.array(point2))
ratio_a = _two_points_distance(
bb1[0], bb1[3]) / _two_points_distance(bb2[0], bb2[3])
ratio_b = _two_points_distance(
bb1[0], bb1[4]) / _two_points_distance(bb2[0], bb2[4])
ratio_c = _two_points_distance(
bb1[0], bb1[1]) / _two_points_distance(bb2[0], bb2[1])
return [ratio_a, ratio_b, ratio_c]
# New object takes location, rotation and rough scale of original object
obj_to_add.location = obj_to_remove.location
obj_to_add.rotation_euler = obj_to_remove.rotation_euler
if scale:
obj_to_add.scale = _bb_ratio(obj_to_remove.bound_box,
obj_to_add.bound_box)
# Check for collision between the new object and other objects in the scene
for obj in get_all_mesh_objects(): # for each object
if obj != obj_to_add and obj_to_remove != obj and obj not in self._ignore_collision_with:
if check_bb_intersection(obj, obj_to_add):
if check_intersection(obj, obj_to_add)[0]:
return False
return True
def _init_bvh_tree(self):
""" Creates a bvh tree which contains all mesh objects in the scene.
Such a tree is later used for fast raycasting.
"""
# Create bmesh which will contain the meshes of all objects
bm = bmesh.new()
# Go through all mesh objects
for obj in get_all_mesh_objects():
# Add object mesh to bmesh (the newly added vertices will be automatically selected)
bm.from_mesh(obj.data)
# Apply world matrix to all selected vertices
bm.transform(obj.matrix_world, filter={"SELECT"})
# Deselect all vertices
for v in bm.verts:
v.select = False
# Create tree from bmesh
self.bvh_tree = mathutils.bvhtree.BVHTree.FromBMesh(bm)
def run(self):
total_noof_cams = self.config.get_int("total_noof_cams", 10)
noof_cams_per_scene = self.config.get_int("noof_cams_per_scene", 5)
for i in range(total_noof_cams):
if i % noof_cams_per_scene == 0:
# sample new object poses
self._object_pose_sampler.run()
# get current keyframe id
frame_id = bpy.context.scene.frame_end
# TODO: Use Getter for selecting objects
for obj in get_all_mesh_objects():
# insert keyframes for current object poses
self._object_pose_sampler.insert_key_frames(obj, frame_id)
# sample new camera poses
self._camera_pose_sampler.run()
def run(self):
"""
:return: Point of interest in the scene. Type: mathutils Vector.
"""
# Init matrix for all points of all bounding boxes
mean_bb_points = []
# For every object in the scene
for obj in get_all_mesh_objects():
# Get bounding box corners
bb_points = get_bounds(obj)
# Compute mean coords of bounding box
mean_bb_points.append(np.mean(bb_points, axis=0))
# Query point - mean of means
mean_bb_point = np.mean(mean_bb_points, axis=0)
# Closest point (from means) to query point (mean of means)
poi = mathutils.Vector(mean_bb_points[np.argmin(
np.linalg.norm(mean_bb_points - mean_bb_point, axis=1))])
return poi
def run(self):
""" Randomizes materials for selected objects.
1. For each object assign a randomly chosen material from the pool.
"""
self.randomization_level = self.config.get_float(
"randomization_level", 0.2)
self._objects_to_manipulate = self.config.get_list(
'manipulated_objects', get_all_mesh_objects())
self._materials_to_replace_with = self.config.get_list(
"materials_to_replace_with", get_all_materials())
self._obj_materials_cond_to_be_replaced = self.config.get_raw_dict(
"obj_materials_cond_to_be_replaced", {})
op_mode = self.config.get_string("mode", "once_for_each")
# if there were no materials selected throw an exception
if not self._materials_to_replace_with:
print(
"Warning: No materials selected inside of the MaterialRandomizer!"
)
return
if op_mode == "once_for_all":
random_material = np.random.choice(self._materials_to_replace_with)
# walk over all objects
for obj in self._objects_to_manipulate:
if hasattr(obj, 'material_slots'):
# walk over all materials
for material in obj.material_slots:
use_mat = True
if self._obj_materials_cond_to_be_replaced:
use_mat = len(
Material.perform_and_condition_check(
self._obj_materials_cond_to_be_replaced, [],
[material.material])) == 1
if use_mat:
if np.random.uniform(0, 1) <= self.randomization_level:
if op_mode == "once_for_each":
random_material = np.random.choice(
self._materials_to_replace_with)
# select a random material to replace the old one with
material.material = random_material
def initialize_bop_groups(self):
""" Get and group all objects to their respective bop dataset
"""
all_mesh_objects = get_all_mesh_objects()
bop_datasets = {}
for obj in all_mesh_objects:
if "bop_dataset_name" in obj:
if obj["bop_dataset_name"] in bop_datasets:
bop_datasets[obj["bop_dataset_name"]].append(obj)
else:
bop_datasets[obj["bop_dataset_name"]] = [obj]
self.bop_datasets = bop_datasets
# For each group, make a seperate output directory
base_path = self._determine_output_dir(False)
self._bop_data_dir = os.path.join(base_path, 'bop_data')
# Create base directory if not exists
if not os.path.exists(self._bop_data_dir):
os.makedirs(self._bop_data_dir)
# Create subdirectorys if don't exist
self._bop_data_sub_dirs = {}
for group in self.bop_datasets:
self._bop_data_sub_dirs[group] = os.path.join(
base_path, 'bop_data', group)
if not os.path.exists(self._bop_data_sub_dirs[group]):
os.makedirs(self._bop_data_sub_dirs[group])
# For each subdirectory create internal file paths
self._scene_gt_path = {}
self._scene_camera_path = {}
self._camera_path = {}
for group in self.bop_datasets:
self._scene_gt_path[group] = os.path.join(
self._bop_data_sub_dirs[group], 'scene_gt.json')
self._scene_camera_path[group] = os.path.join(
self._bop_data_sub_dirs[group], 'scene_camera.json')
self._camera_path[group] = os.path.join(
self._bop_data_sub_dirs[group], 'camera.json')
def render(output_dir, file_prefix="rgb_", output_key="colors"):
""" Render all frames.
This will go through all frames from scene.frame_start to scene.frame_end and render each of them.
:param output_dir: The directory to write images to.
:param file_prefix: The prefix to use for writing the images.
:param output_key: The key to use for registering the output.
"""
if output_key is not None:
Utility.add_output_entry({
"key":
output_key,
"path":
os.path.join(output_dir, file_prefix) + "%04d" +
RendererUtility.map_file_format_to_file_ending(
bpy.context.scene.render.image_settings.file_format),
"version":
"2.0.0"
})
bpy.context.scene.render.filepath = os.path.join(
output_dir, file_prefix)
# Skip if there is nothing to render
if bpy.context.scene.frame_end != bpy.context.scene.frame_start:
if len(get_all_mesh_objects()) == 0:
raise Exception(
"There are no mesh-objects to render, "
"please load an object before invoking the renderer.")
# As frame_end is pointing to the next free frame, decrease it by one, as
# blender will render all frames in [frame_start, frame_ned]
bpy.context.scene.frame_end -= 1
bpy.ops.render.render(animation=True, write_still=True)
# Revert changes
bpy.context.scene.frame_end += 1
def run(self):
"""
:return: Point of interest in the scene. Type: mathutils Vector.
"""
# Init matrix for all points of all bounding boxes
mean_bb_points = []
# For every selected object in the scene
selected_objects = self.config.get_list("selector",
get_all_mesh_objects())
if len(selected_objects) == 0:
raise Exception("No objects were selected!")
for obj in selected_objects:
# Get bounding box corners
bb_points = get_bounds(obj)
# Compute mean coords of bounding box
mean_bb_points.append(np.mean(bb_points, axis=0))
# Query point - mean of means
mean_bb_point = np.mean(mean_bb_points, axis=0)
# Closest point (from means) to query point (mean of means)
poi = mathutils.Vector(mean_bb_points[np.argmin(
np.linalg.norm(mean_bb_points - mean_bb_point, axis=1))])
return poi
def render(output_dir,
temp_dir,
used_attributes,
used_default_values={},
file_prefix="segmap_",
output_key="segmap",
segcolormap_output_file_prefix="class_inst_col_map",
segcolormap_output_key="segcolormap",
use_alpha_channel=False,
render_colorspace_size_per_dimension=2048):
""" Renders segmentation maps for all frames.
:param output_dir: The directory to write images to.
:param temp_dir: The directory to write intermediate data to.
:param used_attributes: The attributes to be used for color mapping.
:param used_default_values: The default values used for the keys used in used_attributes.
:param file_prefix: The prefix to use for writing the images.
:param output_key: The key to use for registering the output.
:param segcolormap_output_file_prefix: The prefix to use for writing the segmation-color map csv.
:param segcolormap_output_key: The key to use for registering the segmation-color map output.
:param use_alpha_channel: If true, the alpha channel stored in .png textures is used.
:param render_colorspace_size_per_dimension: As we use float16 for storing the rendering, the interval of integers which can be precisely stored is [-2048, 2048]. As blender does not allow negative values for colors, we use [0, 2048] ** 3 as our color space which allows ~8 billion different colors/objects. This should be enough.
"""
with Utility.UndoAfterExecution():
RendererUtility.init()
RendererUtility.set_samples(1)
RendererUtility.set_adaptive_sampling(0)
RendererUtility.set_denoiser(None)
RendererUtility.set_light_bounces(1, 0, 0, 1, 0, 8, 0)
# Get objects with meshes (i.e. not lights or cameras)
objs_with_mats = get_all_mesh_objects()
colors, num_splits_per_dimension, used_objects = SegMapRendererUtility._colorize_objects_for_instance_segmentation(
objs_with_mats, use_alpha_channel,
render_colorspace_size_per_dimension)
bpy.context.scene.cycles.filter_width = 0.0
if use_alpha_channel:
MaterialLoaderUtility.add_alpha_channel_to_textures(
blurry_edges=False)
# Determine path for temporary and for final output
temporary_segmentation_file_path = os.path.join(temp_dir, "seg_")
final_segmentation_file_path = os.path.join(
output_dir, file_prefix)
RendererUtility.set_output_format("OPEN_EXR", 16)
RendererUtility.render(temp_dir, "seg_", None)
# Find optimal dtype of output based on max index
for dtype in [np.uint8, np.uint16, np.uint32]:
optimal_dtype = dtype
if np.iinfo(optimal_dtype).max >= len(colors) - 1:
break
if 'class' in used_default_values:
used_default_values['cp_category_id'] = used_default_values[
'class']
if isinstance(used_attributes, str):
# only one result is requested
result_channels = 1
used_attributes = [used_attributes]
elif isinstance(used_attributes, list):
result_channels = len(used_attributes)
else:
raise Exception(
"The type of this is not supported here: {}".format(
used_attributes))
save_in_csv_attributes = {}
# define them for the avoid rendering case
there_was_an_instance_rendering = False
list_of_used_attributes = []
# Check if stereo is enabled
if bpy.context.scene.render.use_multiview:
suffixes = ["_L", "_R"]
else:
suffixes = [""]
# After rendering
for frame in range(
bpy.context.scene.frame_start,
bpy.context.scene.frame_end): # for each rendered frame
for suffix in suffixes:
file_path = temporary_segmentation_file_path + (
"%04d" % frame) + suffix + ".exr"
segmentation = load_image(file_path)
print(file_path, segmentation.shape)
segmap = Utility.map_back_from_equally_spaced_equidistant_values(
segmentation, num_splits_per_dimension,
render_colorspace_size_per_dimension)
segmap = segmap.astype(optimal_dtype)
used_object_ids = np.unique(segmap)
max_id = np.max(used_object_ids)
if max_id >= len(used_objects):
raise Exception(
"There are more object colors than there are objects"
)
combined_result_map = []
there_was_an_instance_rendering = False
list_of_used_attributes = []
used_channels = []
for channel_id in range(result_channels):
resulting_map = np.empty(
(segmap.shape[0], segmap.shape[1]))
was_used = False
current_attribute = used_attributes[channel_id]
org_attribute = current_attribute
# if the class is used the category_id attribute is evaluated
if current_attribute == "class":
current_attribute = "cp_category_id"
# in the instance case the resulting ids are directly used
if current_attribute == "instance":
there_was_an_instance_rendering = True
resulting_map = segmap
was_used = True
# a non default value was also used
non_default_value_was_used = True
else:
if current_attribute != "cp_category_id":
list_of_used_attributes.append(
current_attribute)
# for the current attribute remove cp_ and _csv, if present
used_attribute = current_attribute
if used_attribute.startswith("cp_"):
used_attribute = used_attribute[len("cp_"):]
# check if a default value was specified
default_value_set = False
if current_attribute in used_default_values or used_attribute in used_default_values:
default_value_set = True
if current_attribute in used_default_values:
default_value = used_default_values[
current_attribute]
elif used_attribute in used_default_values:
default_value = used_default_values[
used_attribute]
last_state_save_in_csv = None
# this avoids that for certain attributes only the default value is written
non_default_value_was_used = False
# iterate over all object ids
for object_id in used_object_ids:
is_default_value = False
# get the corresponding object via the id
current_obj = used_objects[object_id]
# if the current obj has a attribute with that name -> get it
if hasattr(current_obj, used_attribute):
used_value = getattr(
current_obj, used_attribute)
# if the current object has a custom property with that name -> get it
elif current_attribute.startswith(
"cp_"
) and used_attribute in current_obj:
used_value = current_obj[used_attribute]
elif current_attribute.startswith("cf_"):
if current_attribute == "cf_basename":
used_value = current_obj.name
if "." in used_value:
used_value = used_value[:used_value
.rfind("."
)]
elif default_value_set:
# if none of the above applies use the default value
used_value = default_value
is_default_value = True
else:
# if the requested current_attribute is not a custom property or a attribute
# or there is a default value stored
# it throws an exception
raise Exception(
"The obj: {} does not have the "
"attribute: {}, striped: {}. Maybe try a default "
"value.".format(
current_obj.name,
current_attribute, used_attribute))
# check if the value should be saved as an image or in the csv file
save_in_csv = False
try:
resulting_map[segmap ==
object_id] = used_value
was_used = True
if not is_default_value:
non_default_value_was_used = True
# save everything which is not instance also in the .csv
if current_attribute != "instance":
save_in_csv = True
except ValueError:
save_in_csv = True
if last_state_save_in_csv is not None and last_state_save_in_csv != save_in_csv:
raise Exception(
"During creating the mapping, the saving to an image or a csv file "
"switched, this might indicated that the used default value, does "
"not have the same type as the returned value, "
"for: {}".format(current_attribute))
last_state_save_in_csv = save_in_csv
if save_in_csv:
if object_id in save_in_csv_attributes:
save_in_csv_attributes[object_id][
used_attribute] = used_value
else:
save_in_csv_attributes[object_id] = {
used_attribute: used_value
}
if was_used and non_default_value_was_used:
used_channels.append(org_attribute)
combined_result_map.append(resulting_map)
fname = final_segmentation_file_path + ("%04d" %
frame) + suffix
# combine all resulting images to one image
resulting_map = np.stack(combined_result_map, axis=2)
# remove the unneeded third dimension
if resulting_map.shape[2] == 1:
resulting_map = resulting_map[:, :, 0]
np.save(fname, resulting_map)
if not there_was_an_instance_rendering:
if len(list_of_used_attributes) > 0:
raise Exception(
"There were attributes specified in the may_by, which could not be saved as "
"there was no \"instance\" may_by key used. This is true for this/these "
"keys: {}".format(", ".join(list_of_used_attributes)))
# if there was no instance rendering no .csv file is generated!
# delete all saved infos about .csv
save_in_csv_attributes = {}
# write color mappings to file
if save_in_csv_attributes:
csv_file_path = os.path.join(
output_dir, segcolormap_output_file_prefix + ".csv")
with open(csv_file_path, 'w', newline='') as csvfile:
# get from the first element the used field names
fieldnames = ["idx"]
# get all used object element keys
for object_element in save_in_csv_attributes.values():
fieldnames.extend(list(object_element.keys()))
break
for channel_name in used_channels:
fieldnames.append("channel_{}".format(channel_name))
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
# save for each object all values in one row
for obj_idx, object_element in save_in_csv_attributes.items(
):
object_element["idx"] = obj_idx
for i, channel_name in enumerate(used_channels):
object_element["channel_{}".format(
channel_name)] = i
writer.writerow(object_element)
Utility.register_output(output_dir, file_prefix, output_key, ".npy",
"2.0.0")
if save_in_csv_attributes:
Utility.register_output(output_dir,
segcolormap_output_file_prefix,
segcolormap_output_key,
".csv",
"2.0.0",
unique_for_camposes=False)
def run(self):
"""
Samples positions and rotations of selected object inside the sampling volume while performing mesh and
bounding box collision checks in the following steps:
1. While we have objects remaining and have not run out of tries - sample a point.
2. If no collisions are found keep the point.
"""
# While we have objects remaining and have not run out of tries - sample a point
# List of successfully placed objects
placed = []
# After this many tries we give up on current object and continue with the rest
max_tries = self.config.get_int("max_iterations", 1000)
objects = self.config.get_list("objects_to_sample", get_all_mesh_objects())
# cache to fasten collision detection
cache = {}
# for every selected object
for obj in objects:
if obj.type == "MESH":
print("Trying to put ", obj.name)
prior_location = obj.location
prior_rotation = obj.rotation_euler
no_collision = True
# Try max_iter amount of times
for i in range(max_tries):
# Put the top object in queue at the sampled point in space
position = self.config.get_vector3d("pos_sampler")
rotation = self.config.get_vector3d("rot_sampler")
# assign it a new position
obj.location = position
# and a rotation
obj.rotation_euler = rotation
# then update scene
bpy.context.view_layer.update()
no_collision = True
# Now check for collisions
for already_placed in placed:
# First check if bounding boxes collides
intersection = check_bb_intersection(obj, already_placed)
# if they do
if intersection:
# then check for more refined collisions
intersection, cache = check_intersection(obj, already_placed)
if intersection:
no_collision = False
break
# If no collision then keep the position, else: reset
if no_collision:
print("No collision detected, moving forward!")
placed.append(obj)
# then stop trying and keep assigned position and orientation
break
# if any collisions then reset object to initial state
else:
print("Collision detected, retrying!!")
obj.location = prior_location
obj.rotation_euler = prior_rotation
bpy.context.view_layer.update()
if not no_collision:
print("Giving up on ", obj.name)
def run(self):
with Utility.UndoAfterExecution():
self._configure_renderer(default_samples=1)
# Get objects with meshes (i.e. not lights or cameras)
objs_with_mats = get_all_mesh_objects()
colors, num_splits_per_dimension, used_objects = self._colorize_objects_for_instance_segmentation(
objs_with_mats)
bpy.context.scene.render.image_settings.file_format = "OPEN_EXR"
bpy.context.scene.render.image_settings.color_depth = "16"
bpy.context.view_layer.cycles.use_denoising = False
bpy.context.scene.cycles.filter_width = 0.0
if self._use_alpha_channel:
self.add_alpha_channel_to_textures(blurry_edges=False)
# Determine path for temporary and for final output
temporary_segmentation_file_path = os.path.join(self._temp_dir, "seg_")
final_segmentation_file_path = os.path.join(self._determine_output_dir(),
self.config.get_string("output_file_prefix", "segmap_"))
# Render the temporary output
self._render("seg_", custom_file_path=temporary_segmentation_file_path)
# Find optimal dtype of output based on max index
for dtype in [np.uint8, np.uint16, np.uint32]:
optimal_dtype = dtype
if np.iinfo(optimal_dtype).max >= len(colors) - 1:
break
# get the type of mappings which should be performed
used_attributes = self.config.get_raw_dict("map_by", "class")
used_default_values = self.config.get_raw_dict("default_values", {})
if 'class' in used_default_values:
used_default_values['cp_category_id'] = used_default_values['class']
if isinstance(used_attributes, str):
# only one result is requested
result_channels = 1
used_attributes = [used_attributes]
elif isinstance(used_attributes, list):
result_channels = len(used_attributes)
else:
raise Exception("The type of this is not supported here: {}".format(used_attributes))
save_in_csv_attributes = {}
# define them for the avoid rendering case
there_was_an_instance_rendering = False
list_of_used_attributes = []
# After rendering
if not self._avoid_rendering:
for frame in range(bpy.context.scene.frame_start, bpy.context.scene.frame_end): # for each rendered frame
file_path = temporary_segmentation_file_path + "%04d" % frame + ".exr"
segmentation = load_image(file_path)
segmap = Utility.map_back_from_equally_spaced_equidistant_values(segmentation,
num_splits_per_dimension,
self.render_colorspace_size_per_dimension)
segmap = segmap.astype(optimal_dtype)
used_object_ids = np.unique(segmap)
max_id = np.max(used_object_ids)
if max_id >= len(used_objects):
raise Exception("There are more object colors than there are objects")
combined_result_map = []
there_was_an_instance_rendering = False
list_of_used_attributes = []
used_channels = []
for channel_id in range(result_channels):
resulting_map = np.empty((segmap.shape[0], segmap.shape[1]))
was_used = False
current_attribute = used_attributes[channel_id]
org_attribute = current_attribute
# if the class is used the category_id attribute is evaluated
if current_attribute == "class":
current_attribute = "cp_category_id"
# in the instance case the resulting ids are directly used
if current_attribute == "instance":
there_was_an_instance_rendering = True
resulting_map = segmap
was_used = True
# a non default value was also used
non_default_value_was_used = True
else:
if current_attribute != "cp_category_id":
list_of_used_attributes.append(current_attribute)
# for the current attribute remove cp_ and _csv, if present
used_attribute = current_attribute
if used_attribute.startswith("cp_"):
used_attribute = used_attribute[len("cp_"):]
# check if a default value was specified
default_value_set = False
if current_attribute in used_default_values or used_attribute in used_default_values:
default_value_set = True
if current_attribute in used_default_values:
default_value = used_default_values[current_attribute]
elif used_attribute in used_default_values:
default_value = used_default_values[used_attribute]
last_state_save_in_csv = None
# this avoids that for certain attributes only the default value is written
non_default_value_was_used = False
# iterate over all object ids
for object_id in used_object_ids:
is_default_value = False
# get the corresponding object via the id
current_obj = used_objects[object_id]
# if the current obj has a attribute with that name -> get it
if hasattr(current_obj, used_attribute):
used_value = getattr(current_obj, used_attribute)
# if the current object has a custom property with that name -> get it
elif current_attribute.startswith("cp_") and used_attribute in current_obj:
used_value = current_obj[used_attribute]
elif current_attribute.startswith("cf_"):
if current_attribute == "cf_basename":
used_value = current_obj.name
if "." in used_value:
used_value = used_value[:used_value.rfind(".")]
elif default_value_set:
# if none of the above applies use the default value
used_value = default_value
is_default_value = True
else:
# if the requested current_attribute is not a custom property or a attribute
# or there is a default value stored
# it throws an exception
raise Exception("The obj: {} does not have the "
"attribute: {}, striped: {}. Maybe try a default "
"value.".format(current_obj.name, current_attribute, used_attribute))
# check if the value should be saved as an image or in the csv file
save_in_csv = False
try:
resulting_map[segmap == object_id] = used_value
was_used = True
if not is_default_value:
non_default_value_was_used = True
# save everything which is not instance also in the .csv
if current_attribute != "instance":
save_in_csv = True
except ValueError:
save_in_csv = True
if last_state_save_in_csv is not None and last_state_save_in_csv != save_in_csv:
raise Exception("During creating the mapping, the saving to an image or a csv file "
"switched, this might indicated that the used default value, does "
"not have the same type as the returned value, "
"for: {}".format(current_attribute))
last_state_save_in_csv = save_in_csv
if save_in_csv:
if object_id in save_in_csv_attributes:
save_in_csv_attributes[object_id][used_attribute] = used_value
else:
save_in_csv_attributes[object_id] = {used_attribute: used_value}
if was_used and non_default_value_was_used:
used_channels.append(org_attribute)
combined_result_map.append(resulting_map)
fname = final_segmentation_file_path + "%04d" % frame
# combine all resulting images to one image
resulting_map = np.stack(combined_result_map, axis=2)
# remove the unneeded third dimension
if resulting_map.shape[2] == 1:
resulting_map = resulting_map[:, :, 0]
np.save(fname, resulting_map)
if not there_was_an_instance_rendering:
if len(list_of_used_attributes) > 0:
raise Exception("There were attributes specified in the may_by, which could not be saved as "
"there was no \"instance\" may_by key used. This is true for this/these "
"keys: {}".format(", ".join(list_of_used_attributes)))
# if there was no instance rendering no .csv file is generated!
# delete all saved infos about .csv
save_in_csv_attributes = {}
# write color mappings to file
if save_in_csv_attributes and not self._avoid_rendering:
csv_file_path = os.path.join(self._determine_output_dir(),
self.config.get_string("segcolormap_output_file_prefix",
"class_inst_col_map") + ".csv")
with open(csv_file_path, 'w', newline='') as csvfile:
# get from the first element the used field names
fieldnames = ["idx"]
# get all used object element keys
for object_element in save_in_csv_attributes.values():
fieldnames.extend(list(object_element.keys()))
break
for channel_name in used_channels:
fieldnames.append("channel_{}".format(channel_name))
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
# save for each object all values in one row
for obj_idx, object_element in save_in_csv_attributes.items():
object_element["idx"] = obj_idx
for i, channel_name in enumerate(used_channels):
object_element["channel_{}".format(channel_name)] = i
writer.writerow(object_element)
self._register_output("segmap_", "segmap", ".npy", "2.0.0")
if save_in_csv_attributes:
self._register_output("class_inst_col_map",
"segcolormap",
".csv",
"2.0.0",
unique_for_camposes=False,
output_key_parameter_name="segcolormap_output_key",
output_file_prefix_parameter_name="segcolormap_output_file_prefix")
def run(self):
""" Writes coco annotations in the following steps:
1. Locat the seg images
2. Locat the rgb maps
3. Locat the seg maps
4. Read color mappings
5. For each frame write the coco annotation
"""
if self._avoid_rendering:
print("Avoid rendering is on, no output produced!")
return
# Find path pattern of segmentation images
segmentation_map_output = self._find_registered_output_by_key(
self.segmap_output_key)
if segmentation_map_output is None:
raise Exception(
"There is no output registered with key " +
self.segmap_output_key +
". Are you sure you ran the SegMapRenderer module before?")
# Find path pattern of rgb images
rgb_output = self._find_registered_output_by_key(self.rgb_output_key)
if rgb_output is None:
raise Exception(
"There is no output registered with key " +
self.rgb_output_key +
". Are you sure you ran the RgbRenderer module before?")
# collect all segmaps
segmentation_map_paths = []
# Find path of name class mapping csv file
segcolormap_output = self._find_registered_output_by_key(
self.segcolormap_output_key)
if segcolormap_output is None:
raise Exception(
"There is no output registered with key " +
self.segcolormap_output_key +
". Are you sure you ran the SegMapRenderer module with 'map_by' set to 'instance' before?"
)
# read colormappings, which include object name/class to integer mapping
color_map = []
with open(segcolormap_output["path"], 'r') as csvfile:
reader = csv.DictReader(csvfile)
for mapping in reader:
color_map.append(mapping)
coco_annotations_path = os.path.join(self._coco_data_dir,
"coco_annotations.json")
# Calculate image numbering offset, if append_to_existing_output is activated and coco data exists
if self.config.get_bool(
"append_to_existing_output",
False) and os.path.exists(coco_annotations_path):
with open(coco_annotations_path, 'r') as fp:
existing_coco_annotations = json.load(fp)
image_offset = max(
[image["id"]
for image in existing_coco_annotations["images"]]) + 1
else:
image_offset = 0
existing_coco_annotations = None
# collect all RGB paths
new_coco_image_paths = []
# for each rendered frame
for frame in range(bpy.context.scene.frame_start,
bpy.context.scene.frame_end):
segmentation_map_paths.append(segmentation_map_output["path"] %
frame)
source_path = rgb_output["path"] % frame
target_path = os.path.join(
self._coco_data_dir,
os.path.basename(rgb_output["path"] % (frame + image_offset)))
shutil.copyfile(source_path, target_path)
new_coco_image_paths.append(os.path.basename(target_path))
# Try to extract supercategory for each object
all_mesh_objects = get_all_mesh_objects()
super_category_mapping = {}
for obj in all_mesh_objects:
# For now the only scheme to extract super category is the afiliation of an object to a Bop dataset
if "bop_dataset_name" in obj:
super_category_mapping[obj.name] = obj["bop_dataset_name"]
# Otherwise assign default supercategory
else:
super_category_mapping[obj.name] = "default_supercategory"
coco_output = CocoUtility.generate_coco_annotations(
segmentation_map_paths, new_coco_image_paths, color_map,
super_category_mapping, "coco_annotations",
existing_coco_annotations)
print("Writing coco annotations to " + coco_annotations_path)
with open(coco_annotations_path, 'w') as fp:
json.dump(coco_output, fp)
def _remove_rigidbody(self):
""" Removes the rigidbody element from all mesh objects. """
for obj in get_all_mesh_objects():
bpy.context.view_layer.objects.active = obj
bpy.ops.rigidbody.object_remove()
def run(self):
"""
Samples positions and rotations of selected object inside the sampling volume while performing mesh and
bounding box collision checks in the following steps:
1. While we have objects remaining and have not run out of tries - sample a point.
2. If no collisions are found keep the point.
"""
# While we have objects remaining and have not run out of tries - sample a point
# List of successfully placed objects
placed = []
# After this many tries we give up on current object and continue with the rest
max_tries = self.config.get_int("max_iterations", 1000)
objects = self.config.get_list("objects_to_sample",
get_all_mesh_objects())
# cache to fasten collision detection
bvh_cache = {}
# for every selected object
for obj in objects:
if obj.type == "MESH":
no_collision = True
# Try max_iter amount of times
for i in range(max_tries):
# Put the top object in queue at the sampled point in space
position = self.config.get_vector3d("pos_sampler")
rotation = self.config.get_vector3d("rot_sampler")
# assign it a new pose
obj.location = position
obj.rotation_euler = rotation
bpy.context.view_layer.update()
# Remove bvh cache, as object has changed
if obj.name in bvh_cache:
del bvh_cache[obj.name]
no_collision = True
# Now check for collisions
for already_placed in placed:
# First check if bounding boxes collides
intersection = check_bb_intersection(
obj, already_placed)
# if they do
if intersection:
# then check for more refined collisions
intersection, bvh_cache = check_intersection(
obj, already_placed, bvh_cache=bvh_cache)
if intersection:
no_collision = False
break
# If no collision then keep the position
if no_collision:
break
placed.append(obj)
if not no_collision:
print("Could not place " + obj.name +
" without a collision.")
else:
print("It took " + str(i + 1) + " tries to place " +
obj.name)
