def __init__(self, config):
LoaderInterface.__init__(self, config)
self._file_path = Utility.resolve_path(
self.config.get_string("file_path"))
self._texture_folder = Utility.resolve_path(
self.config.get_string("texture_folder"))
# the default unknown texture folder is not included inside of the scenenet texture folder
default_unknown_texture_folder = os.path.join(self._texture_folder,
"unknown")
# the textures in this folder are used, if the object has no available texture
self._unknown_texture_folder = Utility.resolve_path(
self.config.get_string("unknown_texture_folder",
default_unknown_texture_folder))
LabelIdMapping.assign_mapping(
Utility.resolve_path(
os.path.join('resources', 'id_mappings', 'nyu_idset.csv')))
if LabelIdMapping.label_id_map:
bpy.context.scene.world[
"category_id"] = LabelIdMapping.label_id_map["void"]
else:
print(
"Warning: The category labeling file could not be found -> no semantic segmentation available!"
)
def _load_room(node: dict, metadata: dict, material_adjustments: list,
transform: Matrix, house_id: str, parent: MeshObject,
room_per_object: dict,
label_mapping: LabelIdMapping) -> List[MeshObject]:
""" Load the room specified in the given node.
:param node: The node dict which contains information from house.json..
:param metadata: A dict of metadata which will be written into the object's custom data.
:param material_adjustments: Adjustments to the materials which were specified inside house.json.
:param transform: The transformation that should be applied to the loaded objects.
:param house_id: The id of the current house.
:param parent: The parent object to which the room should be linked
:param room_per_object: A dict for object -> room lookup (Will be written into)
:return: The list of loaded mesh objects.
"""
# Build empty room object which acts as a parent for all objects inside
room_obj = Entity.create_empty("Room#" + node["id"])
room_obj.set_cp("type", "Room")
room_obj.set_cp("bbox", SuncgLoader._correct_bbox_frame(node["bbox"]))
room_obj.set_cp("roomTypes", node["roomTypes"])
room_obj.set_parent(parent)
loaded_objects = [room_obj]
# Store indices of all contained objects in
if "nodeIndices" in node:
for child_id in node["nodeIndices"]:
room_per_object[child_id] = room_obj
if "hideFloor" not in node or node["hideFloor"] != 1:
metadata["type"] = "Floor"
metadata["category_id"] = label_mapping.id_from_label("floor")
metadata["fine_grained_class"] = "floor"
loaded_objects += SuncgLoader._load_obj(
os.path.join(SuncgLoader._suncg_dir, "room", house_id,
node["modelId"] + "f.obj"), metadata,
material_adjustments, transform, room_obj)
if "hideCeiling" not in node or node["hideCeiling"] != 1:
metadata["type"] = "Ceiling"
metadata["category_id"] = label_mapping.id_from_label("ceiling")
metadata["fine_grained_class"] = "ceiling"
loaded_objects += SuncgLoader._load_obj(
os.path.join(SuncgLoader._suncg_dir, "room", house_id,
node["modelId"] + "c.obj"), metadata,
material_adjustments, transform, room_obj)
if "hideWalls" not in node or node["hideWalls"] != 1:
metadata["type"] = "Wall"
metadata["category_id"] = label_mapping.id_from_label("wall")
metadata["fine_grained_class"] = "wall"
loaded_objects += SuncgLoader._load_obj(
os.path.join(SuncgLoader._suncg_dir, "room", house_id,
node["modelId"] + "w.obj"), metadata,
material_adjustments, transform, room_obj)
return loaded_objects
def __init__(self, config):
LoaderInterface.__init__(self, config)
self.house_path = Utility.resolve_path(self.config.get_string("path"))
suncg_folder_path = os.path.join(os.path.dirname(self.house_path), "../..")
self.suncg_dir = self.config.get_string("suncg_path", suncg_folder_path)
self._collection_of_loaded_objs = {}
# there are only two types of materials, textures and diffuse
self._collection_of_loaded_mats = {"texture": {}, "diffuse": {}}
LabelIdMapping.assign_mapping(Utility.resolve_path(os.path.join('resources', 'id_mappings',
'nyu_idset.csv')))
def __init__(self, config):
LoaderInterface.__init__(self, config)
self.house_path = Utility.resolve_path(self.config.get_string("path"))
suncg_folder_path = os.path.join(os.path.dirname(self.house_path),
"../..")
self.suncg_dir = self.config.get_string("suncg_path",
suncg_folder_path)
LabelIdMapping.assign_mapping(
Utility.resolve_path(
os.path.join('resources', 'id_mappings', 'nyu_idset.csv')))
def _set_category_ids(loaded_objects: List[MeshObject],
label_mapping: LabelIdMapping):
"""
Set the category ids for the objs based on the .csv file loaded in LabelIdMapping
Each object will have a custom property with a label, can be used by the SegMapRenderer.
:param loaded_objects: objects loaded from the .obj file
"""
# Some category names in scenenet objects are written differently than in nyu_idset.csv
normalize_name = {
"floor-mat": "floor_mat",
"refrigerator": "refridgerator",
"shower-curtain": "shower_curtain",
"nightstand": "night_stand",
"Other-structure": "otherstructure",
"Other-furniture": "otherfurniture",
"Other-prop": "otherprop",
"floor_tiles_floor_tiles_0125": "floor",
"ground": "floor",
"floor_enclose": "floor",
"floor_enclose2": "floor",
"floor_base_object01_56": "floor",
"walls1_line01_12": "wall",
"room_skeleton": "wall",
"ceilingwall": "ceiling"
}
for obj in loaded_objects:
obj_name = obj.get_name().lower().split(".")[0]
# If it's one of the cases that the category have different names in both idsets.
if obj_name in normalize_name:
obj_name = normalize_name[obj_name] # Then normalize it.
if label_mapping.has_label(obj_name):
obj.set_cp("category_id",
label_mapping.id_from_label(obj_name))
# Check whether the object's name without suffixes like 's', '1' or '2' exist in the mapping.
elif label_mapping.has_label(obj_name[:-1]):
obj.set_cp("category_id",
label_mapping.id_from_label(obj_name[:-1]))
elif "painting" in obj_name:
obj.set_cp("category_id",
label_mapping.id_from_label("picture"))
else:
print("This object was not specified: {} use objects for it.".
format(obj_name))
obj.set_cp(
"category_id",
label_mapping.id_from_label("otherstructure".lower()))
# Correct names of floor and ceiling objects to make them later easier to identify (e.g. by the FloorExtractor)
if obj.get_cp("category_id") == label_mapping.id_from_label(
"floor"):
obj.set_name("floor")
elif obj.get_cp("category_id") == label_mapping.id_from_label(
"ceiling"):
obj.set_name("ceiling")
def switch_mapping(segmap, source_map, destination_map): # This assumes label names in different mappings are the same. # This function is mainly useful to map from the old class mapping to the now default NYU mapping. source_id_label_map, source_label_id_map = LabelIdMapping.read_csv_mapping(source_map) destination_id_label_map, destination_label_id_map = LabelIdMapping.read_csv_mapping(destination_map) new_segmap = np.zeros_like(segmap) unq = np.unique(segmap) for id in unq: label_name = source_id_label_map[id] if label_name in destination_label_id_map: destination_id = destination_label_id_map[source_id_label_map[id]] new_segmap[segmap == id] = destination_id return new_segmap
def switch_mapping(segmap, source_map, destination_map):
# This assumes label names in different mappings are the same.
# This function is mainly useful to map from the old class mapping to the now default NYU mapping.
source_label_map = LabelIdMapping.from_csv(source_map)
destination_label_map = LabelIdMapping.from_csv(destination_map)
new_segmap = np.zeros_like(segmap)
unq = np.unique(segmap)
for id in unq:
label_name = source_label_map.label_from_id(id)
if destination_label_map.has_label(label_name):
destination_id = destination_label_map.id_from_label(
source_label_map.label_from_id(id))
new_segmap[segmap == id] = destination_id
return new_segmap
def run(self):
label_mapping = LabelIdMapping.from_csv(
Utility.resolve_path(
os.path.join('resources', 'id_mappings', 'nyu_idset.csv')))
# Add label mapping to global storage, s.t. it could be used for naming semantic segmentations.
GlobalStorage.set("label_mapping", label_mapping)
loaded_objects = SuncgLoader.load(self.house_path, label_mapping,
self.suncg_dir)
self._set_properties(loaded_objects)
def __init__(self, config: Config):
LoaderInterface.__init__(self, config)
self.mapping_file = Utility.resolve_path(
self.config.get_string(
"mapping_file",
os.path.join("resources", "front_3D", "3D_front_mapping.csv")))
if not os.path.exists(self.mapping_file):
raise Exception("The mapping file could not be found: {}".format(
self.mapping_file))
_, self.mapping = LabelIdMapping.read_csv_mapping(self.mapping_file)
def __init__(self, config):
Loader.__init__(self, config)
self._file_path = Utility.resolve_path(
self.config.get_string("file_path"))
self._texture_folder = Utility.resolve_path(
self.config.get_string("texture_folder"))
LabelIdMapping.assign_mapping(
Utility.resolve_path(
os.path.join('resources', 'id_mappings', 'nyu_idset.csv')))
if LabelIdMapping.label_id_map:
bpy.data.scenes["Scene"]["num_labels"] = LabelIdMapping.num_labels
bpy.context.scene.world[
"category_id"] = LabelIdMapping.label_id_map["void"]
else:
print(
"Warning: The category labeling file could not be found -> no semantic segmentation available!"
)
def run(self):
label_mapping = LabelIdMapping.from_csv(self.mapping_file)
# Add label mapping to global storage, s.t. it could be used for naming semantic segmentations.
GlobalStorage.set("label_mapping", label_mapping)
loaded_objects = Front3DLoader.load(
json_path=self.config.get_string("json_path"),
future_model_path=self.config.get_string("3D_future_model_path"),
front_3D_texture_path=self.config.get_string(
"3D_front_texture_path"),
label_mapping=label_mapping,
ceiling_light_strength=self.config.get_float(
"ceiling_light_strength", 0.8),
lamp_light_strength=self.config.get_float("lamp_light_strength",
7.0))
self._set_properties(loaded_objects)
def __init__(self, config: Config):
LoaderInterface.__init__(self, config)
self.json_path = Utility.resolve_path(
self.config.get_string("json_path"))
self.future_model_path = Utility.resolve_path(
self.config.get_string("3D_future_model_path"))
self.mapping_file = Utility.resolve_path(
self.config.get_string(
"mapping_file",
os.path.join("resources", "front_3D", "3D_front_mapping.csv")))
if not os.path.exists(self.mapping_file):
raise Exception("The mapping file could not be found: {}".format(
self.mapping_file))
_, self.mapping = LabelIdMapping.read_csv_mapping(self.mapping_file)
# a list of all newly created objects
self.created_objects = []
def run(self):
"""
Run the module, loads all the objects and set the properties correctly (including the category_id)
"""
label_mapping = LabelIdMapping.from_csv(
Utility.resolve_path(
os.path.join('resources', 'id_mappings', 'nyu_idset.csv')))
# Add label mapping to global storage, s.t. it could be used for naming semantic segmentations.
GlobalStorage.set("label_mapping", label_mapping)
# load the objects (Use use_image_search=False as some image names have a "/" prefix which will lead to blender search the whole root directory recursively!
loaded_objects = SceneNetLoader.load(
file_path=self._file_path,
texture_folder=self._texture_folder,
label_mapping=label_mapping,
unknown_texture_folder=self._unknown_texture_folder)
# add custom properties
self._set_properties(loaded_objects)
def _load_ground(node: dict, metadata: dict, material_adjustments: list,
transform: Matrix, house_id: str, parent: MeshObject,
label_mapping: LabelIdMapping) -> List[MeshObject]:
""" Load the ground specified in the given node.
:param node: The node dict which contains information from house.json..
:param metadata: A dict of metadata which will be written into the object's custom data.
:param material_adjustments: Adjustments to the materials which were specified inside house.json.
:param transform: The transformation that should be applied to the loaded objects.
:param house_id: The id of the current house.
:param parent: The parent object to which the ground should be linked
:return: The list of loaded mesh objects.
"""
metadata["type"] = "Ground"
metadata["category_id"] = label_mapping.id_from_label("floor")
metadata["fine_grained_class"] = "ground"
return SuncgLoader._load_obj(
os.path.join(SuncgLoader._suncg_dir, "room", house_id,
node["modelId"] + "f.obj"), metadata,
material_adjustments, transform, parent)
def _load_box(node: dict, material_adjustments: list, transform: Matrix,
parent: MeshObject,
label_mapping: LabelIdMapping) -> List[MeshObject]:
""" Creates a cube inside blender which follows the specifications of the given node.
:param node: The node dict which contains information from house.json..
:param material_adjustments: Adjustments to the materials which were specified inside house.json.
:param transform: The transformation that should be applied to the loaded objects.
:param parent: The parent object to which the ground should be linked
:return: The list of loaded mesh objects.
"""
box = MeshObject.create_primitive("CUBE")
box.set_name("Box#" + node["id"])
# Scale the cube to the required dimensions
box.set_local2world_mat(
Matrix.Scale(node["dimensions"][0] / 2, 4, (1.0, 0.0, 0.0))
@ Matrix.Scale(node["dimensions"][1] / 2, 4,
(0.0, 1.0, 0.0)) @ Matrix.Scale(
node["dimensions"][2] / 2, 4, (0.0, 0.0, 1.0)))
# Create UV mapping (beforehand we apply the scaling from the previous step, such that the resulting uv mapping has the correct aspect)
bpy.ops.object.transform_apply(scale=True)
bpy.ops.object.editmode_toggle()
bpy.ops.uv.cube_project()
bpy.ops.object.editmode_toggle()
# Create an empty material which is filled in the next step
box.new_material("material_0")
SuncgLoader._transform_and_colorize_object(box, material_adjustments,
transform, parent)
# set class to void
box.set_cp("category_id", label_mapping.id_from_label("void"))
# Rotate cube to match objects loaded from .obj, has to be done after transformations have been applied
box.set_local2world_mat(
Matrix.Rotation(math.radians(90), 4, "X") @ Matrix(
box.get_local2world_mat()))
return [box]
def generate_coco_annotations(inst_segmaps,
inst_attribute_maps,
image_paths,
supercategory,
mask_encoding_format,
existing_coco_annotations=None,
label_mapping: LabelIdMapping = None):
"""Generates coco annotations for images
:param inst_segmaps: List of instance segmentation maps
:param inst_attribute_maps: per-frame mappings with idx, class and optionally supercategory/bop_dataset_name
:param image_paths: A list of paths which points to the rendered segmentation maps.
:param supercategory: name of the dataset/supercategory to filter for, e.g. a specific BOP dataset
:param mask_encoding_format: Encoding format of the binary mask. Type: string.
:param existing_coco_annotations: If given, the new coco annotations will be appended to the given coco annotations dict.
:param label_mapping: The label mapping which should be used to label the categories based on their ids.
If None, is given then the `name` field in the csv files is used or - if not existing - the category id itself is used.
:return: dict containing coco annotations
"""
categories = []
visited_categories = []
instance_2_category_maps = []
for inst_attribute_map in inst_attribute_maps:
instance_2_category_map = {}
for inst in inst_attribute_map:
# skip background
if int(inst["category_id"]) != 0:
# take all objects or objects from specified supercategory is defined
inst_supercategory = "coco_annotations"
if "bop_dataset_name" in inst:
inst_supercategory = inst["bop_dataset_name"]
elif "supercategory" in inst:
inst_supercategory = inst["supercategory"]
if supercategory == inst_supercategory or supercategory == 'coco_annotations':
if int(inst["category_id"]) not in visited_categories:
cat_dict = {}
cat_dict['id'] = int(inst["category_id"])
cat_dict['supercategory'] = inst_supercategory
# Determine name of category based on label_mapping, name or category_id
if label_mapping is not None:
cat_dict["name"] = label_mapping.label_from_id(
cat_dict['id'])
elif "name" in inst:
cat_dict["name"] = inst["name"]
else:
cat_dict["name"] = inst["category_id"]
categories.append(cat_dict)
visited_categories.append(cat_dict['id'])
instance_2_category_map[int(inst["idx"])] = int(
inst["category_id"])
instance_2_category_maps.append(instance_2_category_map)
licenses = [{
"id": 1,
"name": "Attribution-NonCommercial-ShareAlike License",
"url": "http://creativecommons.org/licenses/by-nc-sa/2.0/"
}]
info = {
"description": supercategory,
"url": "https://github.com/waspinator/pycococreator",
"version": "0.1.0",
"year": 2020,
"contributor": "Unknown",
"date_created": datetime.datetime.utcnow().isoformat(' ')
}
images = []
annotations = []
for inst_segmap, image_path, instance_2_category_map in zip(
inst_segmaps, image_paths, instance_2_category_maps):
# Add coco info for image
image_id = len(images)
images.append(
CocoWriterUtility.create_image_info(image_id, image_path,
inst_segmap.shape))
# Go through all objects visible in this image
instances = np.unique(inst_segmap)
# Remove background
instances = np.delete(instances, np.where(instances == 0))
for inst in instances:
if inst in instance_2_category_map:
# Calc object mask
binary_inst_mask = np.where(inst_segmap == inst, 1, 0)
# Add coco info for object in this image
annotation = CocoWriterUtility.create_annotation_info(
len(annotations), image_id,
instance_2_category_map[inst], binary_inst_mask,
mask_encoding_format)
if annotation is not None:
annotations.append(annotation)
new_coco_annotations = {
"info": info,
"licenses": licenses,
"categories": categories,
"images": images,
"annotations": annotations
}
if existing_coco_annotations is not None:
new_coco_annotations = CocoWriterUtility.merge_coco_annotations(
existing_coco_annotations, new_coco_annotations)
return new_coco_annotations
def load(house_path: str,
label_mapping: LabelIdMapping,
suncg_dir: str = None) -> List[MeshObject]:
""" Loads a house.json file into blender.
- Loads all objects files specified in the house.json file.
- Orders them hierarchically (level -> room -> object)
- Writes metadata into the custom properties of each object
:param house_path: The path to the house.json file which should be loaded.
:param suncg_dir: The path to the suncg root directory which should be used for loading objects, rooms, textures etc.
:return: The list of loaded mesh objects.
"""
# If not suncg root directory has been given, determine it via the given house directory.
if suncg_dir is None:
suncg_dir = os.path.join(os.path.dirname(house_path), "../..")
SuncgLoader._suncg_dir = suncg_dir
SuncgLoader._collection_of_loaded_objs = {}
# there are only two types of materials, textures and diffuse
SuncgLoader._collection_of_loaded_mats = {"texture": {}, "diffuse": {}}
with open(Utility.resolve_path(house_path), "r") as f:
config = json.load(f)
object_label_map, object_fine_grained_label_map, object_coarse_grained_label_map = SuncgLoader._read_model_category_mapping(
os.path.join('resources', 'suncg', 'Better_labeling_for_NYU.csv'))
house_id = config["id"]
loaded_objects = []
for level in config["levels"]:
# Build empty level object which acts as a parent for all rooms on the level
level_obj = Entity.create_empty("Level#" + level["id"])
level_obj.set_cp("type", "Level")
if "bbox" in level:
level_obj.set_cp(
"bbox", SuncgLoader._correct_bbox_frame(level["bbox"]))
else:
print(
"Warning: The level with id " + level["id"] +
" is missing the bounding box attribute in the given house.json file!"
)
loaded_objects.append(level_obj)
room_per_object = {}
for node in level["nodes"]:
# Skip invalid nodes (This is the same behavior as in the SUNCG Toolbox)
if "valid" in node and node["valid"] == 0:
continue
# Metadata is directly stored in the objects custom data
metadata = {"type": node["type"], "is_suncg": True}
if "modelId" in node:
metadata["modelId"] = node["modelId"]
if node["modelId"] in object_fine_grained_label_map:
metadata[
"fine_grained_class"] = object_fine_grained_label_map[
node["modelId"]]
metadata[
"coarse_grained_class"] = object_coarse_grained_label_map[
node["modelId"]]
metadata["category_id"] = label_mapping.id_from_label(
object_label_map[node["modelId"]])
if "bbox" in node:
metadata["bbox"] = SuncgLoader._correct_bbox_frame(
node["bbox"])
if "transform" in node:
transform = Matrix([
node["transform"][i * 4:(i + 1) * 4] for i in range(4)
])
# Transpose, as given transform matrix was col-wise, but blender expects row-wise
transform.transpose()
else:
transform = None
if "materials" in node:
material_adjustments = node["materials"]
else:
material_adjustments = []
# Lookup if the object belongs to a room
object_id = int(node["id"].split("_")[-1])
if object_id in room_per_object:
parent = room_per_object[object_id]
else:
parent = level_obj
if node["type"] == "Room":
loaded_objects += SuncgLoader._load_room(
node, metadata, material_adjustments, transform,
house_id, level_obj, room_per_object, label_mapping)
elif node["type"] == "Ground":
loaded_objects += SuncgLoader._load_ground(
node, metadata, material_adjustments, transform,
house_id, parent, label_mapping)
elif node["type"] == "Object":
loaded_objects += SuncgLoader._load_object(
node, metadata, material_adjustments, transform,
parent)
elif node["type"] == "Box":
loaded_objects += SuncgLoader._load_box(
node, material_adjustments, transform, parent,
label_mapping)
SuncgLoader._rename_materials()
return loaded_objects
import os
parser = argparse.ArgumentParser()
parser.add_argument(
'house', help="Path to the house.json file of the SUNCG scene to load")
parser.add_argument('output_dir',
nargs='?',
default="examples/datasets/suncg_with_cam_sampling/output",
help="Path to where the final files, will be saved")
args = parser.parse_args()
Initializer.init()
# load the objects into the scene
label_mapping = LabelIdMapping.from_csv(
Utility.resolve_path(
os.path.join('resources', 'id_mappings', 'nyu_idset.csv')))
objs = SuncgLoader.load(args.house, label_mapping)
# makes Suncg objects emit light
SuncgLighting.light()
# Init sampler for sampling locations inside the loaded suncg house
point_sampler = SuncgPointInRoomSampler(objs)
# Init bvh tree containing all mesh objects
bvh_tree = MeshObject.create_bvh_tree_multi_objects(
[o for o in objs if isinstance(o, MeshObject)])
poses = 0
tries = 0
while tries < 10000 and poses < 5:
parser.add_argument("front", help="Path to the 3D front file")
parser.add_argument("future_folder",
help="Path to the 3D Future Model folder.")
parser.add_argument("front_3D_texture_path",
help="Path to the 3D FRONT texture folder.")
parser.add_argument("output_dir",
help="Path to where the data should be saved")
args = parser.parse_args()
if not os.path.exists(args.front) or not os.path.exists(args.future_folder):
raise Exception("One of the two folders does not exist!")
Initializer.init()
mapping_file = Utility.resolve_path(
os.path.join("resources", "front_3D", "3D_front_mapping.csv"))
mapping = LabelIdMapping.from_csv(mapping_file)
# set the light bounces
RendererUtility.set_light_bounces(diffuse_bounces=200,
glossy_bounces=200,
ao_bounces_render=200,
max_bounces=200,
transmission_bounces=200,
transparent_max_bounces=200,
volume_bounces=0)
# load the front 3D objects
loaded_objects = Front3DLoader.load(
json_path=args.front,
future_model_path=args.future_folder,
front_3D_texture_path=args.front_3D_texture_path,