def run(self): self.start() # Create an empty room. self.communicate(TDWUtils.create_empty_room(12, 12)) # Create the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 0, "y": 3, "z": -4 }, look_at=TDWUtils.VECTOR3_ZERO)) # Set the pass mask to _depth only. # Get an image. resp = self.communicate([ self.get_add_object("rh10", object_id=0), { "$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_depth"] }, { "$type": "send_images", "frequency": "once" } ]) images = Images(resp[0]) # Get the depth values of each pixel. depth = TDWUtils.get_depth_values(images.get_image(0)) print(depth)
def save_images(images: Images, filename: str, output_directory="dist", resize_to=None, append_pass: bool = True) -> None: """ Save each image in the Images object. The name of the image will be: pass_filename.extension, e.g.: `"0000"` -> `depth_0000.png` The images object includes the pass and extension information. :param images: The Images object. Contains each capture pass plus metadata. :param output_directory: The directory to write images to. :param filename: The filename of each image, minus the extension. The image pass will be appended as a prefix. :param resize_to: Specify a (width, height) tuple to resize the images to. This is slower than saving as-is. :param append_pass: If false, the image pass will _not_ be appended to the filename as a prefix, e.g.: `"0000"`: -> "`0000.jpg"` """ if not os.path.isdir(output_directory): os.makedirs(output_directory) for i in range(images.get_num_passes()): if append_pass: fi = images.get_pass_mask( i)[1:] + "_" + filename + "." + images.get_extension(i) else: fi = filename + "." + images.get_extension(i) if resize_to: TDWUtils.get_pil_image(images, i).resize((resize_to[0], resize_to[1]), Image.LANCZOS)\ .save(os.path.join(output_directory, fi)) else: with open(os.path.join(output_directory, fi), "wb") as f: f.write(images.get_image(i))
def get_shaped_depth_pass(images: Images, index: int) -> np.array: """ The `_depth` and `_depth_simple` passes are a 1D array of RGB values, as oppposed to a png or jpg like every other pass. This function reshapes the array into a 2D array of RGB values. :param images: The `Images` output data. :param index: The index in `Images` of the depth pass. See: `Images.get_pass_mask()`. :return: A reshaped depth pass. Shape is: `(height, width, 3)`. """ return np.flip( np.reshape(images.get_image(index), (images.get_height(), images.get_width(), 3)), 0)
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(10, 10)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) self.communicate(self.get_add_hdri_skybox("autumn_hockey_4k")) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": 3, "y": 1.3, "z": 0}, look_at=TDWUtils.array_to_vector3([0, 0.5, 0]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("grass_countryside") print(record) self.communicate({"$type": "add_material", "name": "grass_countryside", "url": record.get_url()}) self.communicate({"$type": "set_proc_gen_floor_material", "name": "grass_countryside"}) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) self.add_object(model_name="b05_clochild4", position={"x": 0, "y": 0, "z": 0}, rotation={"x": 0, "y": 30, "z": 0}, library="models_full.json") self.add_object(model_name="baseballbat", position={"x": 0.5, "y": 0, "z": 0.3}, rotation={"x": 0, "y": 30, "z": 0}, library="models_full.json") self.add_object(model_name="base-ball", position={"x": 0.4, "y": 0, "z": 0.22}, rotation={"x": 0, "y": 30, "z": 10}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([0, 0.5, 0])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "kid_baseball", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def get_image(self, record: ModelRecord): o_id = Controller.get_unique_id() self.communicate({"$type": "add_object", "name": record.name, "url": record.get_url(), "scale_factor": record.scale_factor, "rotation": record.canonical_rotation, "id": o_id}) s = TDWUtils.get_unit_scale(record) * 2 # Scale the model and get an image. # Look at the model's centroid. resp = self.communicate([{"$type": "scale_object", "id": o_id, "scale_factor": {"x": s, "y": s, "z": s}}, {"$type": "look_at", "avatar_id": "a", "object_id": o_id, "use_centroid": True}]) # Destroy the model and unload the asset bundle. self.communicate([{"$type": "destroy_object", "id": o_id}, {"$type": "unload_asset_bundles"}]) return Images(resp[0]), resp[-1]
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(32, 32)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": -1.5, "y": 1.5, "z": -15}, look_at=TDWUtils.array_to_vector3([-1.7, 0.7, -2.3]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("concrete_chipped_cracked") self.communicate({"$type": "add_material", "name": "concrete_chipped_cracked", "url": record.get_url()}) self.communicate({"$type": "set_proc_gen_floor_material", "name": "concrete_chipped_cracked"}) # self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"}) self.communicate({"$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar"}) bus = self.add_object(model_name="b06_bus_new", position={"x": -2, "y": 0, "z": 0}, rotation={"x": 0, "y": -70, "z": 0}, library="models_full.json") bus_bounds = self.get_bounds_data(bus) bus_front = bus_bounds.get_front(0) print(bus_front) self.add_object(model_name="3dscan_man_004", position={"x": -3.7, "y": 0, "z": -7.3}, rotation={"x": 0, "y": 0, "z": 0}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([-1.7, 0.7, -2.3])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "bus", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def run(self): # Create the output directory. output_directory = "example_output" if os.path.exists(output_directory): shutil.rmtree(output_directory) sleep(0.5) os.mkdir(output_directory) print(f"Images will be saved to: {output_directory}") self.start() # Create an empty room. self.communicate(TDWUtils.create_empty_room(12, 12)) # Add the objects. lamp_id = self.add_object("alma_floor_lamp", position={"x": 1, "y": 0, "z": 0}, rotation={"x": 0, "y": 90, "z": 0}) self.add_object("live_edge_coffee_table", position={"x": 1.5, "y": 0, "z": 1.5}, rotation={"x": 0, "y": 30, "z": 0}) self.add_object("small_table_green_marble", position={"x": -0.9, "y": 0, "z": -1.35}) # Create the avatar. self.communicate({"$type": "create_avatar", "type": "A_Img_Caps_Kinematic", "id": "a"}) # Try to find a valid position on the NavMesh. x, y, z = TDWUtils.get_random_position_on_nav_mesh(self, 12, 12) # Teleport the avatar to the valid position. # Apply a force to the lamp. # Set the pass masks to _img. # Enable image capture. self.communicate([{"$type": "teleport_avatar_to", "avatar_id": "a", "position": {"x": x, "y": 1.5, "z": z}}, {"$type": "apply_force_to_object", "force": {"x": 2, "y": 1, "z": 0}, "id": lamp_id}, {"$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_img", "_id"]}, {"$type": "send_images", "frequency": "always"}, ]) # Capture 100 images. for i in range(100): # Look at the lamp. resp = self.communicate({"$type": "look_at", "avatar_id": "a", "object_id": lamp_id, "use_centroid": True}) images = Images(resp[0]) # Save the image. TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=output_directory)
def run(self): self.start() commands = [TDWUtils.create_empty_room(self.size, self.size)] commands.extend( TDWUtils.create_avatar(position={ "x": 0, "y": 1.5, "z": 0 }, avatar_id="a")) commands.extend([{ "$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_img"] }, { "$type": "send_images", "frequency": "always" }]) self.communicate(commands) # Listen for keys. # Turn left. self.listen(key="left", commands={ "$type": "rotate_sensor_container_by", "axis": "yaw", "angle": self.angle * -1, "sensor_name": "SensorContainer", "avatar_id": "a" }) # Turn right. self.listen(key="right", commands={ "$type": "rotate_sensor_container_by", "axis": "yaw", "angle": self.angle, "sensor_name": "SensorContainer", "avatar_id": "a" }) # Quit the application. self.listen(key="esc", function=self.stop) # Continue until the quit key is pressed. i = 0 while not self.done: # Listen for keyboard input. Receive output data. resp = self.communicate([]) for r in resp[:-1]: r_id = OutputData.get_data_type_id(r) # Save images. if r_id == "imag": TDWUtils.save_images( images=Images(r), output_directory=self.images_directory, filename=TDWUtils.zero_padding(i, width=4)) # Increment the image number. i += 1 self.communicate({"$type": "terminate"})
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 1280, "height": 962}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.load_streamed_scene(scene="tdw_room_2018") # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 1, "z": 0}, look_at=TDWUtils.array_to_vector3([2, 0.8, -1.5]), avatar_id="avatar")) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) table_id = self.add_object(model_name="glass_table", position={"x": 2, "y": 0, "z": -1.5}, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="flower_vase_set_2", position={"x": 1.6, "y": top[1], "z": -1.5}, library="models_full.json") self.add_object(model_name="flower_vase_set_2", position={"x": 2, "y": top[1], "z": -1.5}, library="models_full.json") self.add_object(model_name="flower_vase_set_2", position={"x": 2.4, "y": top[1], "z": -1.5}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([2, 0.8, -1.5])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "flowers_row", output_directory="replicated_images")
def save_image(self, resp, record: ModelRecord, image_count: int, root_dir: str, wnid: str, train: int, train_count: int) -> None: """ Save an image. :param resp: The raw response data. :param record: The model record. :param image_count: The image count. :param root_dir: The root directory. :param wnid: The wnid. :param train: Number of train images so far. :param train_count: Total number of train images to generate. """ # Get the directory. directory = Path(root_dir).joinpath("train" if train < train_count else "val").joinpath(wnid).resolve() if not os.path.exists(directory): # Try to make the directories. Due to threading, they might already be made. try: os.makedirs(directory) except: pass # Save the image. filename = f"{record.name}_{image_count:04d}" # Save the image without resizing. if self.screen_size == self.output_size: TDWUtils.save_images(Images(resp[0]), filename, output_directory=directory) # Resize the image and save it. else: TDWUtils.save_images(Images(resp[0]), filename, output_directory=directory, resize_to=(self.output_size, self.output_size))
def get_image(self, record: MaterialRecord): self.communicate([self.get_add_material(record.name), {"$type": "set_primitive_visual_material", "id": self.sphere_id, "name": record.name, "quality": "high"}, {"$type": "set_primitive_visual_material", "id": self.cube_id, "name": record.name, "quality": "high"}]) # Capture the image the following frame to allow it to initialize correctly. resp = self.communicate({"$type": "do_nothing"}) return Images(resp[0]), resp[-1]
def get_pil_image(images: Images, index: int) -> Image: """ Converts Images output data to a PIL Image object. Use this function to read and analyze an image in memory. Do NOT use this function to save image data to disk; `save_image` is much faster. :param images: Images data from the build. :param index: The index of the image in Images.get_image :return A PIL image. """ return Image.open(io.BytesIO(images.get_image(index)))
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) width = 8 length = 8 # Create an empty room. self.load_streamed_scene("box_room_2018") # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 1.7, "z": 0}, look_at={"x": 0.3, "y": 0.8, "z": 2.2}, avatar_id="avatar")) self.communicate({"$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar"}) self.add_object(model_name="b06_bikenew", position={"x": 0.3, "y": 0, "z": 2.2}, rotation={"x": 0, "y": 180, "z": 0}, library="models_full.json") self.add_object(model_name="animal_dog_rtsit_1280", position={"x": 0.35, "y": 0, "z": 1.8}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": {"x": 0.3, "y": 0.8, "z": 2.2}}) images = Images(scene_data[0]) TDWUtils.save_images(images, "dog_bike", output_directory="replicated_images/interior")
def _orbit_and_capture_image(c: Controller, x: float, y: float, z: float) -> bool: """ Orbit around an object and check for missing materials. :param c: The controller. :param x: The next x positional coordinate. :param y: The next y positional coordinate. :param z: The next z positional coordinate. :return: True if there is a missing material in the image, False otherwise. """ # Teleport the avatar. # Look at the model. # Receive an image. resp = c.communicate([{ "$type": "teleport_avatar_to", "position": { "x": x, "y": y, "z": z } }, { "$type": "look_at", "object_id": MissingMaterials.OBJECT_ID, "use_centroid": True }]) # Check if there are any pink pixels. images = Images(resp[0]) img = Image.open(BytesIO(images.get_image(0))) for c in img.getcolors(maxcolors=100000): if c[1] == MissingMaterials.HELL_PINK: return True return False
def run(self): # Create the scene. # Set image encoding globals. self.start() commands = [TDWUtils.create_empty_room(12, 12), {"$type": "set_screen_size", "width": 128, "height": 128}, {"$type": "set_img_pass_encoding", "value": False}] # Create the avatar. commands.extend(TDWUtils.create_avatar(position={"x": 2.478, "y": 1.602, "z": 1.412}, look_at=TDWUtils.VECTOR3_ZERO, avatar_id="a")) # Enable all pass masks. Request an image for this frame only. commands.extend([{"$type": "set_pass_masks", "pass_masks": ["_img", "_id", "_category", "_mask", "_depth", "_normals", "_flow", "_depth_simple", "_albedo"], "avatar_id": "a"}, {"$type": "send_images", "ids": ["a"], "frequency": "once"}]) # Add objects. commands.append(self.get_add_object("small_table_green_marble", position=TDWUtils.VECTOR3_ZERO, rotation=TDWUtils.VECTOR3_ZERO, object_id=0)) commands.append(self.get_add_object("rh10", position={"x": 0.7, "y": 0, "z": 0.4}, rotation={"x": 0, "y": 30, "z": 0}, object_id=1)) commands.append(self.get_add_object("jug01", position={"x": -0.3, "y": 0.9, "z": 0.2}, rotation=TDWUtils.VECTOR3_ZERO, object_id=3)) commands.append(self.get_add_object("jug05", position={"x": 0.3, "y": 0.9, "z": -0.2}, rotation=TDWUtils.VECTOR3_ZERO, object_id=4)) # Send the commands. resp = self.communicate(commands) # Save the images. for r in resp[:-1]: r_id = OutputData.get_data_type_id(r) if r_id == "imag": # Save the images. TDWUtils.save_images(output_directory="dist", images=Images(r), filename="0") print(f"Images saved to: {Path('dist').resolve()}") # Stop the build. self.communicate({"$type": "terminate"})
def communicate(self, commands: Union[dict, List[dict]]) -> List[bytes]: """ See `Magnebot.communicate()`. Images are saved per-frame. """ resp = super().communicate(commands=commands) if not self._debug: # Save all images. got_images = False for i in range(len(resp) - 1): r_id = OutputData.get_data_type_id(resp[i]) if r_id == "imag": got_images = True images = Images(resp[i]) avatar_id = images.get_avatar_id() if avatar_id in self.image_directories: TDWUtils.save_images(filename=TDWUtils.zero_padding(self._image_count, 8), output_directory=self.image_directories[avatar_id], images=images) if got_images: self._image_count += 1 return resp
def run(self): self.start() depth_pass = "******" # Create an empty room. # Set the screen size. commands = [TDWUtils.create_empty_room(12, 12), {"$type": "set_screen_size", "width": 512, "height": 512}] # Add the avatar. commands.extend(TDWUtils.create_avatar(position={"x": 1.57, "y": 3, "z": 3.56}, look_at=TDWUtils.VECTOR3_ZERO)) # Add an object. # Request images and camera matrices. commands.extend([self.get_add_object("trunck", object_id=0), {"$type": "set_pass_masks", "pass_masks": [depth_pass]}, {"$type": "send_images"}, {"$type": "send_camera_matrices"}]) resp = self.communicate(commands) depth_image = None camera_matrix = None images = None for i in range(len(resp) - 1): r_id = OutputData.get_data_type_id(resp[i]) # Get the image. if r_id == "imag": images = Images(resp[i]) for j in range(images.get_num_passes()): if images.get_pass_mask(j) == depth_pass: depth_image = images.get_image(j) # Get the camera matrix. elif r_id == "cama": camera_matrix = CameraMatrices(resp[i]).get_camera_matrix() # Save the image. TDWUtils.save_images(images=images, output_directory="D:/depth_shader", filename="0", append_pass=True) # Get the depth values of each pixel. depth = TDWUtils.get_depth_values(image=depth_image, width=images.get_width(), height=images.get_height()) print(np.min(depth), np.max(depth)) print(depth) np.save("depth", depth) np.save("camera_matrix", camera_matrix) # Get a point cloud and write it to disk. point_cloud_filename = "point_cloud.txt" print(f"Point cloud saved to: {Path(point_cloud_filename)}") TDWUtils.get_point_cloud(depth=depth, filename=point_cloud_filename, camera_matrix=camera_matrix) # Show the depth values. plt.imshow(depth) plt.show() self.communicate({"$type": "terminate"})
def _do_frame(self, commands: Union[List[dict], dict]) -> AvatarStickyMitten: """ Send commands to the build. Receive images and sticky mitten avatar data. Save the images. :param commands: The commands for this frame. :return: The avatar data. """ # Add to the list of commands: Look at the avatar. frame_commands = [{ "$type": "look_at_position", "position": { "x": self.avatar_position[0], "y": self.avatar_position[1] + 0.5, "z": self.avatar_position[2] }, "avatar_id": self.cam_id }] if isinstance(commands, dict): frame_commands.append(commands) else: frame_commands.extend(commands) resp = self.communicate(frame_commands) avsm: Optional[AvatarStickyMitten] = None for r in resp[:-1]: r_id = OutputData.get_data_type_id(r) # Save images. if r_id == "imag": TDWUtils.save_images( images=Images(r), filename=TDWUtils.zero_padding(self.frame_number, 4), output_directory=PutObjectOnTable.OUTPUT_DIR) elif r_id == "avsm": avsm = AvatarStickyMitten(r) # Update the position of the avatar and the frame number. self.avatar_position = avsm.get_position() self.frame_number += 1 return avsm
def run(self): """ Generate room using COCO_TDW dataset """ objects_in_scene = 15 object_ids = [] # Get Category-Object mapping TDW_COCO_models = TDW_relationships.get_COCO_TDW_mapping() # print("TDWCOCO:", TDW_COCO_models) # print("KEYS:", TDW_COCO_models.keys()) # Gets COCO categories co-occurring in a scene # +5 is for dealing with failed object insertion attempts COCO_configurations = msCOCO_matrix.get_max_co_occurrence(5, int(objects_in_scene + 5)) configuration_1 = COCO_configurations[0] print("Config 1:", configuration_1) # TDW models/objects objects = [] for COCO_object in configuration_1: print(COCO_object) print(COCO_object.split()) if len(COCO_object.split()) > 1: COCO_object = COCO_object.split()[-1] print(COCO_object) # Check if COCO category is a key in the COCO-TDW mapping if COCO_object in TDW_COCO_models.keys(): # Gets random TDW model (from COCO-to-TDW map) based on COCO category key print(TDW_COCO_models[COCO_object]) model = TDW_COCO_models[COCO_object][random.randint(0, len(TDW_COCO_models[COCO_object]) - 1)] objects.append(model) print("COCO to TDW objects:", objects) # print(len(objects)) # Stores object categories that other objects can be placed upon (e.g. table, chair, couch, bed) surface_properties_list = TDW_COCO_models['table'] + TDW_COCO_models['chair'] + \ TDW_COCO_models['bed'] + TDW_COCO_models['couch'] + \ TDW_COCO_models['bench'] + TDW_COCO_models['refrigerator'] surface_categories = [] for surface_properties in surface_properties_list: surface_categories.append(surface_properties[0]) print("Surface Categories:", surface_categories) # Stores the actual surface object instances/ids alongside number of objects on the surface surface_object_ids = {} self.start() positions_list = [] # Stores current model locations and radii scene_dimensions = [] # Store scene/environment dimensions init_setup_commands = [{"$type": "set_screen_size", "width": 640, "height": 481}, {"$type": "set_render_quality", "render_quality": 1}] self.communicate(init_setup_commands) scene_lib = SceneLibrarian() # Disable physics when adding in new objects (objects float) self.communicate({"$type": "simulate_physics", "value": False}) for scene in scenes[1:]: # Load in scene # print("Scene", scene[0]) if scene[3] == "interior" and scene[0] == "box_room_2018": self.start() scene_record = scene_lib.get_record(scene[0]) self.communicate({"$type": "add_scene", "name": scene_record.name, "url": scene_record.get_url()}) # Gets dimensions of environments (e.g. inside, outside) in the scene # This command returns environment data in the form of a list of serialized byte arrays scene_bytes = self.communicate({"$type": "send_environments", "frequency": "once"}) # Iterating through data and parsing byte array # Ignoring the last element (the frame count) for b in scene_bytes[:-1]: e = Environments(b) for i in range(e.get_num()): center = e.get_center(i) bounds = e.get_bounds(i) env_id = e.get_id(i) scene_dimensions = [center, bounds, env_id] # Center, bounds are tuples # Must come before set_pass_masks avatar_position = TDWUtils.array_to_vector3([0.9 * scene_dimensions[1][0] / 2, scene_dimensions[1][1] / 2, 0]) # print("Avatar Position:", avatar_position) self.communicate(TDWUtils.create_avatar(avatar_id="avatar", position=avatar_position, look_at={"x": 0, "y": scene_dimensions[0][1] / 2, "z": 0})) # Set collision mode self.communicate({"$type": "set_avatar_collision_detection_mode", "mode": "continuous_speculative", "avatar_id": "avatar"}) # Alter FOV self.communicate({"$type": "set_field_of_view", "field_of_view": 80, "avatar_id": "avatar"}) # # Gets rid of header (Model: Category) # objects = TDW_COCO_models[1:] # random.shuffle(objects) obj_count = 0 obj_overlaps = 0 # Number of failed attempts to place object due to over-dense objects area while obj_count < objects_in_scene and obj_overlaps < 5: # Handles if object has been added to a flat surface added_to_surface = False print("Object COUNT:", obj_count) # Need to have random position for Bounds Data to return meaningful info valid_obj_pos = {"x": random.uniform(-1 * scene_dimensions[1][0] / 2, 0.5 * scene_dimensions[1][0] / 2), "y": scene_dimensions[1][1] / 4, "z": random.uniform(-0.9 * scene_dimensions[1][2] / 2, 0.9 * scene_dimensions[1][2] / 2)} print("First random position") # Add in the object at random position # Object will later be removed or updated accordingly after performing collision calculations record = ModelLibrarian(library="models_full.json").get_record(objects[obj_count][0]) print("Record gotten") print(objects[obj_count][0]) o_id = self.communicate({"$type": "add_object", "name": objects[obj_count][0], "url": record.get_url(), "scale_factor": record.scale_factor, "position": valid_obj_pos, "rotation": TDWUtils.VECTOR3_ZERO, "category": record.wcategory, "id": obj_count}) print("Random first add") # Returns bound data for added object bounds_data = self.communicate({"$type": "send_bounds", "frequency": "once"}) print("Bounds returned") # Appends object, with information on position and obj_radius, to positions_list # Length of buffer array should be 1 # print("Bounds Data:", bounds_data) for b in bounds_data[:-1]: # print("Buffer Loop:", b) b_id = OutputData.get_data_type_id(b) if b_id == "boun": # print("BOUNDS") o = Bounds(b) # print("# of Objects:", o.get_num()) # print("# of Failed Attempts:", obj_overlaps) # print("Buffer Array:", b) # print("Bounds Object:", o) for i in range(o.get_num()): print("Object ID:", o.get_id(i)) print("obj_count:", obj_count) print("Object:", objects[obj_count][0], "Category:", objects[obj_count][1]) # print("Object Center:", o.get_center(i)) # Only want to compute valid_position for object we are about to add # Skip any computation if this is not the case if o.get_id(i) != obj_count: continue # Useful for detecting if object fits in environment # print("Calculating if object fits in environment") width = distance.euclidean(o.get_left(i), o.get_right(i)) depth = distance.euclidean(o.get_front(i), o.get_back(i)) height = distance.euclidean(o.get_top(i), o.get_bottom(i)) # print("Width:", width) # print("Depth:", depth) # ("Height:", height) # Useful for avoiding object overlap # print("Calculating Object Bounds") center_to_top = distance.euclidean(o.get_center(i), o.get_top(i)) center_to_bottom = distance.euclidean(o.get_center(i), o.get_bottom(i)) center_to_left = distance.euclidean(o.get_center(i), o.get_left(i)) center_to_right = distance.euclidean(o.get_center(i), o.get_right(i)) center_to_front = distance.euclidean(o.get_center(i), o.get_front(i)) center_to_back = distance.euclidean(o.get_center(i), o.get_back(i)) # Max object radius (center to diagonal of bounding box) obj_radius = \ max(math.sqrt(center_to_top ** 2 + center_to_left ** 2 + center_to_front ** 2), math.sqrt(center_to_top ** 2 + center_to_right ** 2 + center_to_front ** 2), math.sqrt(center_to_top ** 2 + center_to_left ** 2 + center_to_back ** 2), math.sqrt(center_to_top ** 2 + center_to_right ** 2 + center_to_back ** 2), math.sqrt(center_to_bottom ** 2 + center_to_left ** 2 + center_to_front ** 2), math.sqrt(center_to_bottom ** 2 + center_to_right ** 2 + center_to_front ** 2), math.sqrt(center_to_bottom ** 2 + center_to_left ** 2 + center_to_back ** 2), math.sqrt(center_to_bottom ** 2 + center_to_right ** 2 + center_to_back ** 2)) # print("Obj_Radius:", obj_radius) # Set sweeping radius, based on scene plane dimensions l_radius = random.uniform(0, min(0.5 * scene_dimensions[1][0] / 2, 0.5 * scene_dimensions[1][2] / 2)) # Checking that object fits in scene viewing if (width > min(0.7 * scene_dimensions[1][0], 0.7 * scene_dimensions[1][2]) or depth > min(0.7 * scene_dimensions[1][0], 0.7 * scene_dimensions[1][2]) or height > 0.7 * scene_dimensions[1][1]): print("Object does not fit in scene") self.communicate([{"$type": "send_images", "frequency": "never"}, {"$type": "destroy_object", "id": obj_count}]) # Ensures next attempt to load in item is not the same item as before random.shuffle(objects) break # Not possible to find valid object position -- too many overlapping objects elif (not self._get_object_position(scene_dimensions=scene_dimensions, object_positions=positions_list, object_to_add_radius=obj_radius, max_tries=20, location_radius=l_radius)[0]): print("Could not calculate valid object location") self.communicate([{"$type": "send_images", "frequency": "never"}, {"$type": "destroy_object", "id": obj_count}]) obj_overlaps += 1 # Ensures next attempt to load in item is not the same item as before random.shuffle(objects) break # Find appropriate, non-overlapping object position # Reset object position to the valid position else: print("Object fits in scene") # Check if object fits on table, chair, couch, etc. # Add object if it fits, place it somewhere on top of the surface for surface_id in surface_object_ids.keys(): print("Surface ID:", surface_id) # Skip placement feasibility if the object is already a surface-type object # Ex. no chair on top of a table if objects[obj_count][0] in surface_categories: print("Object: %s is already a surface object" % objects[obj_count][0]) break # Check how many objects are on surface if surface_object_ids[surface_id] >= 3: print("Too many objects on surface") print("From surface objects dict:", surface_object_ids[surface_id]) continue surface_bounds = self.get_bounds_data(surface_id) surface_area = distance.euclidean(surface_bounds.get_left(0), surface_bounds.get_right(0)) * \ distance.euclidean(surface_bounds.get_front(0), surface_bounds.get_back(0)) obj_area = width * height if obj_area < surface_area: s_center_to_top = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_top(0)) s_center_to_bottom = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_bottom(0)) s_center_to_left = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_left(0)) s_center_to_right = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_right(0)) s_center_to_front = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_front(0)) s_center_to_back = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_back(0)) surface_radius = \ max(math.sqrt( s_center_to_top ** 2 + s_center_to_left ** 2 + s_center_to_front ** 2), math.sqrt( s_center_to_top ** 2 + s_center_to_right ** 2 + s_center_to_front ** 2), math.sqrt( s_center_to_top ** 2 + s_center_to_left ** 2 + s_center_to_back ** 2), math.sqrt( s_center_to_top ** 2 + s_center_to_right ** 2 + s_center_to_back ** 2), math.sqrt( s_center_to_bottom ** 2 + s_center_to_left ** 2 + s_center_to_front ** 2), math.sqrt( s_center_to_bottom ** 2 + s_center_to_right ** 2 + s_center_to_front ** 2), math.sqrt( s_center_to_bottom ** 2 + s_center_to_left ** 2 + s_center_to_back ** 2), math.sqrt( s_center_to_bottom ** 2 + s_center_to_right ** 2 + s_center_to_back ** 2)) print("Surface-type object") self.communicate({"$type": "destroy_object", "id": obj_count}) # Adding the object to the top of the surface on_pos = surface_bounds.get_top(0) on_y = on_pos[1] on_pos = TDWUtils.get_random_point_in_circle(np.array(on_pos), 0.7 * surface_radius) on_pos[1] = on_y on_pos = TDWUtils.array_to_vector3(on_pos) on_rot = {"x": 0, "y": random.uniform(-45, 45), "z": 0} # Add the object. print("Model Name on Surface:", objects[obj_count][0]) record = ModelLibrarian(library="models_full.json").get_record( objects[obj_count][0]) on_id = self.communicate({"$type": "add_object", "name": objects[obj_count][0], "url": record.get_url(), "scale_factor": record.scale_factor, "position": on_pos, "rotation": on_rot, "category": record.wcategory, "id": obj_count}) obj_count += 1 surface_object_ids[surface_id] += 1 object_ids.append(obj_count) print("Object added on top of surface") added_to_surface = True # Breaking out of surface objects loop break if added_to_surface: print("Breaking out of object loop") # Breaking out of object loop break print("Post-surface") valid_obj_pos = self._get_object_position(scene_dimensions=scene_dimensions, object_positions=positions_list, object_to_add_radius=obj_radius, max_tries=20, location_radius=l_radius)[1] print("Position calculated") positions_list.append(ObjectPosition(valid_obj_pos, obj_radius)) self.communicate([{"$type": "send_images", "frequency": "never"}, {"$type": "destroy_object", "id": obj_count}]) added_object_id = self.communicate({"$type": "add_object", "name": objects[obj_count][0], "url": record.get_url(), "scale_factor": record.scale_factor, "position": valid_obj_pos, "rotation": {"x": 0, "y": 0, "z": 0}, "category": record.wcategory, "id": obj_count}) # print("Object ID:", added_object_id) print("Regular object add") object_ids.append(added_object_id) # If TDW model belongs to surface categories, store id_information if objects[obj_count][0] in surface_categories: surface_object_ids[obj_count] = 0 # Rotate the object randomly print("Rotating object") self.communicate({"$type": "rotate_object_by", "angle": random.uniform(-45, 45), "axis": "yaw", "id": obj_count, "is_world": True}) # Minimal rotating for position differences # Don't rotate the object if doing so will result in overlap into scene if not (o.get_bottom(i)[1] < 0 or o.get_top(i)[1] > 0.9 * scene_dimensions[1][1]): pitch_angle = random.uniform(-45, 45) self.communicate({"$type": "rotate_object_by", "angle": pitch_angle, "axis": "pitch", "id": obj_count, "is_world": True}) roll_angle = random.uniform(-45, 45) self.communicate({"$type": "rotate_object_by", "angle": roll_angle, "axis": "roll", "id": obj_count, "is_world": True}) # Don't need this for just changing positions # Setting random materials/textures # Looping through sub-objects and materials sub_count = 0 for sub_object in record.substructure: # Loop through materials in sub-objects for j in range(len(sub_object)): # Get random material and load in material = random.choice(materials[1:]) self.load_material(material) print("Material loaded") # Set random material on material of sub-object self.communicate({"$type": "set_visual_material", "material_index": j, "material_name": material[0], "object_name": sub_object['name'], "id": obj_count}) print("Material set") sub_count += 1 if sub_count > 10: break break print("Updating count") obj_count += 1 print("Breaking out of object_id loop") break # Break out of buffer loop print("Breaking out of buffer loop") break # Move onto next iteration of while loop (next object to load in) print("Object added - next while loop iteration") continue # for i in range(200): # self.communicate({"$type": "simulate_physics", # "value": False}) # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) # Capture scene # NOTE: THESE SCENES GET REPLACED IN THE TARGET DIRECTORY scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": {"x": 0, "y": scene_dimensions[0][1] / 2, "z": 0}}) images = Images(scene_data[0]) TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=path) print("Object ids:", object_ids)
def _write_frame(self, frames_grp: h5py.Group, resp: List[bytes], frame_num: int) -> \ Tuple[h5py.Group, h5py.Group, dict, bool]: num_objects = len(self.object_ids) # Create a group for this frame. frame = frames_grp.create_group(TDWUtils.zero_padding(frame_num, 4)) # Create a group for images. images = frame.create_group("images") # Transforms data. positions = np.empty(dtype=np.float32, shape=(num_objects, 3)) forwards = np.empty(dtype=np.float32, shape=(num_objects, 3)) rotations = np.empty(dtype=np.float32, shape=(num_objects, 4)) camera_matrices = frame.create_group("camera_matrices") # Parse the data in an ordered manner so that it can be mapped back to the object IDs. tr_dict = dict() for r in resp[:-1]: r_id = OutputData.get_data_type_id(r) if r_id == "tran": tr = Transforms(r) for i in range(tr.get_num()): pos = tr.get_position(i) tr_dict.update({ tr.get_id(i): { "pos": pos, "for": tr.get_forward(i), "rot": tr.get_rotation(i) } }) # Add the Transforms data. for o_id, i in zip(self.object_ids, range(num_objects)): if o_id not in tr_dict: continue positions[i] = tr_dict[o_id]["pos"] forwards[i] = tr_dict[o_id]["for"] rotations[i] = tr_dict[o_id]["rot"] elif r_id == "imag": im = Images(r) # Add each image. for i in range(im.get_num_passes()): pass_mask = im.get_pass_mask(i) # Reshape the depth pass array. if pass_mask == "_depth": image_data = TDWUtils.get_shaped_depth_pass(images=im, index=i) else: image_data = im.get_image(i) images.create_dataset(pass_mask, data=image_data, compression="gzip") # Save PNGs if pass_mask in self.save_passes: filename = pass_mask[1:] + "_" + TDWUtils.zero_padding( frame_num, 4) + "." + im.get_extension(i) path = self.png_dir.joinpath(filename) if pass_mask in ["_depth", "_depth_simple"]: Image.fromarray( TDWUtils.get_shaped_depth_pass( images=im, index=i)).save(path) else: with open(path, "wb") as f: f.write(im.get_image(i)) # Add the camera matrices. elif OutputData.get_data_type_id(r) == "cama": matrices = CameraMatrices(r) camera_matrices.create_dataset( "projection_matrix", data=matrices.get_projection_matrix()) camera_matrices.create_dataset( "camera_matrix", data=matrices.get_camera_matrix()) objs = frame.create_group("objects") objs.create_dataset("positions", data=positions.reshape(num_objects, 3), compression="gzip") objs.create_dataset("forwards", data=forwards.reshape(num_objects, 3), compression="gzip") objs.create_dataset("rotations", data=rotations.reshape(num_objects, 4), compression="gzip") return frame, objs, tr_dict, False
def trial(self, m_o, s_o, b_o, output_directory, nimages): """ Collide a chair with a fridge. :param m_c: mass of the chair. :param s_c: The static friction of the chair. :param b_c: The bounciness of the chair. """ # Destroy all objects currently in the scene. self.communicate({"$type": "destroy_all_objects"}) # Set screen size self.communicate({ "$type": "set_screen_size", "height": 480, "width": 640 }) # Create the avatar. self.communicate({ "$type": "create_avatar", "type": "A_Img_Caps_Kinematic", "id": "a" }) # Teleport the avatar to the valid position. # Enable image capture. self.communicate([ { "$type": "teleport_avatar_to", "avatar_id": "a", "position": { "x": -1.5, "y": 2.5, "z": 7 } }, { "$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_img", "_id"] }, { "$type": "send_images", "frequency": "always" }, ]) # place the ramp ramp_id = self.add_object("ramp_with_platform_30", position={ "x": -2.1, "y": 0, "z": 0 }) self.communicate([{ "$type": "scale_object", "id": ramp_id, "scale_factor": (1, 1, 1) }]) self.communicate([{ "$type": "set_kinematic_state", "id": ramp_id, "is_kinematic": True, "use_gravity": False }]) self.communicate([{ "$type": "set_visual_material", "old_material_index": 1, "new_material_name": "3d_mesh_technical_fabric", "object_name": "ramp_with_platform_30", "id": ramp_id }]) # place the elevated platform platform_id = self.add_object("fridge_box", position={ "x": 1.5, "y": 0, "z": 0 }) self.communicate({ "$type": "set_kinematic_state", "id": platform_id, "is_kinematic": True, "use_gravity": False }) # place a collider object on the ramp (salt_mill_max_2013,b05_fire_extinguisher) collider_id = self.add_object("b05_fire_extinguisher", position={ "x": -4, "y": 0, "z": 0 }) # Set the scale, mass, and friction of the object. self.communicate({ "$type": "scale_object", "id": collider_id, "scale_factor": { "x": 2, "y": 2, "z": 2 } }) self.communicate({"$type": "set_mass", "id": collider_id, "mass": 1}) self.communicate({ "$type": "set_physic_material", "id": collider_id, "static_friction": 0.5, "bounciness": 0.5 }) #self.communicate({"$type":"set_color", # "color":{"r":1, "g":0, "b":1, "a":1}, # "id": collider_id}) #self.communicate({"$type": "nudge_onto_surface", # "id": collider_id, # "is_circle": True, # "nudge_step": .05, # "surface_object_id": ramp_id}) # place the basket on the ramp (bucketnew, bowl_wood_a_01) if 0: basket_id = self.add_object("bowl_wood_a_01", position={ "x": 0.1, "y": 1, "z": 0 }) self.communicate({ "$type": "set_kinematic_state", "id": basket_id, "is_kinematic": True, "use_gravity": False }) # Create the object. # b04_geosphere001, chair_billiani_doll, b04_orange_00, b05_1_(1), b05_baseballnew_v03_12, b05_geosphere001, base-ball, prim_sphere object_id = self.add_object("b04_geosphere001", position={ "x": 0.2, "y": 1, "z": 0 }) # Set the scale, mass, and friction of the object. self.communicate({ "$type": "scale_object", "id": object_id, "scale_factor": { "x": 10, "y": 10, "z": 10 } }) self.communicate({"$type": "set_mass", "id": object_id, "mass": m_o}) self.communicate({ "$type": "set_physic_material", "id": object_id, "static_friction": s_o, "bounciness": b_o }) self.communicate({ "$type": "set_color", "color": { "r": 0.5, "g": 1, "b": 0, "a": 1 }, "id": object_id }) # nudge the ball onto the surface of the platform # self.communicate({"$type": "nudge_onto_surface", # "id": object_id, # "is_circle": False, # "nudge_step": 1.0, # "surface_object_id": ramp_id}) # apply force to the ball self.communicate([{ "$type": "apply_force_to_object", "force": { "x": -8, "y": -0.1, "z": 0.5 }, "id": object_id }]) # Create the output directory. if os.path.exists(output_directory): shutil.rmtree(output_directory) sleep(0.5) os.mkdir(output_directory) # Capture n images. for i in range(nimages): # Look at the object. resp = self.communicate({ "$type": "look_at_position", "avatar_id": "a", "position": { "x": -1.5, "y": 1, "z": 0 } }) images = Images(resp[0]) frame = resp[-1] # Save the image. TDWUtils.save_images(images, frame, output_directory=output_directory)
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 640, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("laser_cut_white_norway_spruce") self.communicate({ "$type": "add_material", "name": "laser_cut_white_norway_spruce", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "laser_cut_white_norway_spruce" }) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 1.5, "y": 1.6, "z": 0 }, look_at=TDWUtils.array_to_vector3( [0, 0.5, 0]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table = self.add_object( model_name="boconcept_lugo_dining_table_vray1.5", position={ "x": 0, "y": 0, "z": 0.0 }, rotation={ "x": 0, "y": -90, "z": 0 }, library="models_full.json") table_bounds = self.get_bounds_data(table) top = table_bounds.get_top(0) self.add_object(model_name='macbook_001', position={ "x": 0, "y": top[1], "z": 0.1 }, rotation={ "x": 0, "y": 90, "z": 0 }, library="models_full.json") self.add_object(model_name='spunlight_designermesh_lamp', position={ "x": -0.19, "y": top[1], "z": -0.4 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") self.add_object(model_name="apple_magic_mouse_2_(2015)_vray", position={ "x": 0, "y": top[1], "z": 0.32 }, rotation={ "x": 0, "y": 86, "z": 0 }, library="models_full.json") self.add_object(model_name="f10_apple_iphone_4", position={ "x": 0.14, "y": top[1], "z": -0.3 }, rotation={ "x": 0, "y": 12, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([0, 0.5, 0]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "phone", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior" )
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 600, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("parquet_wood_mahogany") self.communicate({ "$type": "add_material", "name": "parquet_wood_mahogany", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "parquet_wood_mahogany" }) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": -1, "y": 1.4, "z": -1 }, look_at=TDWUtils.array_to_vector3( [-0.1, 0.7, -0.1]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table_id = self.add_object( model_name= "b03_restoration_hardware_pedestal_salvaged_round_tables", position={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="club_sandwich", position={ "x": 0.45, "y": top[1], "z": 0.25 }, rotation={ "x": 0, "y": 30, "z": 0 }, library="models_full.json") self.add_object(model_name="coffeemug", position={ "x": 0.03, "y": top[1], "z": 0.03 }, library="models_full.json") self.add_object(model_name="knife3", position={ "x": -0.17, "y": top[1], "z": 0.15 }, rotation={ "x": 0, "y": 24, "z": 90 }, library="models_full.json") self.add_object(model_name="notes_02", position={ "x": 0.17, "y": top[1], "z": -0.12 }, rotation={ "x": 0, "y": 24, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([-0.1, 0.7, -0.1]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "sadnwich2", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior" )
def run(self): # Create the output directory. output_directory = "example_output" if os.path.exists(output_directory): shutil.rmtree(output_directory) sleep(0.5) os.mkdir(output_directory) self.start() # Load a streamed scene. This scene covers a relatively small area, and is optimized for use with HDRI maps. self.load_streamed_scene(scene="building_site") # Add the object. lamp_id = self.add_object("alma_floor_lamp", position={ "x": 0, "y": 0, "z": 0 }, rotation={ "x": 0, "y": 90, "z": 0 }) # Create the avatar. self.communicate({ "$type": "create_avatar", "type": "A_Img_Caps_Kinematic", "id": "a" }) # Change the skybox. self.communicate(self.get_add_hdri_skybox("bergen_4k")) # Teleport the avatar to a suitable position. # Set the pass masks to _img. # Enable image capture. self.communicate([ { "$type": "teleport_avatar_to", "avatar_id": "a", "position": { "x": -4.28, "y": 0.85, "z": 4.27 } }, { "$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_img"] }, { "$type": "send_images", "frequency": "always" }, ]) # Create the recommended post-processing setup for HDRI. self.communicate([{ "$type": "set_post_exposure", "post_exposure": 0.6 }, { "$type": "set_contrast", "contrast": -20 }, { "$type": "set_saturation", "saturation": 10 }, { "$type": "set_screen_space_reflections", "enabled": False }, { "$type": "set_vignette", "enabled": False }]) # Set the shadow strength to maximum. self.communicate({"$type": "set_shadow_strength", "strength": 1.0}) # Capture 48 images. for i in range(48): # Look at the lamp. resp = self.communicate({ "$type": "look_at", "avatar_id": "a", "object_id": lamp_id, "use_centroid": True }) # Rotate the skybox by 15 degrees. self.communicate({"$type": "rotate_hdri_skybox_by", "angle": 15}) images = Images(resp[0]) # Save the image. TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=output_directory)
def run(self): self.start() # init_setup_commands = [{"$type": "set_screen_size", # "width": 600, # "height": 480}, # {"$type": "set_render_quality", # "render_quality": 5}] # self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 2.5, "y": 1.5, "z": 0.3 }, look_at=TDWUtils.array_to_vector3( [3, 0.5, 0.5]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("gravel_white") self.communicate({ "$type": "add_material", "name": "gravel_white", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "gravel_white" }) # self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"}) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) bench = self.add_object(model_name="b04_wood_metal_park_bench", position={ "x": 3, "y": 0, "z": 0.5 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") bench_bounds = self.get_bounds_data(bench) top = bench_bounds.get_top(0) self.add_object(model_name="b05_baseball_bat_01", position={ "x": 2.8, "y": top[1] - 0.35, "z": 0 }, rotation={ "x": 0, "y": 90, "z": 90 }, library="models_full.json") self.add_object(model_name="b05_baseballnew_v03_12", position={ "x": 2.9, "y": top[1] - 0.35, "z": 0.4 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([3, 0.5, 0.5]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "bench_ball", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior" )
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 640, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 2.0, "y": 0.8, "z": -0.55 }, look_at=TDWUtils.array_to_vector3( [3, 0.6, 0.35]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table = self.add_object(model_name="b05_table_new", position={ "x": 3, "y": 0, "z": 0.35 }, rotation={ "x": 0, "y": -90, "z": 0 }, library="models_full.json") # record = ModelLibrarian(library='models_full.json').get_record("b03_simpsons_london_-_round_hudson_mirror") # self.communicate({"$type": "add_object", # "name": "b05_tv1970", # "url": record.get_url(), # "scale_factor": 0.5, # "position": {"x": 3, "y": 0.6, "z": 0.35}, # "rotation": {"x": 0, "y": -90, "z": 0}, # "category": record.wcategory, # "id": self.get_unique_id()}) table_bounds = self.get_bounds_data(table) top = table_bounds.get_top(0) self.add_object(model_name="b03_tv_controller_2013__vray", position={ "x": 3, "y": top[1], "z": 0.7 }, rotation={ "x": 0, "y": 0, "z": 90 }, library="models_full.json") # # self.add_object(model_name='cgaxis_models_65_06_vray', # position={"x": 3, "y": top[1], "z": 0.25}, # rotation={"x": 0, "y": 0, "z": 0}, # library="models_full.json") self.add_object(model_name="b04_phone", position={ "x": 3, "y": top[1], "z": 0.25 }, rotation={ "x": 0, "y": -90, "z": 0 }, library="models_full.json") self.add_object(model_name="books_collection__1_2013", position={ "x": 3.3, "y": top[1], "z": -0.2 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([3, 0.5, 0.35]) }) images = Images(scene_data[0]) TDWUtils.save_images(images, "remote1", output_directory="replicated_images")
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.load_streamed_scene(scene="tdw_room_2018") # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": -0.8, "y": 1.6, "z": 0.6}, look_at=TDWUtils.array_to_vector3([0, 0.4, 0]), avatar_id="avatar")) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) table_id = self.add_object(model_name="glass_table_round", position={"x": 0, "y": 0, "z": 0}, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="b03_orange", position={"x": 0, "y": top[1], "z": 0}, library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.17) position[1] = top[1] self.add_object(model_name="b03_orange", position={"x": 0, "y": top[1], "z": 0}, library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.4) position[1] = top[1] self.add_object(model_name="b04_banana", position=TDWUtils.array_to_vector3(position), rotation={"x": 0, "y": 40, "z": 0}, library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.4) position[1] = top[1] self.add_object(model_name="b04_banana", position=TDWUtils.array_to_vector3(position), library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.5) position[1] = top[1] self.add_object(model_name="red_apple", position=TDWUtils.array_to_vector3(position), library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.6) position[1] = top[1] self.add_object(model_name="vk0076_fruitfork", position=TDWUtils.array_to_vector3(position), library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([0, 0.4, 0])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "fruit_table", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior")
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 600, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) # Create an empty room. self.load_streamed_scene(scene="tdw_room_2018") # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": -1, "y": 0.9, "z": -1 }, look_at=TDWUtils.array_to_vector3( [-0.1, 0.5, -0.1]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table_id = self.add_object(model_name="coffee_table_glass_round", position={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="am151_043_00", position={ "x": 0.15, "y": top[1], "z": 0.15 }, rotation={ "x": 0, "y": 30, "z": 0 }, library="models_full.json") self.add_object(model_name="b04_tea_cup", position={ "x": 0.3, "y": top[1], "z": 0.1 }, library="models_full.json") self.add_object(model_name="cgaxis_models_65_06_vray", position={ "x": -0.17, "y": top[1], "z": 0.15 }, rotation={ "x": 0, "y": 24, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([-0.1, 0.45, -0.1]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "sadnwich1", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior" )
def run(self): # init_setup_commands = [{"$type": "set_screen_size", # "width": 600, # "height": 480}, # {"$type": "set_render_quality", # "render_quality": 5}] # self.communicate(init_setup_commands) # Load the streamed scene. self.load_streamed_scene(scene="tdw_room_2018") # Add the objects. self.add_object("live_edge_coffee_table", position={"x": -12.8, "y": 0.96, "z": -5.47}, rotation={"x": 0, "y": -90, "z": 0}) self.add_object(model_name="03_106", position={"x": -14.394, "y": 0.96, "z": -7.06}, rotation={"x": 0, "y": 21.35, "z": 0}) self.add_object("chair_billiani_doll", position={"x": -15.15, "y": 0.96, "z": -6.8}, rotation={"x": 0, "y": 63.25, "z": 0}) # self.add_object("zenblocks", # position={"x": -12.7, "y": 1.288, "z": -5.55}, # rotation={"x": 0, "y": 90, "z": 0}) # Organize all setup commands into a single list. # We want a high-quality result, so set 1080P screen resolution / aspect ratio # and maximum render quality. init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create the avatar and adjust its field of view for a wider camera angle. init_setup_commands.extend([{"$type": "create_avatar", "type": "A_Img_Caps_Kinematic", "id": "a"}, {"$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "a"}]) # Adjust post-processing parameters to achieve a suitable depth of field and exposure, and disable vignette. # Also adjust the ambient occlusion parameters for realistic shadowing in corners and under furniture objects. init_setup_commands.extend([{"$type": "set_aperture", "aperture": 1.6}, {"$type": "set_focus_distance", "focus_distance": 2.25}, {"$type": "set_post_exposure", "post_exposure": 0.4}, {"$type": "set_vignette", "enabled": False}, {"$type": "set_ambient_occlusion_intensity", "intensity": 0.175}, {"$type": "set_ambient_occlusion_thickness_modifier", "thickness": 3.5}]) # Set shadow strength to full. init_setup_commands.append({"$type": "set_shadow_strength", "strength": 1.0}) # Execute the setup commands. self.communicate(init_setup_commands) # Teleport the avatar to the desired position. # Set the pass masks to _img. # Enable image capture. resp = self.communicate([{"$type": "teleport_avatar_to", "avatar_id": "a", "position": {"x": -10.48, "y": 1.81, "z": -6.583}}, {"$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_img"]}, {"$type": "send_images", "frequency": "always"}, {"$type": "look_at_position", "position": {"x": -12.873, "y": 1.85, "z": -5.75}, "avatar_id": "a"}]) # Parse the output image data. images = Images(resp[0]) # Save the image. TDWUtils.save_images(images, "photoreal_living", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior")
"$type": "look_at", "avatar_id": avatar_id, "object_id": box_id }, { "$type": "set_pass_masks", "avatar_id": avatar_id, "pass_masks": ["_img"] }, { "$type": "send_images", "frequency": "once", "avatar_id": avatar_id }]) # Get the image. for r in resp[:-1]: r_id = OutputData.get_data_type_id(r) # Find the image data. if r_id == "imag": img = Images(r) # Usually, you'll want to use one of these functions, but not both of them: # Use this to save a .jpg TDWUtils.save_images(img, filename="test_img") print(f"Image saved to: {Path('dist/test_img.jpg').resolve()}") # Use this to convert the image to a PIL image, which can be processed by a ML system at runtime. # The index is 0 because we know that there is only one pass ("_img"). pil_img = TDWUtils.get_pil_image(img, index=0)