def run(self): self.start() width = 12 length = 12 # Create an empty room. self.communicate(TDWUtils.create_empty_room(width, length)) # Disable physics. self.communicate({"$type": "simulate_physics", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": -5, "y": 10, "z": -2 }, look_at=TDWUtils.VECTOR3_ZERO)) # Create the dining table. self.dining_table() # Create the sofa. self.sofa(width, length) # Bake the NavMesh. self.communicate({"$type": "bake_nav_mesh", "carve_type": "none"}) # Get a random position on the NavMesh. position = TDWUtils.get_random_position_on_nav_mesh( self, width, length) position = TDWUtils.array_to_vector3(position) # Add a side table. self.side_table(position)
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(10, 10)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) self.communicate(self.get_add_hdri_skybox("autumn_hockey_4k")) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": 3, "y": 1.3, "z": 0}, look_at=TDWUtils.array_to_vector3([0, 0.5, 0]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("grass_countryside") print(record) self.communicate({"$type": "add_material", "name": "grass_countryside", "url": record.get_url()}) self.communicate({"$type": "set_proc_gen_floor_material", "name": "grass_countryside"}) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) self.add_object(model_name="b05_clochild4", position={"x": 0, "y": 0, "z": 0}, rotation={"x": 0, "y": 30, "z": 0}, library="models_full.json") self.add_object(model_name="baseballbat", position={"x": 0.5, "y": 0, "z": 0.3}, rotation={"x": 0, "y": 30, "z": 0}, library="models_full.json") self.add_object(model_name="base-ball", position={"x": 0.4, "y": 0, "z": 0.22}, rotation={"x": 0, "y": 30, "z": 10}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([0, 0.5, 0])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "kid_baseball", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def run(self): self.start() # Create an empty room. self.communicate(TDWUtils.create_empty_room(12, 12)) # Create the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 0, "y": 3, "z": -4 }, look_at=TDWUtils.VECTOR3_ZERO)) # Set the pass mask to _depth only. # Get an image. resp = self.communicate([ self.get_add_object("rh10", object_id=0), { "$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_depth"] }, { "$type": "send_images", "frequency": "once" } ]) images = Images(resp[0]) # Get the depth values of each pixel. depth = TDWUtils.get_depth_values(images.get_image(0)) print(depth)
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(32, 32)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": -1.5, "y": 1.5, "z": -15}, look_at=TDWUtils.array_to_vector3([-1.7, 0.7, -2.3]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("concrete_chipped_cracked") self.communicate({"$type": "add_material", "name": "concrete_chipped_cracked", "url": record.get_url()}) self.communicate({"$type": "set_proc_gen_floor_material", "name": "concrete_chipped_cracked"}) # self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"}) self.communicate({"$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar"}) bus = self.add_object(model_name="b06_bus_new", position={"x": -2, "y": 0, "z": 0}, rotation={"x": 0, "y": -70, "z": 0}, library="models_full.json") bus_bounds = self.get_bounds_data(bus) bus_front = bus_bounds.get_front(0) print(bus_front) self.add_object(model_name="3dscan_man_004", position={"x": -3.7, "y": 0, "z": -7.3}, rotation={"x": 0, "y": 0, "z": 0}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([-1.7, 0.7, -2.3])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "bus", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def get_scene_initialization_commands(self) -> List[dict]: # Load the proc-gen room. # Disable images. return [{ "$type": "load_scene", "scene_name": "ProcGenScene" }, TDWUtils.create_empty_room(100, 100)]
def _get_scene_init_commands(self, scene: str = None, layout: int = None, room: int = -1) -> List[dict]: commands = [{"$type": "load_scene", "scene_name": "ProcGenScene"}, TDWUtils.create_empty_room(4, 4)] self.o_id, o_commands = self._add_object("trunck", position={"x": 0, "y": 0, "z": 1}) commands.extend(o_commands) return commands
def run(self): self.start() commands = [TDWUtils.create_empty_room(self.size, self.size)] commands.extend( TDWUtils.create_avatar(position={ "x": 0, "y": 1.5, "z": 0 }, avatar_id="a")) commands.extend([{ "$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_img"] }, { "$type": "send_images", "frequency": "always" }]) self.communicate(commands) # Listen for keys. # Turn left. self.listen(key="left", commands={ "$type": "rotate_sensor_container_by", "axis": "yaw", "angle": self.angle * -1, "sensor_name": "SensorContainer", "avatar_id": "a" }) # Turn right. self.listen(key="right", commands={ "$type": "rotate_sensor_container_by", "axis": "yaw", "angle": self.angle, "sensor_name": "SensorContainer", "avatar_id": "a" }) # Quit the application. self.listen(key="esc", function=self.stop) # Continue until the quit key is pressed. i = 0 while not self.done: # Listen for keyboard input. Receive output data. resp = self.communicate([]) for r in resp[:-1]: r_id = OutputData.get_data_type_id(r) # Save images. if r_id == "imag": TDWUtils.save_images( images=Images(r), output_directory=self.images_directory, filename=TDWUtils.zero_padding(i, width=4)) # Increment the image number. i += 1 self.communicate({"$type": "terminate"})
def run(self): # Create the output directory. output_directory = "example_output" if os.path.exists(output_directory): shutil.rmtree(output_directory) sleep(0.5) os.mkdir(output_directory) print(f"Images will be saved to: {output_directory}") self.start() # Create an empty room. self.communicate(TDWUtils.create_empty_room(12, 12)) # Add the objects. lamp_id = self.add_object("alma_floor_lamp", position={"x": 1, "y": 0, "z": 0}, rotation={"x": 0, "y": 90, "z": 0}) self.add_object("live_edge_coffee_table", position={"x": 1.5, "y": 0, "z": 1.5}, rotation={"x": 0, "y": 30, "z": 0}) self.add_object("small_table_green_marble", position={"x": -0.9, "y": 0, "z": -1.35}) # Create the avatar. self.communicate({"$type": "create_avatar", "type": "A_Img_Caps_Kinematic", "id": "a"}) # Try to find a valid position on the NavMesh. x, y, z = TDWUtils.get_random_position_on_nav_mesh(self, 12, 12) # Teleport the avatar to the valid position. # Apply a force to the lamp. # Set the pass masks to _img. # Enable image capture. self.communicate([{"$type": "teleport_avatar_to", "avatar_id": "a", "position": {"x": x, "y": 1.5, "z": z}}, {"$type": "apply_force_to_object", "force": {"x": 2, "y": 1, "z": 0}, "id": lamp_id}, {"$type": "set_pass_masks", "avatar_id": "a", "pass_masks": ["_img", "_id"]}, {"$type": "send_images", "frequency": "always"}, ]) # Capture 100 images. for i in range(100): # Look at the lamp. resp = self.communicate({"$type": "look_at", "avatar_id": "a", "object_id": lamp_id, "use_centroid": True}) images = Images(resp[0]) # Save the image. TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=output_directory)
def run(self): self.start() # Create the room. self.communicate(TDWUtils.create_empty_room(20, 20, 1)) # Run a trial. # self.trial(1.5, 0.05, 0.2, "./fall_roll_ramp_collide_output/images output/basket/run1_m1.5_s0.05_b0.2_blue", 200) self.trial(1.25, 0.05, 0.1, "./output/images output/collision2", 200)
def create_scene(self): """ Initialize a blank room. """ self.start() commands = [TDWUtils.create_empty_room(12, 12)] commands.extend(TDWUtils.create_avatar(position={"x": 5, "y": 10, "z": 5}, look_at=TDWUtils.VECTOR3_ZERO)) self.communicate(commands)
def _initialize_scene(self, c: Controller) -> List[dict]: return [{"$type": "load_scene"}, TDWUtils.create_empty_room(self._room_size[0], self._room_size[1]), {"$type": "destroy_all_objects"}, self._reverb, {"$type": "create_avatar", "type": "A_Img_Caps_Kinematic", "id": "a"}, {"$type": "add_environ_audio_sensor"}, {"$type": "toggle_image_sensor"}]
def next(self): """ Starts a new trial of an experiment. Populates a room with randomly placed lamps and a single avatar. """ # Get a random avatar position. ret_dict = {} avatar_position = [random.uniform(-self.room_dim * 0.33, self.room_dim * 0.33), 0, random.uniform(-self.room_dim * 0.33, self.room_dim * 0.33)] config_actions = [] # Begin placing objects in the room. current_objects = [] for good_obj in range(self.num_good_objects): current_objects.append( self.get_object_dict(self.find_open_position(current_objects, avatar_position), self.good_obj_name)) for bad_obj in range(self.num_bad_objects): current_objects.append( self.get_object_dict(self.find_open_position(current_objects, avatar_position), self.bad_obj_name)) # Add the TDW configurations to a list to be sent inside TDWGenerator.restart_trial() # Initialize the scene. # Set the render quality to as low as possible. # Tell the build to return transform, rigidbody, and image data. config_actions.append([{'$type': "load_scene", 'scene_name': "ProcGenScene"}]) config_actions.append([{"$type": "send_transforms", "frequency": "always"}, {"$type": "send_collisions", "frequency": "always"}, {"$type": "send_rigidbodies", "frequency": "always"}, {"$type": "send_images", "frequency": "always"}, {"$type": "set_render_quality", "render_quality": 0}, {"$type": "set_screen_size", "width": 256, "height": 256}, {"$type": "set_img_pass_encoding", "value": False}, {"$type": "set_post_process", "value": False}, TDWUtils.create_empty_room(self.room_dim, self.room_dim)]) for c_o in current_objects: config_actions.append(self.get_add_object(model_name=c_o["name"], object_id=self.get_unique_id(), position=c_o["position"], rotation=c_o["orientation"])) config_actions.append([{"$type": "create_avatar", "type": self.avatar_type, "id": "a", "position": avatar_position, "rotation": [0.0, 0.0, 0.0]}, {"$type": "teleport_avatar_to", "position": {"x": avatar_position[0], "y": 1.0, "z": avatar_position[2]}} ]) config_actions.append([{"$type": "set_pass_masks", "pass_masks": ["_img"]}]) config_actions.append([{"$type": "send_id_pass_segmentation_colors", "frequency": "always"}]) config_actions.append([{"$type": "set_avatar_drag", "drag": 80, "angular_drag": 4}]) # Set up the environment and start up the trial ret_dict['latents'] = self.restart_trial(config_actions) return ret_dict
def run(self): self.start() depth_pass = "******" # Create an empty room. # Set the screen size. commands = [TDWUtils.create_empty_room(12, 12), {"$type": "set_screen_size", "width": 512, "height": 512}] # Add the avatar. commands.extend(TDWUtils.create_avatar(position={"x": 1.57, "y": 3, "z": 3.56}, look_at=TDWUtils.VECTOR3_ZERO)) # Add an object. # Request images and camera matrices. commands.extend([self.get_add_object("trunck", object_id=0), {"$type": "set_pass_masks", "pass_masks": [depth_pass]}, {"$type": "send_images"}, {"$type": "send_camera_matrices"}]) resp = self.communicate(commands) depth_image = None camera_matrix = None images = None for i in range(len(resp) - 1): r_id = OutputData.get_data_type_id(resp[i]) # Get the image. if r_id == "imag": images = Images(resp[i]) for j in range(images.get_num_passes()): if images.get_pass_mask(j) == depth_pass: depth_image = images.get_image(j) # Get the camera matrix. elif r_id == "cama": camera_matrix = CameraMatrices(resp[i]).get_camera_matrix() # Save the image. TDWUtils.save_images(images=images, output_directory="D:/depth_shader", filename="0", append_pass=True) # Get the depth values of each pixel. depth = TDWUtils.get_depth_values(image=depth_image, width=images.get_width(), height=images.get_height()) print(np.min(depth), np.max(depth)) print(depth) np.save("depth", depth) np.save("camera_matrix", camera_matrix) # Get a point cloud and write it to disk. point_cloud_filename = "point_cloud.txt" print(f"Point cloud saved to: {Path(point_cloud_filename)}") TDWUtils.get_point_cloud(depth=depth, filename=point_cloud_filename, camera_matrix=camera_matrix) # Show the depth values. plt.imshow(depth) plt.show() self.communicate({"$type": "terminate"})
def run(self): self.start() # Create the room. self.communicate(TDWUtils.create_empty_room(12, 12)) # Run a series of trials. for j in range(5): self.trial() # Terminate the build. self.communicate({"$type": "terminate"})
def run(self): self.start() # Create the room. self.communicate(TDWUtils.create_empty_room(12, 12)) # Run a trial. self.trial(0.1, 0.12, 0.6, 0.8, 0.75, 0.1, ["obj"]) # Run a trial with different friction and bounciness parameters. # Listen for environment collisions. self.trial(0.9, 0.85, 1.0, 0.2, 0.75, 1.0, ["obj", "env"])
def run(self, drag_on_land, angular_drag_on_land): self.start() self.communicate(TDWUtils.create_empty_room(40, 40)) # Create the avatar. self.communicate(TDWUtils.create_avatar(avatar_type="A_Simple_Body", position={"x": 0, "y": 1, "z": 0})) # When the y value of the avatar's position is this value, the avatar is on the ground. ground_y = 1 # When the y value of the avatar's position exceeds ground_y by this much, it is in the air. in_air_threshold = 0.05 # If true, the avatar is in the air. is_in_air = False # Set high default drag values. # Set a low mass. # Apply a force. # Request avatar data. resp = self.communicate([{"$type": "set_avatar_drag", "avatar_id": "a", "drag": 30, "angular_drag": 80}, {"$type": "send_avatars", "frequency": "always"}, {"$type": "apply_force_to_avatar", "avatar_id": "a", "direction": {"x": 0.3, "y": 0.7, "z": 0}, "magnitude": 900}]) for i in range(500): x, y, z = AvatarSimpleBody(resp[0]).get_position() # If the avatar just launched in to the air, set its drag values to 0. if y > ground_y + in_air_threshold and not is_in_air: is_in_air = True resp = self.communicate({"$type": "set_avatar_drag", "avatar_id": "a", "drag": 0, "angular_drag": 0}) print("The avatar is in the air on frame: " + str(Controller.get_frame(resp[-1]))) # If the avatar just landed on the ground, set drag values. elif y <= ground_y + in_air_threshold and is_in_air: resp = self.communicate({"$type": "set_avatar_drag", "avatar_id": "a", "drag": drag_on_land, "angular_drag": angular_drag_on_land}) is_in_air = False print("The avatar is on the ground on frame: " + str(Controller.get_frame(resp[-1]))) # Wait. else: resp = self.communicate({"$type": "do_nothing"})
def run(self): # Create the scene. # Set image encoding globals. self.start() commands = [TDWUtils.create_empty_room(12, 12), {"$type": "set_screen_size", "width": 128, "height": 128}, {"$type": "set_img_pass_encoding", "value": False}] # Create the avatar. commands.extend(TDWUtils.create_avatar(position={"x": 2.478, "y": 1.602, "z": 1.412}, look_at=TDWUtils.VECTOR3_ZERO, avatar_id="a")) # Enable all pass masks. Request an image for this frame only. commands.extend([{"$type": "set_pass_masks", "pass_masks": ["_img", "_id", "_category", "_mask", "_depth", "_normals", "_flow", "_depth_simple", "_albedo"], "avatar_id": "a"}, {"$type": "send_images", "ids": ["a"], "frequency": "once"}]) # Add objects. commands.append(self.get_add_object("small_table_green_marble", position=TDWUtils.VECTOR3_ZERO, rotation=TDWUtils.VECTOR3_ZERO, object_id=0)) commands.append(self.get_add_object("rh10", position={"x": 0.7, "y": 0, "z": 0.4}, rotation={"x": 0, "y": 30, "z": 0}, object_id=1)) commands.append(self.get_add_object("jug01", position={"x": -0.3, "y": 0.9, "z": 0.2}, rotation=TDWUtils.VECTOR3_ZERO, object_id=3)) commands.append(self.get_add_object("jug05", position={"x": 0.3, "y": 0.9, "z": -0.2}, rotation=TDWUtils.VECTOR3_ZERO, object_id=4)) # Send the commands. resp = self.communicate(commands) # Save the images. for r in resp[:-1]: r_id = OutputData.get_data_type_id(r) if r_id == "imag": # Save the images. TDWUtils.save_images(output_directory="dist", images=Images(r), filename="0") print(f"Images saved to: {Path('dist').resolve()}") # Stop the build. self.communicate({"$type": "terminate"})
def run(self): self.start() self.communicate(TDWUtils.create_empty_room(20, 20)) self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 3, "z": -6}, look_at=TDWUtils.VECTOR3_ZERO)) model_name = "rh10" z = -3 x = -1.5 print("With the add_object command (complex syntax but you have maximum control):") record = ModelLibrarian().get_record(model_name) self.communicate({"$type": "add_object", "name": model_name, "url": record.get_url(), "scale_factor": record.scale_factor, "position": {"x": x, "y": 0, "z": z}, "rotation": TDWUtils.VECTOR3_ZERO, "category": record.wcategory, "id": self.get_unique_id()}) x = 0 print("With the wrapper function Controller.add_object() " "(easy to use, but you can't add additional commands to this frame):") self.add_object(model_name=model_name, position={"x": x, "y": 0, "z": z}, rotation=TDWUtils.VECTOR3_ZERO, library="models_core.json") x = 1.5 print("With the wrapper function Controller.get_add_object() " "(harder to use, but you can add commands to this frame):") self.communicate(self.get_add_object(model_name=model_name, object_id=self.get_unique_id(), position={"x": x, "y": 0, "z": 0}, rotation=TDWUtils.VECTOR3_ZERO, library="models_core.json")) print("With the add_object command, minus all optional parameters (the model won't scale properly!):") self.communicate({"$type": "add_object", "name": model_name, "url": record.get_url(), "id": self.get_unique_id()}) print("With the wrapper function Controller.add_object(), minus all optional parameters:") self.add_object(model_name) print("With the wrapper function Controller.get_add_object(), minus all optional parameters:") self.communicate(self.get_add_object(model_name=model_name, object_id=self.get_unique_id()))
def run(self): # Open the image file. with open("DT4897.jpg", "rb") as f: image = b64encode(f.read()).decode("utf-8") # The expected dimensions of the painting. dimensions = {"x": 479, "y": 332} self.start() # Create an empty room. self.communicate(TDWUtils.create_empty_room(8, 8)) # Create a painting. # Apply the texture. painting_id = self.get_unique_id() painting_position = {"x": 1, "y": 2, "z": 3} self.communicate([{ "$type": "create_painting", "position": painting_position, "size": { "x": 5, "y": 3 }, "euler_angles": { "x": 0, "y": 30, "z": 0 }, "id": painting_id }, { "$type": "set_painting_texture", "id": painting_id, "dimensions": dimensions, "image": image }]) # Create the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 0, "y": 5, "z": 0 }, look_at=painting_position))
def run(self): self.start() self.communicate(TDWUtils.create_empty_room(12, 12)) o_id = self.add_object("rh10") self.communicate( TDWUtils.create_avatar(position={ "x": 1, "y": 3, "z": 0 })) self.communicate([{ "$type": "set_mass", "id": o_id, "mass": 15 }, { "$type": "rotate_object_by", "axis": "pitch", "id": o_id, "angle": 45 }, { "$type": "apply_force_magnitude_to_object", "id": o_id, "magnitude": 250.0 }]) for i in range(100): self.communicate({ "$type": "look_at", "avatar_id": "a", "object_id": o_id }) sleep(1) # Print the recorded commands to the console. for commands in self.record: print(commands) # Play back all of the commands. self.playback()
def init_scene(self, scene: str = None, layout: int = None, room: int = None, goal_room: int = None) -> ActionStatus: origin = np.array([0, 0, 0]) commands = [{ "$type": "load_scene", "scene_name": "ProcGenScene" }, TDWUtils.create_empty_room(12, 12)] self._add_container(model_name="basket_18inx18inx12iin", position={ "x": 0.354, "y": 0, "z": 0.549 }, rotation={ "x": 0, "y": -70, "z": 0 }) # Add target objects in a circle. num_objects = 10 d_theta = 360 / num_objects theta = d_theta / 2 pos = np.array([2, 0, 0]) for j in range(num_objects): object_position = TDWUtils.rotate_position_around(origin=origin, position=pos, angle=theta) self._add_target_object( "jug05", position=TDWUtils.array_to_vector3(object_position)) theta += d_theta commands.extend(self._get_scene_init_commands()) resp = self.communicate(commands) self._cache_static_data(resp=resp) # Wait for the Magnebot to reset to its neutral position. status = self._do_arm_motion() self._end_action() return status
def run(self): # Parse the substructure of fridge_large. record = ModelLibrarian().get_record("fridge_large") self.start() self.communicate(TDWUtils.create_empty_room(12, 12)) # Create the object. o_id = self.add_object("fridge_large") # Add an avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 0, "y": 1.886, "z": -0.15 }, look_at=TDWUtils.VECTOR3_ZERO)) # Disable post-processing, so the changes to the material aren't blurry. self.communicate({"$type": "set_post_process", "value": False}) for quality in ["low", "med", "high"]: # Set the visual materials. self.communicate( TDWUtils.set_visual_material(self, record.substructure, o_id, "parquet_long_horizontal_clean", quality=quality)) sleep(5) self.communicate({ "$type": "unload_asset_bundles", "bundle_type": "materials" }) self.communicate({ "$type": "unload_asset_bundles", "bundle_type": "models" })
def sound20k_set(self) -> None: """ Generate a dataset analogous to Sound20K. """ sound20k_init_commands = [ { "$type": "load_scene" }, TDWUtils.create_empty_room(12, 12), { "$type": "set_proc_gen_walls_scale", "walls": TDWUtils.get_box(12, 12), "scale": { "x": 1, "y": 4, "z": 1 } }, { "$type": "set_reverb_space_simple", "env_id": 0, "reverb_floor_material": "parquet", "reverb_ceiling_material": "acousticTile", "reverb_front_wall_material": "smoothPlaster", "reverb_back_wall_material": "smoothPlaster", "reverb_left_wall_material": "smoothPlaster", "reverb_right_wall_material": "smoothPlaster" }, { "$type": "create_avatar", "type": "A_Img_Caps_Kinematic", "id": "a" }, { "$type": "add_environ_audio_sensor" }, { "$type": "toggle_image_sensor" } ] self.process_sub_set("Sound20K", "models_per_material_sound20k", sound20k_init_commands, get_sound20k_scenes())
from tdw.output_data import CompositeObjects, SegmentationColors from sticky_mitten_avatar.util import get_data """ Get default audio data for each sub-object of a composite object. """ if __name__ == "__main__": from argparse import ArgumentParser parser = ArgumentParser() parser.add_argument("--model", default="puzzle_box_composite") args = parser.parse_args() c = Controller() c.start() resp = c.communicate([ TDWUtils.create_empty_room(12, 12), c.get_add_object(model_name=args.model, object_id=0), { "$type": "send_composite_objects" }, { "$type": "send_segmentation_colors" } ]) c.communicate({"$type": "terminate"}) # Get the name of each object. colors = get_data(resp=resp, d_type=SegmentationColors) names = dict() for i in range(colors.get_num()): names[colors.get_object_id(i)] = colors.get_object_name(i) # Parse the composite object data.
def get_scene_initialization_commands(self) -> List[dict]: return [{ "$type": "load_scene", "scene_name": "ProcGenScene" }, TDWUtils.create_empty_room(12, 12)]
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 600, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("parquet_wood_mahogany") self.communicate({ "$type": "add_material", "name": "parquet_wood_mahogany", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "parquet_wood_mahogany" }) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": -1, "y": 1.4, "z": -1 }, look_at=TDWUtils.array_to_vector3( [-0.1, 0.7, -0.1]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table_id = self.add_object( model_name= "b03_restoration_hardware_pedestal_salvaged_round_tables", position={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="club_sandwich", position={ "x": 0.45, "y": top[1], "z": 0.25 }, rotation={ "x": 0, "y": 30, "z": 0 }, library="models_full.json") self.add_object(model_name="coffeemug", position={ "x": 0.03, "y": top[1], "z": 0.03 }, library="models_full.json") self.add_object(model_name="knife3", position={ "x": -0.17, "y": top[1], "z": 0.15 }, rotation={ "x": 0, "y": 24, "z": 90 }, library="models_full.json") self.add_object(model_name="notes_02", position={ "x": 0.17, "y": top[1], "z": -0.12 }, rotation={ "x": 0, "y": 24, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([-0.1, 0.7, -0.1]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "sadnwich2", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior" )
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 640, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("laser_cut_white_norway_spruce") self.communicate({ "$type": "add_material", "name": "laser_cut_white_norway_spruce", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "laser_cut_white_norway_spruce" }) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 1.5, "y": 1.6, "z": 0 }, look_at=TDWUtils.array_to_vector3( [0, 0.5, 0]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table = self.add_object( model_name="boconcept_lugo_dining_table_vray1.5", position={ "x": 0, "y": 0, "z": 0.0 }, rotation={ "x": 0, "y": -90, "z": 0 }, library="models_full.json") table_bounds = self.get_bounds_data(table) top = table_bounds.get_top(0) self.add_object(model_name='macbook_001', position={ "x": 0, "y": top[1], "z": 0.1 }, rotation={ "x": 0, "y": 90, "z": 0 }, library="models_full.json") self.add_object(model_name='spunlight_designermesh_lamp', position={ "x": -0.19, "y": top[1], "z": -0.4 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") self.add_object(model_name="apple_magic_mouse_2_(2015)_vray", position={ "x": 0, "y": top[1], "z": 0.32 }, rotation={ "x": 0, "y": 86, "z": 0 }, library="models_full.json") self.add_object(model_name="f10_apple_iphone_4", position={ "x": 0.14, "y": top[1], "z": -0.3 }, rotation={ "x": 0, "y": 12, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([0, 0.5, 0]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "phone", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior" )
def run(self): self.start() # init_setup_commands = [{"$type": "set_screen_size", # "width": 600, # "height": 480}, # {"$type": "set_render_quality", # "render_quality": 5}] # self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 2.5, "y": 1.5, "z": 0.3 }, look_at=TDWUtils.array_to_vector3( [3, 0.5, 0.5]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("gravel_white") self.communicate({ "$type": "add_material", "name": "gravel_white", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "gravel_white" }) # self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"}) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) bench = self.add_object(model_name="b04_wood_metal_park_bench", position={ "x": 3, "y": 0, "z": 0.5 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") bench_bounds = self.get_bounds_data(bench) top = bench_bounds.get_top(0) self.add_object(model_name="b05_baseball_bat_01", position={ "x": 2.8, "y": top[1] - 0.35, "z": 0 }, rotation={ "x": 0, "y": 90, "z": 90 }, library="models_full.json") self.add_object(model_name="b05_baseballnew_v03_12", position={ "x": 2.9, "y": top[1] - 0.35, "z": 0.4 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([3, 0.5, 0.5]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "bench_ball", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior" )
def trial(self): """ Select random objects and collide them to produce impact sounds. """ p = PyImpact(initial_amp=0.25) # Set the environment audio materials. floor = AudioMaterial.ceramic wall = AudioMaterial.wood # Create an empty room. # Set the screen size. # Adjust framerate so that physics is closer to realtime. commands = [ TDWUtils.create_empty_room(12, 12), { "$type": "destroy_all_objects" }, { "$type": "set_screen_size", "width": 1024, "height": 1024 }, { "$type": "set_target_framerate", "framerate": 100 } ] # Create the avatar. commands.extend( TDWUtils.create_avatar(avatar_type="A_Img_Caps_Kinematic", position={ "x": 1, "y": 1.2, "z": 1.2 }, look_at=TDWUtils.VECTOR3_ZERO)) # Add the audio sensor. commands.extend([{ "$type": "add_audio_sensor", "avatar_id": "a" }, { "$type": "set_focus_distance", "focus_distance": 2 }]) # Select a random pair of objects. obj1_names = [ "trapezoidal_table", "glass_table_round", "yellow_side_chair", "table_square", "marble_table" ] obj2_names = ["vase_06", "spoon1", "glass3", "jug02"] obj1_name = random.choice(obj1_names) obj2_name = random.choice(obj2_names) # Get initialization data from the default audio data (which includes mass, friction values, etc.). obj1_init_data = AudioInitData(name=obj1_name) obj2_init_data = AudioInitData(name=obj2_name, position={ "x": 0, "y": 2, "z": 0 }, rotation={ "x": 135, "y": 0, "z": 30 }) # Convert the initialization data to commands. obj1_id, obj1_commands = obj1_init_data.get_commands() obj2_id, obj2_commands = obj2_init_data.get_commands() # Cache the IDs and names of each object for PyImpact. object_names = {obj1_id: obj1_name, obj2_id: obj2_name} p.set_default_audio_info(object_names=object_names) # Add the objects. commands.extend(obj1_commands) commands.extend(obj2_commands) # Apply a small force to the dropped object. # Request collision and rigidbody output data. commands.extend([{ "$type": "apply_force_to_object", "force": { "x": 0, "y": -0.01, "z": 0 }, "id": obj2_id }, { "$type": "send_collisions", "enter": True, "stay": False, "exit": False, "collision_types": ["obj", "env"] }, { "$type": "send_rigidbodies", "frequency": "always", "ids": [obj2_id, obj1_id] }]) # Send all of the commands. resp = self.communicate(commands) # Iterate through 200 frames. # Every frame, listen for collisions, and parse the output data. for i in range(200): # Use PyImpact to generate audio from the output data and then convert the audio to TDW commands. # If no audio is generated, then `commands` is an empty list. commands = p.get_audio_commands(resp=resp, floor=floor, wall=wall) # Send the commands to TDW in order to play the audio. resp = self.communicate(commands)
def run(self): rng = np.random.RandomState(0) self.model_librarian = ModelLibrarian("models_special.json") self.start() commands = [TDWUtils.create_empty_room(100, 100)] # The starting height of the objects. y = 10 # The radius of the circle of objects. r = 7.0 # The mass of each object. mass = 5 # Get all points within the circle defined by the radius. p0 = np.array((0, 0)) o_id = 0 for x in np.arange(-r, r, 1): for z in np.arange(-r, r, 1): p1 = np.array((x, z)) dist = np.linalg.norm(p0 - p1) if dist < r: # Add an object. # Set its mass, physics properties, and color. commands.extend([self.get_add_object("prim_cone", object_id=o_id, position={"x": x, "y": y, "z": z}, rotation={"x": 0, "y": 0, "z": 180}), {"$type": "set_mass", "id": o_id, "mass": mass}, {"$type": "set_physic_material", "dynamic_friction": 0.8, "static_friction": 0.7, "bounciness": 0.5, "id": o_id}, {"$type": "set_color", "color": {"r": rng.random_sample(), "g": rng.random_sample(), "b": rng.random_sample(), "a": 1.0}, "id": o_id}]) o_id += 1 # Request transforms per frame. commands.extend([{"$type": "send_transforms", "frequency": "always"}]) # Create an avatar to observe the grisly spectacle. avatar_position = {"x": -20, "y": 8, "z": 18} commands.extend(TDWUtils.create_avatar(position=avatar_position, look_at=TDWUtils.VECTOR3_ZERO)) commands.append({"$type": "set_focus_distance", "focus_distance": TDWUtils.get_distance(avatar_position, TDWUtils.VECTOR3_ZERO)}) resp = self.communicate(commands) # If an objects are this far away from (0, 0, 0) the forcefield "activates". forcefield_radius = 5 # The forcefield will bounce objects away at this force. forcefield_force = -10 zeros = np.array((0, 0, 0)) for i in range(1000): transforms = Transforms(resp[0]) commands = [] for j in range(transforms.get_num()): pos = transforms.get_position(j) pos = np.array(pos) # If the object is in the forcefield, apply a force. if TDWUtils.get_distance(TDWUtils.array_to_vector3(pos), TDWUtils.VECTOR3_ZERO) <= forcefield_radius: # Get the normalized directional vector and multiply it by the force magnitude. d = zeros - pos d = d / np.linalg.norm(d) d = d * forcefield_force commands.append({"$type": "apply_force_to_object", "id": transforms.get_id(j), "force": TDWUtils.array_to_vector3(d)}) resp = self.communicate(commands)