def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(10, 10)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) self.communicate(self.get_add_hdri_skybox("autumn_hockey_4k")) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": 3, "y": 1.3, "z": 0}, look_at=TDWUtils.array_to_vector3([0, 0.5, 0]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("grass_countryside") print(record) self.communicate({"$type": "add_material", "name": "grass_countryside", "url": record.get_url()}) self.communicate({"$type": "set_proc_gen_floor_material", "name": "grass_countryside"}) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) self.add_object(model_name="b05_clochild4", position={"x": 0, "y": 0, "z": 0}, rotation={"x": 0, "y": 30, "z": 0}, library="models_full.json") self.add_object(model_name="baseballbat", position={"x": 0.5, "y": 0, "z": 0.3}, rotation={"x": 0, "y": 30, "z": 0}, library="models_full.json") self.add_object(model_name="base-ball", position={"x": 0.4, "y": 0, "z": 0.22}, rotation={"x": 0, "y": 30, "z": 10}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([0, 0.5, 0])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "kid_baseball", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def _get_object_position(object_positions: List[ObjectPosition], max_tries: int = 1000, radius: float = 2) -> \ Dict[str, float]: """ Try to get a valid random position that doesn't interpentrate with other objects. :param object_positions: The positions and radii of all objects so far that will be added to the scene. :param max_tries: Try this many times to get a valid position before giving up. :param radius: The radius to pick a position in. :return: A valid position that doesn't interpentrate with other objects. """ o_pos = TDWUtils.array_to_vector3( TDWUtils.get_random_point_in_circle(center=np.array([0, 0, 0]), radius=radius)) # Pick a position away from other objects. ok = False count = 0 while not ok and count < max_tries: count += 1 ok = True for o in object_positions: # If the object is too close to another object, try another position. if TDWUtils.get_distance(o.position, o_pos) <= o.radius: ok = False o_pos = TDWUtils.array_to_vector3( TDWUtils.get_random_point_in_circle(center=np.array( [0, 0, 0]), radius=radius)) return o_pos
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(32, 32)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": -1.5, "y": 1.5, "z": -15}, look_at=TDWUtils.array_to_vector3([-1.7, 0.7, -2.3]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("concrete_chipped_cracked") self.communicate({"$type": "add_material", "name": "concrete_chipped_cracked", "url": record.get_url()}) self.communicate({"$type": "set_proc_gen_floor_material", "name": "concrete_chipped_cracked"}) # self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"}) self.communicate({"$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar"}) bus = self.add_object(model_name="b06_bus_new", position={"x": -2, "y": 0, "z": 0}, rotation={"x": 0, "y": -70, "z": 0}, library="models_full.json") bus_bounds = self.get_bounds_data(bus) bus_front = bus_bounds.get_front(0) print(bus_front) self.add_object(model_name="3dscan_man_004", position={"x": -3.7, "y": 0, "z": -7.3}, rotation={"x": 0, "y": 0, "z": 0}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([-1.7, 0.7, -2.3])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "bus", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 1280, "height": 962}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.load_streamed_scene(scene="tdw_room_2018") # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 1, "z": 0}, look_at=TDWUtils.array_to_vector3([2, 0.8, -1.5]), avatar_id="avatar")) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) table_id = self.add_object(model_name="glass_table", position={"x": 2, "y": 0, "z": -1.5}, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="flower_vase_set_2", position={"x": 1.6, "y": top[1], "z": -1.5}, library="models_full.json") self.add_object(model_name="flower_vase_set_2", position={"x": 2, "y": top[1], "z": -1.5}, library="models_full.json") self.add_object(model_name="flower_vase_set_2", position={"x": 2.4, "y": top[1], "z": -1.5}, library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([2, 0.8, -1.5])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "flowers_row", output_directory="replicated_images")
def push_into_other(self) -> List[dict]: """ :return: A list of commands to push one object into another object. """ # Get two points some distance from each other. p0 = np.array([0, 0, 0]) p1 = np.array([0, 0, 0]) center = np.array([0, 0, 0]) count = 0 while count < 1000 and np.linalg.norm(np.abs(p1 - p0)) < 1.5: p0 = TDWUtils.get_random_point_in_circle(center=center, radius=2) p1 = TDWUtils.get_random_point_in_circle(center=center, radius=2) count += 1 p0 = TDWUtils.array_to_vector3(p0) p0["y"] = 0.2 p1 = TDWUtils.array_to_vector3(p1) p1["y"] = 0.1 # Get commands to create the first object and push it. commands = self._push(position=p0, target=p1, force_mag=random.uniform(1000, 2000)) # Add commands for the second object. second_object_commands, s_id = self._get_squishable(position=p1, rotation={ "x": 0, "y": random.uniform( 0, 360), "z": 0 }) commands.extend(second_object_commands) # Set the avatar. # Get the midpoint between the objects and use this to place the avatar. p_med = TDWUtils.array_to_vector3( np.array([(p0["x"] + p1["x"]) / 2, 0, (p0["z"] + p1["z"]) / 2])) commands.extend( self._set_avatar(a_pos=self.get_random_avatar_position( radius_min=1.8, radius_max=2.3, y_min=1, y_max=1.75, center=p_med), cam_aim=p1)) return commands
def _push(self, position: Dict[str, float], target: Dict[str, float], force_mag: float) -> List[dict]: """ :param position: The initial position. :param target: The target position (used for the force vector). :param force_mag: The force magnitutde. :return: A list of commands to push the object along the floor. """ commands, soft_id = self._get_squishable(position=position, rotation={ "x": 0, "y": random.uniform(0, 360), "z": 0 }) # Get a force vector towards the target. p0 = TDWUtils.vector3_to_array(position) p1 = TDWUtils.vector3_to_array(target) force = ((p1 - p0) / (np.linalg.norm(p1 - p0))) * force_mag # Add the force. commands.append({ "$type": "apply_force_to_flex_object", "force": TDWUtils.array_to_vector3(force), "id": soft_id }) return commands
def get_falling_commands(self, mass: float = 3) -> List[List[dict]]: """ :param mass: Objects with <= this mass might receive a force. :return: A list of lists; per-frame commands to make small objects fly up. """ per_frame_commands: List[List[dict]] = [] # Get a list of all small objects. small_ids = self.get_objects_by_mass(mass) random.shuffle(small_ids) max_num_objects = len(small_ids) if len(small_ids) < 8 else 8 min_num_objects = max_num_objects - 3 if min_num_objects <= 0: min_num_objects = 1 # Add some objects. for i in range(random.randint(min_num_objects, max_num_objects)): o_id = small_ids.pop(0) force_dir = np.array([random.uniform(-0.125, 0.125), random.uniform(0.7, 1), random.uniform(-0.125, 0.125)]) force_dir = force_dir / np.linalg.norm(force_dir) min_force = self.physics_info[o_id].mass * 2 max_force = self.physics_info[o_id].mass * 4 force = TDWUtils.array_to_vector3(force_dir * random.uniform(min_force, max_force)) per_frame_commands.append([{"$type": "apply_force_to_object", "force": force, "id": o_id}]) # Wait some frames. for j in range(10, 30): per_frame_commands.append([]) return per_frame_commands
def run(self): self.start() width = 12 length = 12 # Create an empty room. self.communicate(TDWUtils.create_empty_room(width, length)) # Disable physics. self.communicate({"$type": "simulate_physics", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": -5, "y": 10, "z": -2 }, look_at=TDWUtils.VECTOR3_ZERO)) # Create the dining table. self.dining_table() # Create the sofa. self.sofa(width, length) # Bake the NavMesh. self.communicate({"$type": "bake_nav_mesh", "carve_type": "none"}) # Get a random position on the NavMesh. position = TDWUtils.get_random_position_on_nav_mesh( self, width, length) position = TDWUtils.array_to_vector3(position) # Add a side table. self.side_table(position)
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 1280, "height": 962 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) # Create an empty room. self.load_streamed_scene(scene="tdw_room_2018") # Disable physics. self.communicate({"$type": "simulate_physics", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 0.9, "y": 1.5, "z": 0.9 }, look_at=TDWUtils.array_to_vector3( [0, 0.8, 0]), avatar_id="avatar")) # Create the dining table. self.dining_table()
def go_to_start(self) -> None: """ Navigate back to the start position. """ path = np.flip(Demo.PATH[:-1], axis=0) for waypoint in path: self.move_to(target=TDWUtils.array_to_vector3(waypoint))
def get_p_1(self) -> Dict[str, float]: """ :return: A position in the c_1 sub-sector. """ return TDWUtils.array_to_vector3( TDWUtils.get_random_point_in_circle(center=self.c_1, radius=self.RADIUS))
def is_done(self, resp: List[bytes], frame: int) -> bool: for r in resp[:-1]: r_id = OutputData.get_data_type_id(r) # If the ball reaches or overshoots the destination, the trial is done. if r_id == "tran": t = Transforms(r) d0 = TDWUtils.get_distance( TDWUtils.array_to_vector3(t.get_position(0)), self._p0) d1 = TDWUtils.get_distance(self._p0, self._p1) return d0 > d1 * 1.5 return False
def side_table(self, position, parent=None): """ 1. Create a side table. 2. Maybe create an object on the side table. :param position: The position of the table (Vector3). :param parent: THe parent parameters: (position, rotation). """ # Create the table. table_id = self.add_object(random.choice( ProcGenInteriorDesign.SIDE_TABLE), position=position) if parent: # Place the object to the side of the parent and match its rotation. parent_position = parent[0] parent_rotation = parent[1] self.side_of(table_id, parent_position, parent_rotation) else: # Apply a random rotation. self.communicate({ "$type": "rotate_object_by", "angle": random.uniform(-45, 45), "axis": "yaw", "id": table_id, "is_world": True }) # Maybe put an object on the table. if random.random() > 0.25: # Get the top position of the table. table_bounds = self.get_bounds_data(table_id) on_pos = table_bounds.get_top(0) on_y = on_pos[1] # Perturb the position slightly. on_pos = TDWUtils.get_random_point_in_circle( np.array(on_pos), 0.125) on_pos[1] = on_y on_pos = TDWUtils.array_to_vector3(on_pos) on_rot = {"x": 0, "y": random.uniform(-45, 45), "z": 0} # Add the object. self.add_object(random.choice(ProcGenInteriorDesign.SIDE_TABLE_ON), position=on_pos, rotation=on_rot)
def get_move_along_direction(pos: Dict[str, float], target: Dict[str, float], d: float, noise: float = 0) -> \ Dict[str, float]: """ :param pos: The object's position. :param target: The target position. :param d: The distance to teleport. :param noise: Add a little noise to the teleport. :return: A position from pos by distance d along a directional vector defined by pos, target. """ direction = TDWUtils.array_to_vector3( (TDWUtils.vector3_to_array(target) - TDWUtils.vector3_to_array(pos)) / TDWUtils.get_distance(pos, target)) return { "x": pos["x"] + direction["x"] * d + random.uniform(-noise, noise), "y": pos["y"], "z": pos["z"] + direction["z"] * d + random.uniform(-noise, noise) }
def init_scene(self, scene: str = None, layout: int = None, room: int = None, goal_room: int = None) -> ActionStatus: origin = np.array([0, 0, 0]) commands = [{ "$type": "load_scene", "scene_name": "ProcGenScene" }, TDWUtils.create_empty_room(12, 12)] self._add_container(model_name="basket_18inx18inx12iin", position={ "x": 0.354, "y": 0, "z": 0.549 }, rotation={ "x": 0, "y": -70, "z": 0 }) # Add target objects in a circle. num_objects = 10 d_theta = 360 / num_objects theta = d_theta / 2 pos = np.array([2, 0, 0]) for j in range(num_objects): object_position = TDWUtils.rotate_position_around(origin=origin, position=pos, angle=theta) self._add_target_object( "jug05", position=TDWUtils.array_to_vector3(object_position)) theta += d_theta commands.extend(self._get_scene_init_commands()) resp = self.communicate(commands) self._cache_static_data(resp=resp) # Wait for the Magnebot to reset to its neutral position. status = self._do_arm_motion() self._end_action() return status
def transport(self) -> None: """ Transport some objects to the other room. """ for i in range(4): # Get the closest object that still needs to be transported. self._to_transport = list(sorted(self._to_transport, key=lambda x: np.linalg.norm( self.state.object_transforms[x].position - self.state.magnebot_transform.position))) object_id = self._to_transport[0] # Go to the object and pick it up. self.move_to(target=object_id) self.pick_up(target=object_id, arm=Arm.right) # Put the object in the container. self.put_in() # Record this object as done. self._to_transport = self._to_transport[1:] # Follow the path to the other room. path = Demo.PATH[1:] for waypoint in path: self.move_to(target=TDWUtils.array_to_vector3(waypoint)) self.pour_out()
def reach_for_target(self, arm: Arm, target: np.array, stop_on_mitten_collision: bool, target_orientation: np.array = None, precision: float = 0.05) -> List[dict]: """ Get an IK solution to move a mitten to a target position. :param arm: The arm (left or right). :param target: The target position for the mitten. :param target_orientation: Target IK orientation. Usually you should leave this as None (the default). :param stop_on_mitten_collision: If true, stop moving when the mitten collides with something. :param precision: The distance threshold to the target position. :return: A list of commands to begin bending the arm. """ # Get the IK solution. rotations, ik_target = self._get_ik( target=target, arm=arm, target_orientation=target_orientation) angle = TDWUtils.get_angle_between(v1=FORWARD, v2=self.frame.get_forward()) target = TDWUtils.rotate_position_around( position=ik_target, angle=angle) + self.frame.get_position() rotation_targets = dict() for c, r in zip(self._arms[arm].links[1:-1], rotations[1:-1]): rotation_targets[c.name] = r self._ik_goals[arm] = _IKGoal( target=target, stop_on_mitten_collision=stop_on_mitten_collision, rotations=rotation_targets, precision=precision) commands = [self.get_start_bend_sticky_mitten_profile(arm=arm)] # If the avatar is holding something, strengthen the wrist. held = self.frame.get_held_left( ) if arm == Arm.left else self.frame.get_held_right() if len(held) > 0: commands.append({ "$type": "set_joint_angular_drag", "joint": f"wrist_{arm.name}", "axis": "roll", "angular_drag": 50, "avatar_id": self.id }) if self._debug: print([np.rad2deg(r) for r in rotations]) self._plot_ik(target=ik_target, arm=arm) # Show the target. commands.extend([{ "$type": "remove_position_markers" }, { "$type": "add_position_marker", "position": TDWUtils.array_to_vector3(target) }]) a = arm.name for c in self._ik_goals[arm].rotations: j = c.split("_") # Apply the motion. commands.extend([{ "$type": "bend_arm_joint_to", "angle": np.rad2deg(self._ik_goals[arm].rotations[c]), "joint": f"{j[0]}_{a}", "axis": j[1], "avatar_id": self.id }]) return commands
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 600, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("parquet_wood_mahogany") self.communicate({ "$type": "add_material", "name": "parquet_wood_mahogany", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "parquet_wood_mahogany" }) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": -1, "y": 1.4, "z": -1 }, look_at=TDWUtils.array_to_vector3( [-0.1, 0.7, -0.1]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table_id = self.add_object( model_name= "b03_restoration_hardware_pedestal_salvaged_round_tables", position={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="club_sandwich", position={ "x": 0.45, "y": top[1], "z": 0.25 }, rotation={ "x": 0, "y": 30, "z": 0 }, library="models_full.json") self.add_object(model_name="coffeemug", position={ "x": 0.03, "y": top[1], "z": 0.03 }, library="models_full.json") self.add_object(model_name="knife3", position={ "x": -0.17, "y": top[1], "z": 0.15 }, rotation={ "x": 0, "y": 24, "z": 90 }, library="models_full.json") self.add_object(model_name="notes_02", position={ "x": 0.17, "y": top[1], "z": -0.12 }, rotation={ "x": 0, "y": 24, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([-0.1, 0.7, -0.1]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "sadnwich2", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior" )
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 640, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("laser_cut_white_norway_spruce") self.communicate({ "$type": "add_material", "name": "laser_cut_white_norway_spruce", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "laser_cut_white_norway_spruce" }) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 1.5, "y": 1.6, "z": 0 }, look_at=TDWUtils.array_to_vector3( [0, 0.5, 0]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table = self.add_object( model_name="boconcept_lugo_dining_table_vray1.5", position={ "x": 0, "y": 0, "z": 0.0 }, rotation={ "x": 0, "y": -90, "z": 0 }, library="models_full.json") table_bounds = self.get_bounds_data(table) top = table_bounds.get_top(0) self.add_object(model_name='macbook_001', position={ "x": 0, "y": top[1], "z": 0.1 }, rotation={ "x": 0, "y": 90, "z": 0 }, library="models_full.json") self.add_object(model_name='spunlight_designermesh_lamp', position={ "x": -0.19, "y": top[1], "z": -0.4 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") self.add_object(model_name="apple_magic_mouse_2_(2015)_vray", position={ "x": 0, "y": top[1], "z": 0.32 }, rotation={ "x": 0, "y": 86, "z": 0 }, library="models_full.json") self.add_object(model_name="f10_apple_iphone_4", position={ "x": 0.14, "y": top[1], "z": -0.3 }, rotation={ "x": 0, "y": 12, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([0, 0.5, 0]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "phone", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior" )
def run(self): self.start() # init_setup_commands = [{"$type": "set_screen_size", # "width": 600, # "height": 480}, # {"$type": "set_render_quality", # "render_quality": 5}] # self.communicate(init_setup_commands) # Create an empty room. self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 2.5, "y": 1.5, "z": 0.3 }, look_at=TDWUtils.array_to_vector3( [3, 0.5, 0.5]), avatar_id="avatar")) lib = MaterialLibrarian(library="materials_med.json") record = lib.get_record("gravel_white") self.communicate({ "$type": "add_material", "name": "gravel_white", "url": record.get_url() }) self.communicate({ "$type": "set_proc_gen_floor_material", "name": "gravel_white" }) # self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"}) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) bench = self.add_object(model_name="b04_wood_metal_park_bench", position={ "x": 3, "y": 0, "z": 0.5 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") bench_bounds = self.get_bounds_data(bench) top = bench_bounds.get_top(0) self.add_object(model_name="b05_baseball_bat_01", position={ "x": 2.8, "y": top[1] - 0.35, "z": 0 }, rotation={ "x": 0, "y": 90, "z": 90 }, library="models_full.json") self.add_object(model_name="b05_baseballnew_v03_12", position={ "x": 2.9, "y": top[1] - 0.35, "z": 0.4 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([3, 0.5, 0.5]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "bench_ball", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior" )
def sofa(self, width, length): """ 1. Create a sofa in the corner in the room. 2. Create some objects in front of the sofa. 3. Create some objects the sides of the sofa. :param width: The width of the room. :param length: The length of the room. """ # Add the sofa in the corner of the room. sofa_position = {"x": -(width / 2 - 2), "y": 0, "z": length / 2 - 2} sofa_id = self.add_object(random.choice(ProcGenInteriorDesign.SOFA), position=sofa_position) # Rotate the sofa to face the center of the room. self.communicate({ "$type": "object_look_at_position", "id": sofa_id, "position": TDWUtils.VECTOR3_ZERO }) sofa_bounds = self.get_bounds_data(sofa_id) # Set the rotation to match the sofa. sofa_transform = self.get_transforms_data([sofa_id]) sofa_rotation = sofa_transform.get_rotation(0) sofa_rotation = { "x": sofa_rotation[0], "y": sofa_rotation[1], "z": sofa_rotation[2], "w": sofa_rotation[3] } # Get a focal point for objects to place in front of the sofa. centerpoint = TDWUtils.extend_line(np.array(sofa_bounds.get_center(0)), np.array(sofa_bounds.get_front(0)), 0.5) centerpoint[1] = 0 # Place objects in front of the sofa at various positions near the centerpoint. for i in range(random.randint(1, 3)): pos = TDWUtils.get_random_point_in_circle(centerpoint, 0.125) pos = TDWUtils.array_to_vector3(pos) self.add_object(random.choice(ProcGenInteriorDesign.SOFA_FRONT_OF), position=pos, rotation={ "x": 0, "y": random.uniform(-30, 30), "z": 0 }) # Create 1-2 objects to the sides of the sofa. positions = [sofa_bounds.get_left(0), sofa_bounds.get_right(0)] random.shuffle(positions) if random.random() > 0.5: positions.pop(0) for side_position in positions: side_position = [side_position[0], 0, side_position[2]] side_position = TDWUtils.array_to_vector3(side_position) # Create a side table. if random.random() > 0.5: self.side_table(side_position, (sofa_position, sofa_rotation)) # Create some other object. else: # Create the object. side_id = self.add_object(random.choice( ProcGenInteriorDesign.SOFA_SIDE_OF), position=side_position) self.side_of(side_id, sofa_position, sofa_rotation)
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 1280, "height": 962 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) width = 8 length = 8 # Create an empty room. self.communicate(TDWUtils.create_empty_room(width, length)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 3, "y": 1.7, "z": 2 }, look_at={ "x": 0, "y": 0.8, "z": 2 }, avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) self.add_object(model_name="b05_bed_giorgetti_rea", position={ "x": 0.3, "y": 0, "z": 2.2 }, rotation={ "x": 0, "y": 180, "z": 0 }, library="models_full.json") # Create the dining table. # self.dining_table() # Create the sofa. self.sofa(width, length) # Bake the NavMesh. self.communicate({"$type": "bake_nav_mesh"}) print('mesh') # Get a random position on the NavMesh. position = TDWUtils.get_random_position_on_nav_mesh( self, width, length) position = TDWUtils.array_to_vector3(position) # Add a side table. self.side_table(position) position = TDWUtils.get_random_position_on_nav_mesh( self, width, length) position = TDWUtils.array_to_vector3(position) self.add_object(model_name="648972_chair_poliform_harmony", position=position, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": { "x": 0, "y": 0.8, "z": 2 } }) images = Images(scene_data[0]) TDWUtils.save_images(images, "bedroom1", output_directory="replicated_images")
def run(self): rng = np.random.RandomState(0) self.model_librarian = ModelLibrarian("models_special.json") self.start() commands = [TDWUtils.create_empty_room(100, 100)] # The starting height of the objects. y = 10 # The radius of the circle of objects. r = 7.0 # The mass of each object. mass = 5 # Get all points within the circle defined by the radius. p0 = np.array((0, 0)) o_id = 0 for x in np.arange(-r, r, 1): for z in np.arange(-r, r, 1): p1 = np.array((x, z)) dist = np.linalg.norm(p0 - p1) if dist < r: # Add an object. # Set its mass, physics properties, and color. commands.extend([self.get_add_object("prim_cone", object_id=o_id, position={"x": x, "y": y, "z": z}, rotation={"x": 0, "y": 0, "z": 180}), {"$type": "set_mass", "id": o_id, "mass": mass}, {"$type": "set_physic_material", "dynamic_friction": 0.8, "static_friction": 0.7, "bounciness": 0.5, "id": o_id}, {"$type": "set_color", "color": {"r": rng.random_sample(), "g": rng.random_sample(), "b": rng.random_sample(), "a": 1.0}, "id": o_id}]) o_id += 1 # Request transforms per frame. commands.extend([{"$type": "send_transforms", "frequency": "always"}]) # Create an avatar to observe the grisly spectacle. avatar_position = {"x": -20, "y": 8, "z": 18} commands.extend(TDWUtils.create_avatar(position=avatar_position, look_at=TDWUtils.VECTOR3_ZERO)) commands.append({"$type": "set_focus_distance", "focus_distance": TDWUtils.get_distance(avatar_position, TDWUtils.VECTOR3_ZERO)}) resp = self.communicate(commands) # If an objects are this far away from (0, 0, 0) the forcefield "activates". forcefield_radius = 5 # The forcefield will bounce objects away at this force. forcefield_force = -10 zeros = np.array((0, 0, 0)) for i in range(1000): transforms = Transforms(resp[0]) commands = [] for j in range(transforms.get_num()): pos = transforms.get_position(j) pos = np.array(pos) # If the object is in the forcefield, apply a force. if TDWUtils.get_distance(TDWUtils.array_to_vector3(pos), TDWUtils.VECTOR3_ZERO) <= forcefield_radius: # Get the normalized directional vector and multiply it by the force magnitude. d = zeros - pos d = d / np.linalg.norm(d) d = d * forcefield_force commands.append({"$type": "apply_force_to_object", "id": transforms.get_id(j), "force": TDWUtils.array_to_vector3(d)}) resp = self.communicate(commands)
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 1280, "height": 962 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) # Create an empty room. self.load_streamed_scene(scene="tdw_room_2018") # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": -2, "y": 1.5, "z": 0 }, look_at=TDWUtils.array_to_vector3( [4, 0.8, 0]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) self.add_object(model_name="b05_c-5_classic_series_bench-2012", position={ "x": 3, "y": 0, "z": -1.5 }, library="models_full.json") self.add_object(model_name="b05_c-5_classic_series_bench-2012", position={ "x": 3, "y": 0, "z": 1.5 }, library="models_full.json") self.add_object(model_name="baleri_italia_folded_black_table", position={ "x": 2, "y": 0, "z": -1.5 }, rotation=TDWUtils.array_to_vector3([0, 90, 0]), library="models_full.json") self.add_object(model_name="baleri_italia_folded_black_table", position={ "x": 2, "y": 0, "z": 1.5 }, rotation=TDWUtils.array_to_vector3([0, 90, 0]), library="models_full.json") self.add_object(model_name="linen_dining_chair", position={ "x": 1, "y": 0, "z": -1.5 }, rotation=TDWUtils.array_to_vector3([0, 90, 0]), library="models_full.json") self.add_object(model_name="linen_dining_chair", position={ "x": 1, "y": 0, "z": 1.5 }, rotation=TDWUtils.array_to_vector3([0, 90, 0]), library="models_full.json") # self.communicate({"$type": "rotate_object_by", # "angle": 180, # "axis": "yaw", # "id": chair_id, # "is_world": True}) # record = ModelLibrarian(library='models_full.json').get_record("b05_tv1970") # self.communicate({"$type": "add_object", # "name": "b05_tv1970", # "url": record.get_url(), # "scale_factor": 30, # "position": {"x": 0, "y": 0, "z": 3.5}, # "category": record.wcategory, # "id": self.get_unique_id()}) # # # side_table = self.add_object(model_name="side_table_wood", # position={"x": 1.5, "y": 0, "z": 4}, # library="models_full.json") # side_table_bounds = self.get_bounds_data(side_table) # # top = side_table_bounds.get_top(0) # self.add_object(model_name="acacia_table_lamp_jamie_young", # position={"x": 1.5, "y": top[1], "z": 4}, # library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([4, 0.8, 0]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "two_tables_v2", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images" )
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 640, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) self.communicate(TDWUtils.create_empty_room(8, 8)) # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": 2.0, "y": 0.8, "z": -0.55 }, look_at=TDWUtils.array_to_vector3( [3, 0.6, 0.35]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table = self.add_object(model_name="b05_table_new", position={ "x": 3, "y": 0, "z": 0.35 }, rotation={ "x": 0, "y": -90, "z": 0 }, library="models_full.json") # record = ModelLibrarian(library='models_full.json').get_record("b03_simpsons_london_-_round_hudson_mirror") # self.communicate({"$type": "add_object", # "name": "b05_tv1970", # "url": record.get_url(), # "scale_factor": 0.5, # "position": {"x": 3, "y": 0.6, "z": 0.35}, # "rotation": {"x": 0, "y": -90, "z": 0}, # "category": record.wcategory, # "id": self.get_unique_id()}) table_bounds = self.get_bounds_data(table) top = table_bounds.get_top(0) self.add_object(model_name="b03_tv_controller_2013__vray", position={ "x": 3, "y": top[1], "z": 0.7 }, rotation={ "x": 0, "y": 0, "z": 90 }, library="models_full.json") # # self.add_object(model_name='cgaxis_models_65_06_vray', # position={"x": 3, "y": top[1], "z": 0.25}, # rotation={"x": 0, "y": 0, "z": 0}, # library="models_full.json") self.add_object(model_name="b04_phone", position={ "x": 3, "y": top[1], "z": 0.25 }, rotation={ "x": 0, "y": -90, "z": 0 }, library="models_full.json") self.add_object(model_name="books_collection__1_2013", position={ "x": 3.3, "y": top[1], "z": -0.2 }, rotation={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([3, 0.5, 0.35]) }) images = Images(scene_data[0]) TDWUtils.save_images(images, "remote1", output_directory="replicated_images")
def get_trial_initialization_commands(self) -> List[dict]: commands = [] lib = MODEL_LIBRARIES["models_full.json"] random.shuffle(self.ramp_positions) random.shuffle(self.ramp_rotations) # Add ramps. for i in range(4): ramp_id = self.get_unique_id() commands.extend( self.add_physics_object( record=lib.get_record("ramp_with_platform_30"), position=self.ramp_positions[i], rotation=self.ramp_rotations[i], mass=self.RAMP_MASS, dynamic_friction=random.uniform(0.1, 0.9), static_friction=random.uniform(0.1, 0.9), bounciness=random.uniform(0.1, 0.9), o_id=ramp_id)) commands.extend([{ "$type": "scale_object", "id": ramp_id, "scale_factor": { "x": 0.75, "y": 0.75, "z": 0.75 } }, { "$type": "set_object_collision_detection_mode", "mode": "continuous_speculative", "id": ramp_id }, { "$type": "set_kinematic_state", "id": ramp_id, "is_kinematic": True, "use_gravity": True }]) # Teleport the avatar. cam_pos = random.choice(self.CAM_POSITIONS) cam_aim = {"x": 0, "y": 0.45, "z": 0} commands.extend([{ "$type": "teleport_avatar_to", "position": cam_pos }, { "$type": "look_at_position", "position": cam_aim }]) # Rotate the sensor container a little. for axis in ["pitch", "yaw"]: commands.extend([{ "$type": "rotate_sensor_container_by", "axis": axis, "angle": random.uniform(-10, 10) }, { "$type": "set_focus_distance", "focus_distance": TDWUtils.get_distance(cam_pos, cam_aim) }]) # Add bouncing objects. random.shuffle(self.toy_records) for i in range(random.randint(2, 6)): toy_id = self.get_unique_id() pos = TDWUtils.get_random_point_in_circle(center=np.array( [0, 0, 0]), radius=1.5) pos[1] = random.uniform(0.7, 2) record = self.toy_records[i] commands.extend( self.add_physics_object( record=record, position=TDWUtils.array_to_vector3(pos), rotation=TDWUtils.VECTOR3_ZERO, mass=random.uniform(0.5, 4), dynamic_friction=random.uniform(0.1, 0.7), static_friction=random.uniform(0.1, 0.7), bounciness=random.uniform(0.7, 1), o_id=toy_id)) s = TDWUtils.get_unit_scale(record) * random.uniform(0.85, 1.12) look_at = TDWUtils.array_to_vector3( TDWUtils.get_random_point_in_circle(center=np.array([0, 0, 0]), radius=5)) # Scale to "toy size" and look at a random point on the ground. commands.extend([{ "$type": "scale_object", "scale_factor": { "x": s, "y": s, "z": s }, "id": toy_id }, { "$type": "object_look_at_position", "position": look_at, "id": toy_id }]) # Rotate the object randomly. for axis in ["yaw", "roll"]: commands.append({ "$type": "rotate_object_by", "angle": random.uniform(-15, 15), "id": toy_id, "axis": axis, "is_world": False }) # Apply a force. commands.append({ "$type": "apply_force_magnitude_to_object", "magnitude": random.uniform(20, 40), "id": toy_id }) return commands
def run(self): self.start() init_setup_commands = [{ "$type": "set_screen_size", "width": 600, "height": 480 }, { "$type": "set_render_quality", "render_quality": 5 }] self.communicate(init_setup_commands) # Create an empty room. self.load_streamed_scene(scene="tdw_room_2018") # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate( TDWUtils.create_avatar(position={ "x": -1, "y": 0.9, "z": -1 }, look_at=TDWUtils.array_to_vector3( [-0.1, 0.5, -0.1]), avatar_id="avatar")) self.communicate({ "$type": "set_field_of_view", "field_of_view": 68.0, "avatar_id": "avatar" }) table_id = self.add_object(model_name="coffee_table_glass_round", position={ "x": 0, "y": 0, "z": 0 }, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="am151_043_00", position={ "x": 0.15, "y": top[1], "z": 0.15 }, rotation={ "x": 0, "y": 30, "z": 0 }, library="models_full.json") self.add_object(model_name="b04_tea_cup", position={ "x": 0.3, "y": top[1], "z": 0.1 }, library="models_full.json") self.add_object(model_name="cgaxis_models_65_06_vray", position={ "x": -0.17, "y": top[1], "z": 0.15 }, rotation={ "x": 0, "y": 24, "z": 0 }, library="models_full.json") # Enable image capture self.communicate({ "$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"] }) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({ "$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([-0.1, 0.45, -0.1]) }) images = Images(scene_data[0]) TDWUtils.save_images( images, "sadnwich1", output_directory= "/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior" )
def run(self): """ Generate room using COCO_TDW dataset """ objects_in_scene = 15 object_ids = [] # Get Category-Object mapping TDW_COCO_models = TDW_relationships.get_COCO_TDW_mapping() # print("TDWCOCO:", TDW_COCO_models) # print("KEYS:", TDW_COCO_models.keys()) # Gets COCO categories co-occurring in a scene # +5 is for dealing with failed object insertion attempts COCO_configurations = msCOCO_matrix.get_max_co_occurrence(5, int(objects_in_scene + 5)) configuration_1 = COCO_configurations[0] print("Config 1:", configuration_1) # TDW models/objects objects = [] for COCO_object in configuration_1: print(COCO_object) print(COCO_object.split()) if len(COCO_object.split()) > 1: COCO_object = COCO_object.split()[-1] print(COCO_object) # Check if COCO category is a key in the COCO-TDW mapping if COCO_object in TDW_COCO_models.keys(): # Gets random TDW model (from COCO-to-TDW map) based on COCO category key print(TDW_COCO_models[COCO_object]) model = TDW_COCO_models[COCO_object][random.randint(0, len(TDW_COCO_models[COCO_object]) - 1)] objects.append(model) print("COCO to TDW objects:", objects) # print(len(objects)) # Stores object categories that other objects can be placed upon (e.g. table, chair, couch, bed) surface_properties_list = TDW_COCO_models['table'] + TDW_COCO_models['chair'] + \ TDW_COCO_models['bed'] + TDW_COCO_models['couch'] + \ TDW_COCO_models['bench'] + TDW_COCO_models['refrigerator'] surface_categories = [] for surface_properties in surface_properties_list: surface_categories.append(surface_properties[0]) print("Surface Categories:", surface_categories) # Stores the actual surface object instances/ids alongside number of objects on the surface surface_object_ids = {} self.start() positions_list = [] # Stores current model locations and radii scene_dimensions = [] # Store scene/environment dimensions init_setup_commands = [{"$type": "set_screen_size", "width": 640, "height": 481}, {"$type": "set_render_quality", "render_quality": 1}] self.communicate(init_setup_commands) scene_lib = SceneLibrarian() # Disable physics when adding in new objects (objects float) self.communicate({"$type": "simulate_physics", "value": False}) for scene in scenes[1:]: # Load in scene # print("Scene", scene[0]) if scene[3] == "interior" and scene[0] == "box_room_2018": self.start() scene_record = scene_lib.get_record(scene[0]) self.communicate({"$type": "add_scene", "name": scene_record.name, "url": scene_record.get_url()}) # Gets dimensions of environments (e.g. inside, outside) in the scene # This command returns environment data in the form of a list of serialized byte arrays scene_bytes = self.communicate({"$type": "send_environments", "frequency": "once"}) # Iterating through data and parsing byte array # Ignoring the last element (the frame count) for b in scene_bytes[:-1]: e = Environments(b) for i in range(e.get_num()): center = e.get_center(i) bounds = e.get_bounds(i) env_id = e.get_id(i) scene_dimensions = [center, bounds, env_id] # Center, bounds are tuples # Must come before set_pass_masks avatar_position = TDWUtils.array_to_vector3([0.9 * scene_dimensions[1][0] / 2, scene_dimensions[1][1] / 2, 0]) # print("Avatar Position:", avatar_position) self.communicate(TDWUtils.create_avatar(avatar_id="avatar", position=avatar_position, look_at={"x": 0, "y": scene_dimensions[0][1] / 2, "z": 0})) # Set collision mode self.communicate({"$type": "set_avatar_collision_detection_mode", "mode": "continuous_speculative", "avatar_id": "avatar"}) # Alter FOV self.communicate({"$type": "set_field_of_view", "field_of_view": 80, "avatar_id": "avatar"}) # # Gets rid of header (Model: Category) # objects = TDW_COCO_models[1:] # random.shuffle(objects) obj_count = 0 obj_overlaps = 0 # Number of failed attempts to place object due to over-dense objects area while obj_count < objects_in_scene and obj_overlaps < 5: # Handles if object has been added to a flat surface added_to_surface = False print("Object COUNT:", obj_count) # Need to have random position for Bounds Data to return meaningful info valid_obj_pos = {"x": random.uniform(-1 * scene_dimensions[1][0] / 2, 0.5 * scene_dimensions[1][0] / 2), "y": scene_dimensions[1][1] / 4, "z": random.uniform(-0.9 * scene_dimensions[1][2] / 2, 0.9 * scene_dimensions[1][2] / 2)} print("First random position") # Add in the object at random position # Object will later be removed or updated accordingly after performing collision calculations record = ModelLibrarian(library="models_full.json").get_record(objects[obj_count][0]) print("Record gotten") print(objects[obj_count][0]) o_id = self.communicate({"$type": "add_object", "name": objects[obj_count][0], "url": record.get_url(), "scale_factor": record.scale_factor, "position": valid_obj_pos, "rotation": TDWUtils.VECTOR3_ZERO, "category": record.wcategory, "id": obj_count}) print("Random first add") # Returns bound data for added object bounds_data = self.communicate({"$type": "send_bounds", "frequency": "once"}) print("Bounds returned") # Appends object, with information on position and obj_radius, to positions_list # Length of buffer array should be 1 # print("Bounds Data:", bounds_data) for b in bounds_data[:-1]: # print("Buffer Loop:", b) b_id = OutputData.get_data_type_id(b) if b_id == "boun": # print("BOUNDS") o = Bounds(b) # print("# of Objects:", o.get_num()) # print("# of Failed Attempts:", obj_overlaps) # print("Buffer Array:", b) # print("Bounds Object:", o) for i in range(o.get_num()): print("Object ID:", o.get_id(i)) print("obj_count:", obj_count) print("Object:", objects[obj_count][0], "Category:", objects[obj_count][1]) # print("Object Center:", o.get_center(i)) # Only want to compute valid_position for object we are about to add # Skip any computation if this is not the case if o.get_id(i) != obj_count: continue # Useful for detecting if object fits in environment # print("Calculating if object fits in environment") width = distance.euclidean(o.get_left(i), o.get_right(i)) depth = distance.euclidean(o.get_front(i), o.get_back(i)) height = distance.euclidean(o.get_top(i), o.get_bottom(i)) # print("Width:", width) # print("Depth:", depth) # ("Height:", height) # Useful for avoiding object overlap # print("Calculating Object Bounds") center_to_top = distance.euclidean(o.get_center(i), o.get_top(i)) center_to_bottom = distance.euclidean(o.get_center(i), o.get_bottom(i)) center_to_left = distance.euclidean(o.get_center(i), o.get_left(i)) center_to_right = distance.euclidean(o.get_center(i), o.get_right(i)) center_to_front = distance.euclidean(o.get_center(i), o.get_front(i)) center_to_back = distance.euclidean(o.get_center(i), o.get_back(i)) # Max object radius (center to diagonal of bounding box) obj_radius = \ max(math.sqrt(center_to_top ** 2 + center_to_left ** 2 + center_to_front ** 2), math.sqrt(center_to_top ** 2 + center_to_right ** 2 + center_to_front ** 2), math.sqrt(center_to_top ** 2 + center_to_left ** 2 + center_to_back ** 2), math.sqrt(center_to_top ** 2 + center_to_right ** 2 + center_to_back ** 2), math.sqrt(center_to_bottom ** 2 + center_to_left ** 2 + center_to_front ** 2), math.sqrt(center_to_bottom ** 2 + center_to_right ** 2 + center_to_front ** 2), math.sqrt(center_to_bottom ** 2 + center_to_left ** 2 + center_to_back ** 2), math.sqrt(center_to_bottom ** 2 + center_to_right ** 2 + center_to_back ** 2)) # print("Obj_Radius:", obj_radius) # Set sweeping radius, based on scene plane dimensions l_radius = random.uniform(0, min(0.5 * scene_dimensions[1][0] / 2, 0.5 * scene_dimensions[1][2] / 2)) # Checking that object fits in scene viewing if (width > min(0.7 * scene_dimensions[1][0], 0.7 * scene_dimensions[1][2]) or depth > min(0.7 * scene_dimensions[1][0], 0.7 * scene_dimensions[1][2]) or height > 0.7 * scene_dimensions[1][1]): print("Object does not fit in scene") self.communicate([{"$type": "send_images", "frequency": "never"}, {"$type": "destroy_object", "id": obj_count}]) # Ensures next attempt to load in item is not the same item as before random.shuffle(objects) break # Not possible to find valid object position -- too many overlapping objects elif (not self._get_object_position(scene_dimensions=scene_dimensions, object_positions=positions_list, object_to_add_radius=obj_radius, max_tries=20, location_radius=l_radius)[0]): print("Could not calculate valid object location") self.communicate([{"$type": "send_images", "frequency": "never"}, {"$type": "destroy_object", "id": obj_count}]) obj_overlaps += 1 # Ensures next attempt to load in item is not the same item as before random.shuffle(objects) break # Find appropriate, non-overlapping object position # Reset object position to the valid position else: print("Object fits in scene") # Check if object fits on table, chair, couch, etc. # Add object if it fits, place it somewhere on top of the surface for surface_id in surface_object_ids.keys(): print("Surface ID:", surface_id) # Skip placement feasibility if the object is already a surface-type object # Ex. no chair on top of a table if objects[obj_count][0] in surface_categories: print("Object: %s is already a surface object" % objects[obj_count][0]) break # Check how many objects are on surface if surface_object_ids[surface_id] >= 3: print("Too many objects on surface") print("From surface objects dict:", surface_object_ids[surface_id]) continue surface_bounds = self.get_bounds_data(surface_id) surface_area = distance.euclidean(surface_bounds.get_left(0), surface_bounds.get_right(0)) * \ distance.euclidean(surface_bounds.get_front(0), surface_bounds.get_back(0)) obj_area = width * height if obj_area < surface_area: s_center_to_top = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_top(0)) s_center_to_bottom = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_bottom(0)) s_center_to_left = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_left(0)) s_center_to_right = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_right(0)) s_center_to_front = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_front(0)) s_center_to_back = distance.euclidean(surface_bounds.get_center(0), surface_bounds.get_back(0)) surface_radius = \ max(math.sqrt( s_center_to_top ** 2 + s_center_to_left ** 2 + s_center_to_front ** 2), math.sqrt( s_center_to_top ** 2 + s_center_to_right ** 2 + s_center_to_front ** 2), math.sqrt( s_center_to_top ** 2 + s_center_to_left ** 2 + s_center_to_back ** 2), math.sqrt( s_center_to_top ** 2 + s_center_to_right ** 2 + s_center_to_back ** 2), math.sqrt( s_center_to_bottom ** 2 + s_center_to_left ** 2 + s_center_to_front ** 2), math.sqrt( s_center_to_bottom ** 2 + s_center_to_right ** 2 + s_center_to_front ** 2), math.sqrt( s_center_to_bottom ** 2 + s_center_to_left ** 2 + s_center_to_back ** 2), math.sqrt( s_center_to_bottom ** 2 + s_center_to_right ** 2 + s_center_to_back ** 2)) print("Surface-type object") self.communicate({"$type": "destroy_object", "id": obj_count}) # Adding the object to the top of the surface on_pos = surface_bounds.get_top(0) on_y = on_pos[1] on_pos = TDWUtils.get_random_point_in_circle(np.array(on_pos), 0.7 * surface_radius) on_pos[1] = on_y on_pos = TDWUtils.array_to_vector3(on_pos) on_rot = {"x": 0, "y": random.uniform(-45, 45), "z": 0} # Add the object. print("Model Name on Surface:", objects[obj_count][0]) record = ModelLibrarian(library="models_full.json").get_record( objects[obj_count][0]) on_id = self.communicate({"$type": "add_object", "name": objects[obj_count][0], "url": record.get_url(), "scale_factor": record.scale_factor, "position": on_pos, "rotation": on_rot, "category": record.wcategory, "id": obj_count}) obj_count += 1 surface_object_ids[surface_id] += 1 object_ids.append(obj_count) print("Object added on top of surface") added_to_surface = True # Breaking out of surface objects loop break if added_to_surface: print("Breaking out of object loop") # Breaking out of object loop break print("Post-surface") valid_obj_pos = self._get_object_position(scene_dimensions=scene_dimensions, object_positions=positions_list, object_to_add_radius=obj_radius, max_tries=20, location_radius=l_radius)[1] print("Position calculated") positions_list.append(ObjectPosition(valid_obj_pos, obj_radius)) self.communicate([{"$type": "send_images", "frequency": "never"}, {"$type": "destroy_object", "id": obj_count}]) added_object_id = self.communicate({"$type": "add_object", "name": objects[obj_count][0], "url": record.get_url(), "scale_factor": record.scale_factor, "position": valid_obj_pos, "rotation": {"x": 0, "y": 0, "z": 0}, "category": record.wcategory, "id": obj_count}) # print("Object ID:", added_object_id) print("Regular object add") object_ids.append(added_object_id) # If TDW model belongs to surface categories, store id_information if objects[obj_count][0] in surface_categories: surface_object_ids[obj_count] = 0 # Rotate the object randomly print("Rotating object") self.communicate({"$type": "rotate_object_by", "angle": random.uniform(-45, 45), "axis": "yaw", "id": obj_count, "is_world": True}) # Minimal rotating for position differences # Don't rotate the object if doing so will result in overlap into scene if not (o.get_bottom(i)[1] < 0 or o.get_top(i)[1] > 0.9 * scene_dimensions[1][1]): pitch_angle = random.uniform(-45, 45) self.communicate({"$type": "rotate_object_by", "angle": pitch_angle, "axis": "pitch", "id": obj_count, "is_world": True}) roll_angle = random.uniform(-45, 45) self.communicate({"$type": "rotate_object_by", "angle": roll_angle, "axis": "roll", "id": obj_count, "is_world": True}) # Don't need this for just changing positions # Setting random materials/textures # Looping through sub-objects and materials sub_count = 0 for sub_object in record.substructure: # Loop through materials in sub-objects for j in range(len(sub_object)): # Get random material and load in material = random.choice(materials[1:]) self.load_material(material) print("Material loaded") # Set random material on material of sub-object self.communicate({"$type": "set_visual_material", "material_index": j, "material_name": material[0], "object_name": sub_object['name'], "id": obj_count}) print("Material set") sub_count += 1 if sub_count > 10: break break print("Updating count") obj_count += 1 print("Breaking out of object_id loop") break # Break out of buffer loop print("Breaking out of buffer loop") break # Move onto next iteration of while loop (next object to load in) print("Object added - next while loop iteration") continue # for i in range(200): # self.communicate({"$type": "simulate_physics", # "value": False}) # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) # Capture scene # NOTE: THESE SCENES GET REPLACED IN THE TARGET DIRECTORY scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": {"x": 0, "y": scene_dimensions[0][1] / 2, "z": 0}}) images = Images(scene_data[0]) TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=path) print("Object ids:", object_ids)
def _get_object_position(scene_dimensions: List[Tuple], object_positions: List[ObjectPosition], object_to_add_radius: float, max_tries: int = 1000, location_radius: float = 2) -> \ Dict[str, float]: """ Try to get a valid random position that doesn't interpenetrate with other objects. :param object_positions: The positions and radii of all objects so far that will be added to the scene. :param max_tries: Try this many times to get a valid position before giving up. :param location_radius: The radius to pick a position in. :return: A valid position that doesn't interpenetrate with other objects. """ o_pos = TDWUtils.get_random_point_in_circle(center=np.array([random.uniform(-0.9 * scene_dimensions[1][0] / 2, 0.9 * scene_dimensions[1][0] / 2), 0, random.uniform(-0.9 * scene_dimensions[1][2] / 2, 0.9 * scene_dimensions[1][2] / 2)]), radius=location_radius) # Scale circular point --> elliptic point for non-square rooms # Width > Depth of scene if scene_dimensions[1][0] > scene_dimensions[1][2]: o_pos[0] = o_pos[0] * scene_dimensions[1][0] / scene_dimensions[1][2] # Depth > Width of scene else: o_pos[2] = o_pos[2] * scene_dimensions[1][2] / scene_dimensions[1][0] # Start with everything on ground # # Set height value # o_pos[1] = random.uniform(0, 0.9 * scene_dimensions[1][1] / 3) o_pos = TDWUtils.array_to_vector3(o_pos) # print("O_Pos:", o_pos) # Pick a position away from other objects. ok = False count = 0 while not ok and count < max_tries: count += 1 ok = True for o in object_positions: # If the object is too close to another object, try another position. # Also check that object bounds are generated within reasonable viewing window # Check Radius of old objects as well if (TDWUtils.get_distance(o.position, o_pos) <= 1.1 * o.obj_radius or TDWUtils.get_distance(o.position, o_pos) <= 1.1 * object_to_add_radius or o_pos["x"] < -0.9 * scene_dimensions[1][0] / 2 or o_pos["x"] > 0.8 * scene_dimensions[1][0] / 2 or o_pos["y"] < 0 or o_pos["y"] > scene_dimensions[1][1] / 2 or o_pos["z"] < -0.8 * scene_dimensions[1][2] / 2 or o_pos["z"] > 0.8 * scene_dimensions[1][2] / 2): ok = False # o_pos = TDWUtils.get_random_point_in_circle(center=np.array([ # random.uniform(-0.9 * scene_dimensions[1][0] / 2 # , 0.9 * scene_dimensions[1][0] / 2), # 0, # random.uniform(-0.9 * scene_dimensions[1][2] / 2 # , 0.9 * scene_dimensions[1][2] / 2)]), # radius=location_radius) o_pos = TDWUtils.get_random_point_in_circle(center=np.array([0, 0, 0]), radius=location_radius) # Scale circular point --> elliptic point for non-square rooms # Width > Depth of scene if scene_dimensions[1][0] > scene_dimensions[1][2]: o_pos[0] = o_pos[0] * scene_dimensions[1][0] / scene_dimensions[1][2] # Depth > Width of scene else: o_pos[2] = o_pos[2] * scene_dimensions[1][2] / scene_dimensions[1][0] # Don't change height for now # o_pos[1] = random.uniform(0, 0.9 * scene_dimensions[1][1]) o_pos = TDWUtils.array_to_vector3(o_pos) print("O_Pos:", o_pos) return [ok, o_pos]
def run(self): self.start() init_setup_commands = [{"$type": "set_screen_size", "width": 600, "height": 480}, {"$type": "set_render_quality", "render_quality": 5}] self.communicate(init_setup_commands) # Create an empty room. self.load_streamed_scene(scene="tdw_room_2018") # Disable physics. self.communicate({"$type": "set_gravity", "value": False}) # Add the avatar. self.communicate(TDWUtils.create_avatar(position={"x": -0.8, "y": 1.6, "z": 0.6}, look_at=TDWUtils.array_to_vector3([0, 0.4, 0]), avatar_id="avatar")) # self.communicate({"$type": "set_field_of_view", # "field_of_view": 68.0, # "avatar_id": "avatar"}) table_id = self.add_object(model_name="glass_table_round", position={"x": 0, "y": 0, "z": 0}, library="models_full.json") table_bounds = self.get_bounds_data(table_id) top = table_bounds.get_top(0) self.add_object(model_name="b03_orange", position={"x": 0, "y": top[1], "z": 0}, library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.17) position[1] = top[1] self.add_object(model_name="b03_orange", position={"x": 0, "y": top[1], "z": 0}, library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.4) position[1] = top[1] self.add_object(model_name="b04_banana", position=TDWUtils.array_to_vector3(position), rotation={"x": 0, "y": 40, "z": 0}, library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.4) position[1] = top[1] self.add_object(model_name="b04_banana", position=TDWUtils.array_to_vector3(position), library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.5) position[1] = top[1] self.add_object(model_name="red_apple", position=TDWUtils.array_to_vector3(position), library="models_full.json") position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.6) position[1] = top[1] self.add_object(model_name="vk0076_fruitfork", position=TDWUtils.array_to_vector3(position), library="models_full.json") # Enable image capture self.communicate({"$type": "set_pass_masks", "avatar_id": "avatar", "pass_masks": ["_img", "_id"]}) self.communicate({"$type": "send_images", "frequency": "always"}) scene_data = self.communicate({"$type": "look_at_position", "avatar_id": "avatar", "position": TDWUtils.array_to_vector3([0, 0.4, 0])}) images = Images(scene_data[0]) TDWUtils.save_images(images, "fruit_table", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior")