def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(10, 10))
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
self.communicate(self.get_add_hdri_skybox("autumn_hockey_4k"))
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": 3, "y": 1.3, "z": 0},
look_at=TDWUtils.array_to_vector3([0, 0.5, 0]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("grass_countryside")
print(record)
self.communicate({"$type": "add_material", "name": "grass_countryside", "url": record.get_url()})
self.communicate({"$type": "set_proc_gen_floor_material", "name": "grass_countryside"})
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
self.add_object(model_name="b05_clochild4",
position={"x": 0, "y": 0, "z": 0},
rotation={"x": 0, "y": 30, "z": 0},
library="models_full.json")
self.add_object(model_name="baseballbat",
position={"x": 0.5, "y": 0, "z": 0.3},
rotation={"x": 0, "y": 30, "z": 0},
library="models_full.json")
self.add_object(model_name="base-ball",
position={"x": 0.4, "y": 0, "z": 0.22},
rotation={"x": 0, "y": 30, "z": 10},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([0, 0.5, 0])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "kid_baseball", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def _get_object_position(object_positions: List[ObjectPosition], max_tries: int = 1000, radius: float = 2) -> \
Dict[str, float]:
"""
Try to get a valid random position that doesn't interpentrate with other objects.
:param object_positions: The positions and radii of all objects so far that will be added to the scene.
:param max_tries: Try this many times to get a valid position before giving up.
:param radius: The radius to pick a position in.
:return: A valid position that doesn't interpentrate with other objects.
"""
o_pos = TDWUtils.array_to_vector3(
TDWUtils.get_random_point_in_circle(center=np.array([0, 0, 0]),
radius=radius))
# Pick a position away from other objects.
ok = False
count = 0
while not ok and count < max_tries:
count += 1
ok = True
for o in object_positions:
# If the object is too close to another object, try another position.
if TDWUtils.get_distance(o.position, o_pos) <= o.radius:
ok = False
o_pos = TDWUtils.array_to_vector3(
TDWUtils.get_random_point_in_circle(center=np.array(
[0, 0, 0]),
radius=radius))
return o_pos
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(32, 32))
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": -1.5, "y": 1.5, "z": -15},
look_at=TDWUtils.array_to_vector3([-1.7, 0.7, -2.3]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("concrete_chipped_cracked")
self.communicate({"$type": "add_material", "name": "concrete_chipped_cracked", "url": record.get_url()})
self.communicate({"$type": "set_proc_gen_floor_material", "name": "concrete_chipped_cracked"})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
self.communicate({"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"})
bus = self.add_object(model_name="b06_bus_new",
position={"x": -2, "y": 0, "z": 0},
rotation={"x": 0, "y": -70, "z": 0},
library="models_full.json")
bus_bounds = self.get_bounds_data(bus)
bus_front = bus_bounds.get_front(0)
print(bus_front)
self.add_object(model_name="3dscan_man_004",
position={"x": -3.7, "y": 0, "z": -7.3},
rotation={"x": 0, "y": 0, "z": 0},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([-1.7, 0.7, -2.3])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "bus",
output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 1280,
"height": 962},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 1, "z": 0},
look_at=TDWUtils.array_to_vector3([2, 0.8, -1.5]),
avatar_id="avatar"))
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
table_id = self.add_object(model_name="glass_table",
position={"x": 2, "y": 0, "z": -1.5},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="flower_vase_set_2",
position={"x": 1.6, "y": top[1], "z": -1.5},
library="models_full.json")
self.add_object(model_name="flower_vase_set_2",
position={"x": 2, "y": top[1], "z": -1.5},
library="models_full.json")
self.add_object(model_name="flower_vase_set_2",
position={"x": 2.4, "y": top[1], "z": -1.5},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([2, 0.8, -1.5])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "flowers_row", output_directory="replicated_images")
def push_into_other(self) -> List[dict]:
"""
:return: A list of commands to push one object into another object.
"""
# Get two points some distance from each other.
p0 = np.array([0, 0, 0])
p1 = np.array([0, 0, 0])
center = np.array([0, 0, 0])
count = 0
while count < 1000 and np.linalg.norm(np.abs(p1 - p0)) < 1.5:
p0 = TDWUtils.get_random_point_in_circle(center=center, radius=2)
p1 = TDWUtils.get_random_point_in_circle(center=center, radius=2)
count += 1
p0 = TDWUtils.array_to_vector3(p0)
p0["y"] = 0.2
p1 = TDWUtils.array_to_vector3(p1)
p1["y"] = 0.1
# Get commands to create the first object and push it.
commands = self._push(position=p0,
target=p1,
force_mag=random.uniform(1000, 2000))
# Add commands for the second object.
second_object_commands, s_id = self._get_squishable(position=p1,
rotation={
"x":
0,
"y":
random.uniform(
0, 360),
"z":
0
})
commands.extend(second_object_commands)
# Set the avatar.
# Get the midpoint between the objects and use this to place the avatar.
p_med = TDWUtils.array_to_vector3(
np.array([(p0["x"] + p1["x"]) / 2, 0, (p0["z"] + p1["z"]) / 2]))
commands.extend(
self._set_avatar(a_pos=self.get_random_avatar_position(
radius_min=1.8,
radius_max=2.3,
y_min=1,
y_max=1.75,
center=p_med),
cam_aim=p1))
return commands
def _push(self, position: Dict[str, float], target: Dict[str, float],
force_mag: float) -> List[dict]:
"""
:param position: The initial position.
:param target: The target position (used for the force vector).
:param force_mag: The force magnitutde.
:return: A list of commands to push the object along the floor.
"""
commands, soft_id = self._get_squishable(position=position,
rotation={
"x": 0,
"y":
random.uniform(0, 360),
"z": 0
})
# Get a force vector towards the target.
p0 = TDWUtils.vector3_to_array(position)
p1 = TDWUtils.vector3_to_array(target)
force = ((p1 - p0) / (np.linalg.norm(p1 - p0))) * force_mag
# Add the force.
commands.append({
"$type": "apply_force_to_flex_object",
"force": TDWUtils.array_to_vector3(force),
"id": soft_id
})
return commands
def get_falling_commands(self, mass: float = 3) -> List[List[dict]]:
"""
:param mass: Objects with <= this mass might receive a force.
:return: A list of lists; per-frame commands to make small objects fly up.
"""
per_frame_commands: List[List[dict]] = []
# Get a list of all small objects.
small_ids = self.get_objects_by_mass(mass)
random.shuffle(small_ids)
max_num_objects = len(small_ids) if len(small_ids) < 8 else 8
min_num_objects = max_num_objects - 3
if min_num_objects <= 0:
min_num_objects = 1
# Add some objects.
for i in range(random.randint(min_num_objects, max_num_objects)):
o_id = small_ids.pop(0)
force_dir = np.array([random.uniform(-0.125, 0.125), random.uniform(0.7, 1), random.uniform(-0.125, 0.125)])
force_dir = force_dir / np.linalg.norm(force_dir)
min_force = self.physics_info[o_id].mass * 2
max_force = self.physics_info[o_id].mass * 4
force = TDWUtils.array_to_vector3(force_dir * random.uniform(min_force, max_force))
per_frame_commands.append([{"$type": "apply_force_to_object",
"force": force,
"id": o_id}])
# Wait some frames.
for j in range(10, 30):
per_frame_commands.append([])
return per_frame_commands
def run(self):
self.start()
width = 12
length = 12
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(width, length))
# Disable physics.
self.communicate({"$type": "simulate_physics", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -5,
"y": 10,
"z": -2
},
look_at=TDWUtils.VECTOR3_ZERO))
# Create the dining table.
self.dining_table()
# Create the sofa.
self.sofa(width, length)
# Bake the NavMesh.
self.communicate({"$type": "bake_nav_mesh", "carve_type": "none"})
# Get a random position on the NavMesh.
position = TDWUtils.get_random_position_on_nav_mesh(
self, width, length)
position = TDWUtils.array_to_vector3(position)
# Add a side table.
self.side_table(position)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 1280,
"height": 962
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "simulate_physics", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0.9,
"y": 1.5,
"z": 0.9
},
look_at=TDWUtils.array_to_vector3(
[0, 0.8, 0]),
avatar_id="avatar"))
# Create the dining table.
self.dining_table()
def go_to_start(self) -> None:
"""
Navigate back to the start position.
"""
path = np.flip(Demo.PATH[:-1], axis=0)
for waypoint in path:
self.move_to(target=TDWUtils.array_to_vector3(waypoint))
def get_p_1(self) -> Dict[str, float]:
"""
:return: A position in the c_1 sub-sector.
"""
return TDWUtils.array_to_vector3(
TDWUtils.get_random_point_in_circle(center=self.c_1,
radius=self.RADIUS))
def is_done(self, resp: List[bytes], frame: int) -> bool:
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
# If the ball reaches or overshoots the destination, the trial is done.
if r_id == "tran":
t = Transforms(r)
d0 = TDWUtils.get_distance(
TDWUtils.array_to_vector3(t.get_position(0)), self._p0)
d1 = TDWUtils.get_distance(self._p0, self._p1)
return d0 > d1 * 1.5
return False
def side_table(self, position, parent=None):
"""
1. Create a side table.
2. Maybe create an object on the side table.
:param position: The position of the table (Vector3).
:param parent: THe parent parameters: (position, rotation).
"""
# Create the table.
table_id = self.add_object(random.choice(
ProcGenInteriorDesign.SIDE_TABLE),
position=position)
if parent:
# Place the object to the side of the parent and match its rotation.
parent_position = parent[0]
parent_rotation = parent[1]
self.side_of(table_id, parent_position, parent_rotation)
else:
# Apply a random rotation.
self.communicate({
"$type": "rotate_object_by",
"angle": random.uniform(-45, 45),
"axis": "yaw",
"id": table_id,
"is_world": True
})
# Maybe put an object on the table.
if random.random() > 0.25:
# Get the top position of the table.
table_bounds = self.get_bounds_data(table_id)
on_pos = table_bounds.get_top(0)
on_y = on_pos[1]
# Perturb the position slightly.
on_pos = TDWUtils.get_random_point_in_circle(
np.array(on_pos), 0.125)
on_pos[1] = on_y
on_pos = TDWUtils.array_to_vector3(on_pos)
on_rot = {"x": 0, "y": random.uniform(-45, 45), "z": 0}
# Add the object.
self.add_object(random.choice(ProcGenInteriorDesign.SIDE_TABLE_ON),
position=on_pos,
rotation=on_rot)
def get_move_along_direction(pos: Dict[str, float], target: Dict[str, float], d: float, noise: float = 0) -> \
Dict[str, float]:
"""
:param pos: The object's position.
:param target: The target position.
:param d: The distance to teleport.
:param noise: Add a little noise to the teleport.
:return: A position from pos by distance d along a directional vector defined by pos, target.
"""
direction = TDWUtils.array_to_vector3(
(TDWUtils.vector3_to_array(target) - TDWUtils.vector3_to_array(pos)) /
TDWUtils.get_distance(pos, target))
return {
"x": pos["x"] + direction["x"] * d + random.uniform(-noise, noise),
"y": pos["y"],
"z": pos["z"] + direction["z"] * d + random.uniform(-noise, noise)
}
def init_scene(self,
scene: str = None,
layout: int = None,
room: int = None,
goal_room: int = None) -> ActionStatus:
origin = np.array([0, 0, 0])
commands = [{
"$type": "load_scene",
"scene_name": "ProcGenScene"
},
TDWUtils.create_empty_room(12, 12)]
self._add_container(model_name="basket_18inx18inx12iin",
position={
"x": 0.354,
"y": 0,
"z": 0.549
},
rotation={
"x": 0,
"y": -70,
"z": 0
})
# Add target objects in a circle.
num_objects = 10
d_theta = 360 / num_objects
theta = d_theta / 2
pos = np.array([2, 0, 0])
for j in range(num_objects):
object_position = TDWUtils.rotate_position_around(origin=origin,
position=pos,
angle=theta)
self._add_target_object(
"jug05", position=TDWUtils.array_to_vector3(object_position))
theta += d_theta
commands.extend(self._get_scene_init_commands())
resp = self.communicate(commands)
self._cache_static_data(resp=resp)
# Wait for the Magnebot to reset to its neutral position.
status = self._do_arm_motion()
self._end_action()
return status
def transport(self) -> None:
"""
Transport some objects to the other room.
"""
for i in range(4):
# Get the closest object that still needs to be transported.
self._to_transport = list(sorted(self._to_transport, key=lambda x: np.linalg.norm(
self.state.object_transforms[x].position - self.state.magnebot_transform.position)))
object_id = self._to_transport[0]
# Go to the object and pick it up.
self.move_to(target=object_id)
self.pick_up(target=object_id, arm=Arm.right)
# Put the object in the container.
self.put_in()
# Record this object as done.
self._to_transport = self._to_transport[1:]
# Follow the path to the other room.
path = Demo.PATH[1:]
for waypoint in path:
self.move_to(target=TDWUtils.array_to_vector3(waypoint))
self.pour_out()
def reach_for_target(self,
arm: Arm,
target: np.array,
stop_on_mitten_collision: bool,
target_orientation: np.array = None,
precision: float = 0.05) -> List[dict]:
"""
Get an IK solution to move a mitten to a target position.
:param arm: The arm (left or right).
:param target: The target position for the mitten.
:param target_orientation: Target IK orientation. Usually you should leave this as None (the default).
:param stop_on_mitten_collision: If true, stop moving when the mitten collides with something.
:param precision: The distance threshold to the target position.
:return: A list of commands to begin bending the arm.
"""
# Get the IK solution.
rotations, ik_target = self._get_ik(
target=target, arm=arm, target_orientation=target_orientation)
angle = TDWUtils.get_angle_between(v1=FORWARD,
v2=self.frame.get_forward())
target = TDWUtils.rotate_position_around(
position=ik_target, angle=angle) + self.frame.get_position()
rotation_targets = dict()
for c, r in zip(self._arms[arm].links[1:-1], rotations[1:-1]):
rotation_targets[c.name] = r
self._ik_goals[arm] = _IKGoal(
target=target,
stop_on_mitten_collision=stop_on_mitten_collision,
rotations=rotation_targets,
precision=precision)
commands = [self.get_start_bend_sticky_mitten_profile(arm=arm)]
# If the avatar is holding something, strengthen the wrist.
held = self.frame.get_held_left(
) if arm == Arm.left else self.frame.get_held_right()
if len(held) > 0:
commands.append({
"$type": "set_joint_angular_drag",
"joint": f"wrist_{arm.name}",
"axis": "roll",
"angular_drag": 50,
"avatar_id": self.id
})
if self._debug:
print([np.rad2deg(r) for r in rotations])
self._plot_ik(target=ik_target, arm=arm)
# Show the target.
commands.extend([{
"$type": "remove_position_markers"
}, {
"$type": "add_position_marker",
"position": TDWUtils.array_to_vector3(target)
}])
a = arm.name
for c in self._ik_goals[arm].rotations:
j = c.split("_")
# Apply the motion.
commands.extend([{
"$type":
"bend_arm_joint_to",
"angle":
np.rad2deg(self._ik_goals[arm].rotations[c]),
"joint":
f"{j[0]}_{a}",
"axis":
j[1],
"avatar_id":
self.id
}])
return commands
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("parquet_wood_mahogany")
self.communicate({
"$type": "add_material",
"name": "parquet_wood_mahogany",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "parquet_wood_mahogany"
})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -1,
"y": 1.4,
"z": -1
},
look_at=TDWUtils.array_to_vector3(
[-0.1, 0.7, -0.1]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table_id = self.add_object(
model_name=
"b03_restoration_hardware_pedestal_salvaged_round_tables",
position={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="club_sandwich",
position={
"x": 0.45,
"y": top[1],
"z": 0.25
},
rotation={
"x": 0,
"y": 30,
"z": 0
},
library="models_full.json")
self.add_object(model_name="coffeemug",
position={
"x": 0.03,
"y": top[1],
"z": 0.03
},
library="models_full.json")
self.add_object(model_name="knife3",
position={
"x": -0.17,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 24,
"z": 90
},
library="models_full.json")
self.add_object(model_name="notes_02",
position={
"x": 0.17,
"y": top[1],
"z": -0.12
},
rotation={
"x": 0,
"y": 24,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([-0.1, 0.7, -0.1])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"sadnwich2",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("laser_cut_white_norway_spruce")
self.communicate({
"$type": "add_material",
"name": "laser_cut_white_norway_spruce",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "laser_cut_white_norway_spruce"
})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 1.5,
"y": 1.6,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[0, 0.5, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table = self.add_object(
model_name="boconcept_lugo_dining_table_vray1.5",
position={
"x": 0,
"y": 0,
"z": 0.0
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table)
top = table_bounds.get_top(0)
self.add_object(model_name='macbook_001',
position={
"x": 0,
"y": top[1],
"z": 0.1
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
self.add_object(model_name='spunlight_designermesh_lamp',
position={
"x": -0.19,
"y": top[1],
"z": -0.4
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
self.add_object(model_name="apple_magic_mouse_2_(2015)_vray",
position={
"x": 0,
"y": top[1],
"z": 0.32
},
rotation={
"x": 0,
"y": 86,
"z": 0
},
library="models_full.json")
self.add_object(model_name="f10_apple_iphone_4",
position={
"x": 0.14,
"y": top[1],
"z": -0.3
},
rotation={
"x": 0,
"y": 12,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.5, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"phone",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
self.start()
# init_setup_commands = [{"$type": "set_screen_size",
# "width": 600,
# "height": 480},
# {"$type": "set_render_quality",
# "render_quality": 5}]
# self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 2.5,
"y": 1.5,
"z": 0.3
},
look_at=TDWUtils.array_to_vector3(
[3, 0.5, 0.5]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("gravel_white")
self.communicate({
"$type": "add_material",
"name": "gravel_white",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "gravel_white"
})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
bench = self.add_object(model_name="b04_wood_metal_park_bench",
position={
"x": 3,
"y": 0,
"z": 0.5
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
bench_bounds = self.get_bounds_data(bench)
top = bench_bounds.get_top(0)
self.add_object(model_name="b05_baseball_bat_01",
position={
"x": 2.8,
"y": top[1] - 0.35,
"z": 0
},
rotation={
"x": 0,
"y": 90,
"z": 90
},
library="models_full.json")
self.add_object(model_name="b05_baseballnew_v03_12",
position={
"x": 2.9,
"y": top[1] - 0.35,
"z": 0.4
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([3, 0.5, 0.5])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"bench_ball",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def sofa(self, width, length):
"""
1. Create a sofa in the corner in the room.
2. Create some objects in front of the sofa.
3. Create some objects the sides of the sofa.
:param width: The width of the room.
:param length: The length of the room.
"""
# Add the sofa in the corner of the room.
sofa_position = {"x": -(width / 2 - 2), "y": 0, "z": length / 2 - 2}
sofa_id = self.add_object(random.choice(ProcGenInteriorDesign.SOFA),
position=sofa_position)
# Rotate the sofa to face the center of the room.
self.communicate({
"$type": "object_look_at_position",
"id": sofa_id,
"position": TDWUtils.VECTOR3_ZERO
})
sofa_bounds = self.get_bounds_data(sofa_id)
# Set the rotation to match the sofa.
sofa_transform = self.get_transforms_data([sofa_id])
sofa_rotation = sofa_transform.get_rotation(0)
sofa_rotation = {
"x": sofa_rotation[0],
"y": sofa_rotation[1],
"z": sofa_rotation[2],
"w": sofa_rotation[3]
}
# Get a focal point for objects to place in front of the sofa.
centerpoint = TDWUtils.extend_line(np.array(sofa_bounds.get_center(0)),
np.array(sofa_bounds.get_front(0)),
0.5)
centerpoint[1] = 0
# Place objects in front of the sofa at various positions near the centerpoint.
for i in range(random.randint(1, 3)):
pos = TDWUtils.get_random_point_in_circle(centerpoint, 0.125)
pos = TDWUtils.array_to_vector3(pos)
self.add_object(random.choice(ProcGenInteriorDesign.SOFA_FRONT_OF),
position=pos,
rotation={
"x": 0,
"y": random.uniform(-30, 30),
"z": 0
})
# Create 1-2 objects to the sides of the sofa.
positions = [sofa_bounds.get_left(0), sofa_bounds.get_right(0)]
random.shuffle(positions)
if random.random() > 0.5:
positions.pop(0)
for side_position in positions:
side_position = [side_position[0], 0, side_position[2]]
side_position = TDWUtils.array_to_vector3(side_position)
# Create a side table.
if random.random() > 0.5:
self.side_table(side_position, (sofa_position, sofa_rotation))
# Create some other object.
else:
# Create the object.
side_id = self.add_object(random.choice(
ProcGenInteriorDesign.SOFA_SIDE_OF),
position=side_position)
self.side_of(side_id, sofa_position, sofa_rotation)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 1280,
"height": 962
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
width = 8
length = 8
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(width, length))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 3,
"y": 1.7,
"z": 2
},
look_at={
"x": 0,
"y": 0.8,
"z": 2
},
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
self.add_object(model_name="b05_bed_giorgetti_rea",
position={
"x": 0.3,
"y": 0,
"z": 2.2
},
rotation={
"x": 0,
"y": 180,
"z": 0
},
library="models_full.json")
# Create the dining table.
# self.dining_table()
# Create the sofa.
self.sofa(width, length)
# Bake the NavMesh.
self.communicate({"$type": "bake_nav_mesh"})
print('mesh')
# Get a random position on the NavMesh.
position = TDWUtils.get_random_position_on_nav_mesh(
self, width, length)
position = TDWUtils.array_to_vector3(position)
# Add a side table.
self.side_table(position)
position = TDWUtils.get_random_position_on_nav_mesh(
self, width, length)
position = TDWUtils.array_to_vector3(position)
self.add_object(model_name="648972_chair_poliform_harmony",
position=position,
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type": "look_at_position",
"avatar_id": "avatar",
"position": {
"x": 0,
"y": 0.8,
"z": 2
}
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"bedroom1",
output_directory="replicated_images")
def run(self):
rng = np.random.RandomState(0)
self.model_librarian = ModelLibrarian("models_special.json")
self.start()
commands = [TDWUtils.create_empty_room(100, 100)]
# The starting height of the objects.
y = 10
# The radius of the circle of objects.
r = 7.0
# The mass of each object.
mass = 5
# Get all points within the circle defined by the radius.
p0 = np.array((0, 0))
o_id = 0
for x in np.arange(-r, r, 1):
for z in np.arange(-r, r, 1):
p1 = np.array((x, z))
dist = np.linalg.norm(p0 - p1)
if dist < r:
# Add an object.
# Set its mass, physics properties, and color.
commands.extend([self.get_add_object("prim_cone",
object_id=o_id,
position={"x": x, "y": y, "z": z},
rotation={"x": 0, "y": 0, "z": 180}),
{"$type": "set_mass",
"id": o_id,
"mass": mass},
{"$type": "set_physic_material",
"dynamic_friction": 0.8,
"static_friction": 0.7,
"bounciness": 0.5,
"id": o_id},
{"$type": "set_color",
"color": {"r": rng.random_sample(),
"g": rng.random_sample(),
"b": rng.random_sample(),
"a": 1.0},
"id": o_id}])
o_id += 1
# Request transforms per frame.
commands.extend([{"$type": "send_transforms",
"frequency": "always"}])
# Create an avatar to observe the grisly spectacle.
avatar_position = {"x": -20, "y": 8, "z": 18}
commands.extend(TDWUtils.create_avatar(position=avatar_position, look_at=TDWUtils.VECTOR3_ZERO))
commands.append({"$type": "set_focus_distance",
"focus_distance": TDWUtils.get_distance(avatar_position, TDWUtils.VECTOR3_ZERO)})
resp = self.communicate(commands)
# If an objects are this far away from (0, 0, 0) the forcefield "activates".
forcefield_radius = 5
# The forcefield will bounce objects away at this force.
forcefield_force = -10
zeros = np.array((0, 0, 0))
for i in range(1000):
transforms = Transforms(resp[0])
commands = []
for j in range(transforms.get_num()):
pos = transforms.get_position(j)
pos = np.array(pos)
# If the object is in the forcefield, apply a force.
if TDWUtils.get_distance(TDWUtils.array_to_vector3(pos), TDWUtils.VECTOR3_ZERO) <= forcefield_radius:
# Get the normalized directional vector and multiply it by the force magnitude.
d = zeros - pos
d = d / np.linalg.norm(d)
d = d * forcefield_force
commands.append({"$type": "apply_force_to_object",
"id": transforms.get_id(j),
"force": TDWUtils.array_to_vector3(d)})
resp = self.communicate(commands)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 1280,
"height": 962
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -2,
"y": 1.5,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[4, 0.8, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
self.add_object(model_name="b05_c-5_classic_series_bench-2012",
position={
"x": 3,
"y": 0,
"z": -1.5
},
library="models_full.json")
self.add_object(model_name="b05_c-5_classic_series_bench-2012",
position={
"x": 3,
"y": 0,
"z": 1.5
},
library="models_full.json")
self.add_object(model_name="baleri_italia_folded_black_table",
position={
"x": 2,
"y": 0,
"z": -1.5
},
rotation=TDWUtils.array_to_vector3([0, 90, 0]),
library="models_full.json")
self.add_object(model_name="baleri_italia_folded_black_table",
position={
"x": 2,
"y": 0,
"z": 1.5
},
rotation=TDWUtils.array_to_vector3([0, 90, 0]),
library="models_full.json")
self.add_object(model_name="linen_dining_chair",
position={
"x": 1,
"y": 0,
"z": -1.5
},
rotation=TDWUtils.array_to_vector3([0, 90, 0]),
library="models_full.json")
self.add_object(model_name="linen_dining_chair",
position={
"x": 1,
"y": 0,
"z": 1.5
},
rotation=TDWUtils.array_to_vector3([0, 90, 0]),
library="models_full.json")
# self.communicate({"$type": "rotate_object_by",
# "angle": 180,
# "axis": "yaw",
# "id": chair_id,
# "is_world": True})
# record = ModelLibrarian(library='models_full.json').get_record("b05_tv1970")
# self.communicate({"$type": "add_object",
# "name": "b05_tv1970",
# "url": record.get_url(),
# "scale_factor": 30,
# "position": {"x": 0, "y": 0, "z": 3.5},
# "category": record.wcategory,
# "id": self.get_unique_id()})
#
#
# side_table = self.add_object(model_name="side_table_wood",
# position={"x": 1.5, "y": 0, "z": 4},
# library="models_full.json")
# side_table_bounds = self.get_bounds_data(side_table)
#
# top = side_table_bounds.get_top(0)
# self.add_object(model_name="acacia_table_lamp_jamie_young",
# position={"x": 1.5, "y": top[1], "z": 4},
# library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([4, 0.8, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"two_tables_v2",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 2.0,
"y": 0.8,
"z": -0.55
},
look_at=TDWUtils.array_to_vector3(
[3, 0.6, 0.35]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table = self.add_object(model_name="b05_table_new",
position={
"x": 3,
"y": 0,
"z": 0.35
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
# record = ModelLibrarian(library='models_full.json').get_record("b03_simpsons_london_-_round_hudson_mirror")
# self.communicate({"$type": "add_object",
# "name": "b05_tv1970",
# "url": record.get_url(),
# "scale_factor": 0.5,
# "position": {"x": 3, "y": 0.6, "z": 0.35},
# "rotation": {"x": 0, "y": -90, "z": 0},
# "category": record.wcategory,
# "id": self.get_unique_id()})
table_bounds = self.get_bounds_data(table)
top = table_bounds.get_top(0)
self.add_object(model_name="b03_tv_controller_2013__vray",
position={
"x": 3,
"y": top[1],
"z": 0.7
},
rotation={
"x": 0,
"y": 0,
"z": 90
},
library="models_full.json")
#
# self.add_object(model_name='cgaxis_models_65_06_vray',
# position={"x": 3, "y": top[1], "z": 0.25},
# rotation={"x": 0, "y": 0, "z": 0},
# library="models_full.json")
self.add_object(model_name="b04_phone",
position={
"x": 3,
"y": top[1],
"z": 0.25
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
self.add_object(model_name="books_collection__1_2013",
position={
"x": 3.3,
"y": top[1],
"z": -0.2
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([3, 0.5, 0.35])
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"remote1",
output_directory="replicated_images")
def get_trial_initialization_commands(self) -> List[dict]:
commands = []
lib = MODEL_LIBRARIES["models_full.json"]
random.shuffle(self.ramp_positions)
random.shuffle(self.ramp_rotations)
# Add ramps.
for i in range(4):
ramp_id = self.get_unique_id()
commands.extend(
self.add_physics_object(
record=lib.get_record("ramp_with_platform_30"),
position=self.ramp_positions[i],
rotation=self.ramp_rotations[i],
mass=self.RAMP_MASS,
dynamic_friction=random.uniform(0.1, 0.9),
static_friction=random.uniform(0.1, 0.9),
bounciness=random.uniform(0.1, 0.9),
o_id=ramp_id))
commands.extend([{
"$type": "scale_object",
"id": ramp_id,
"scale_factor": {
"x": 0.75,
"y": 0.75,
"z": 0.75
}
}, {
"$type": "set_object_collision_detection_mode",
"mode": "continuous_speculative",
"id": ramp_id
}, {
"$type": "set_kinematic_state",
"id": ramp_id,
"is_kinematic": True,
"use_gravity": True
}])
# Teleport the avatar.
cam_pos = random.choice(self.CAM_POSITIONS)
cam_aim = {"x": 0, "y": 0.45, "z": 0}
commands.extend([{
"$type": "teleport_avatar_to",
"position": cam_pos
}, {
"$type": "look_at_position",
"position": cam_aim
}])
# Rotate the sensor container a little.
for axis in ["pitch", "yaw"]:
commands.extend([{
"$type": "rotate_sensor_container_by",
"axis": axis,
"angle": random.uniform(-10, 10)
}, {
"$type":
"set_focus_distance",
"focus_distance":
TDWUtils.get_distance(cam_pos, cam_aim)
}])
# Add bouncing objects.
random.shuffle(self.toy_records)
for i in range(random.randint(2, 6)):
toy_id = self.get_unique_id()
pos = TDWUtils.get_random_point_in_circle(center=np.array(
[0, 0, 0]),
radius=1.5)
pos[1] = random.uniform(0.7, 2)
record = self.toy_records[i]
commands.extend(
self.add_physics_object(
record=record,
position=TDWUtils.array_to_vector3(pos),
rotation=TDWUtils.VECTOR3_ZERO,
mass=random.uniform(0.5, 4),
dynamic_friction=random.uniform(0.1, 0.7),
static_friction=random.uniform(0.1, 0.7),
bounciness=random.uniform(0.7, 1),
o_id=toy_id))
s = TDWUtils.get_unit_scale(record) * random.uniform(0.85, 1.12)
look_at = TDWUtils.array_to_vector3(
TDWUtils.get_random_point_in_circle(center=np.array([0, 0, 0]),
radius=5))
# Scale to "toy size" and look at a random point on the ground.
commands.extend([{
"$type": "scale_object",
"scale_factor": {
"x": s,
"y": s,
"z": s
},
"id": toy_id
}, {
"$type": "object_look_at_position",
"position": look_at,
"id": toy_id
}])
# Rotate the object randomly.
for axis in ["yaw", "roll"]:
commands.append({
"$type": "rotate_object_by",
"angle": random.uniform(-15, 15),
"id": toy_id,
"axis": axis,
"is_world": False
})
# Apply a force.
commands.append({
"$type": "apply_force_magnitude_to_object",
"magnitude": random.uniform(20, 40),
"id": toy_id
})
return commands
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -1,
"y": 0.9,
"z": -1
},
look_at=TDWUtils.array_to_vector3(
[-0.1, 0.5, -0.1]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table_id = self.add_object(model_name="coffee_table_glass_round",
position={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="am151_043_00",
position={
"x": 0.15,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 30,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b04_tea_cup",
position={
"x": 0.3,
"y": top[1],
"z": 0.1
},
library="models_full.json")
self.add_object(model_name="cgaxis_models_65_06_vray",
position={
"x": -0.17,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 24,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([-0.1, 0.45, -0.1])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"sadnwich1",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
""" Generate room using COCO_TDW dataset """
objects_in_scene = 15
object_ids = []
# Get Category-Object mapping
TDW_COCO_models = TDW_relationships.get_COCO_TDW_mapping()
# print("TDWCOCO:", TDW_COCO_models)
# print("KEYS:", TDW_COCO_models.keys())
# Gets COCO categories co-occurring in a scene
# +5 is for dealing with failed object insertion attempts
COCO_configurations = msCOCO_matrix.get_max_co_occurrence(5, int(objects_in_scene + 5))
configuration_1 = COCO_configurations[0]
print("Config 1:", configuration_1)
# TDW models/objects
objects = []
for COCO_object in configuration_1:
print(COCO_object)
print(COCO_object.split())
if len(COCO_object.split()) > 1:
COCO_object = COCO_object.split()[-1]
print(COCO_object)
# Check if COCO category is a key in the COCO-TDW mapping
if COCO_object in TDW_COCO_models.keys():
# Gets random TDW model (from COCO-to-TDW map) based on COCO category key
print(TDW_COCO_models[COCO_object])
model = TDW_COCO_models[COCO_object][random.randint(0, len(TDW_COCO_models[COCO_object]) - 1)]
objects.append(model)
print("COCO to TDW objects:", objects)
# print(len(objects))
# Stores object categories that other objects can be placed upon (e.g. table, chair, couch, bed)
surface_properties_list = TDW_COCO_models['table'] + TDW_COCO_models['chair'] + \
TDW_COCO_models['bed'] + TDW_COCO_models['couch'] + \
TDW_COCO_models['bench'] + TDW_COCO_models['refrigerator']
surface_categories = []
for surface_properties in surface_properties_list:
surface_categories.append(surface_properties[0])
print("Surface Categories:", surface_categories)
# Stores the actual surface object instances/ids alongside number of objects on the surface
surface_object_ids = {}
self.start()
positions_list = [] # Stores current model locations and radii
scene_dimensions = [] # Store scene/environment dimensions
init_setup_commands = [{"$type": "set_screen_size",
"width": 640,
"height": 481},
{"$type": "set_render_quality",
"render_quality": 1}]
self.communicate(init_setup_commands)
scene_lib = SceneLibrarian()
# Disable physics when adding in new objects (objects float)
self.communicate({"$type": "simulate_physics",
"value": False})
for scene in scenes[1:]:
# Load in scene
# print("Scene", scene[0])
if scene[3] == "interior" and scene[0] == "box_room_2018":
self.start()
scene_record = scene_lib.get_record(scene[0])
self.communicate({"$type": "add_scene",
"name": scene_record.name,
"url": scene_record.get_url()})
# Gets dimensions of environments (e.g. inside, outside) in the scene
# This command returns environment data in the form of a list of serialized byte arrays
scene_bytes = self.communicate({"$type": "send_environments",
"frequency": "once"})
# Iterating through data and parsing byte array
# Ignoring the last element (the frame count)
for b in scene_bytes[:-1]:
e = Environments(b)
for i in range(e.get_num()):
center = e.get_center(i)
bounds = e.get_bounds(i)
env_id = e.get_id(i)
scene_dimensions = [center, bounds, env_id] # Center, bounds are tuples
# Must come before set_pass_masks
avatar_position = TDWUtils.array_to_vector3([0.9 * scene_dimensions[1][0] / 2,
scene_dimensions[1][1] / 2,
0])
# print("Avatar Position:", avatar_position)
self.communicate(TDWUtils.create_avatar(avatar_id="avatar",
position=avatar_position,
look_at={"x": 0,
"y": scene_dimensions[0][1] / 2,
"z": 0}))
# Set collision mode
self.communicate({"$type": "set_avatar_collision_detection_mode",
"mode": "continuous_speculative",
"avatar_id": "avatar"})
# Alter FOV
self.communicate({"$type": "set_field_of_view",
"field_of_view": 80,
"avatar_id": "avatar"})
# # Gets rid of header (Model: Category)
# objects = TDW_COCO_models[1:]
# random.shuffle(objects)
obj_count = 0
obj_overlaps = 0 # Number of failed attempts to place object due to over-dense objects area
while obj_count < objects_in_scene and obj_overlaps < 5:
# Handles if object has been added to a flat surface
added_to_surface = False
print("Object COUNT:", obj_count)
# Need to have random position for Bounds Data to return meaningful info
valid_obj_pos = {"x": random.uniform(-1 * scene_dimensions[1][0] / 2,
0.5 * scene_dimensions[1][0] / 2),
"y": scene_dimensions[1][1] / 4,
"z": random.uniform(-0.9 * scene_dimensions[1][2] / 2,
0.9 * scene_dimensions[1][2] / 2)}
print("First random position")
# Add in the object at random position
# Object will later be removed or updated accordingly after performing collision calculations
record = ModelLibrarian(library="models_full.json").get_record(objects[obj_count][0])
print("Record gotten")
print(objects[obj_count][0])
o_id = self.communicate({"$type": "add_object",
"name": objects[obj_count][0],
"url": record.get_url(),
"scale_factor": record.scale_factor,
"position": valid_obj_pos,
"rotation": TDWUtils.VECTOR3_ZERO,
"category": record.wcategory,
"id": obj_count})
print("Random first add")
# Returns bound data for added object
bounds_data = self.communicate({"$type": "send_bounds",
"frequency": "once"})
print("Bounds returned")
# Appends object, with information on position and obj_radius, to positions_list
# Length of buffer array should be 1
# print("Bounds Data:", bounds_data)
for b in bounds_data[:-1]:
# print("Buffer Loop:", b)
b_id = OutputData.get_data_type_id(b)
if b_id == "boun":
# print("BOUNDS")
o = Bounds(b)
# print("# of Objects:", o.get_num())
# print("# of Failed Attempts:", obj_overlaps)
# print("Buffer Array:", b)
# print("Bounds Object:", o)
for i in range(o.get_num()):
print("Object ID:", o.get_id(i))
print("obj_count:", obj_count)
print("Object:", objects[obj_count][0], "Category:", objects[obj_count][1])
# print("Object Center:", o.get_center(i))
# Only want to compute valid_position for object we are about to add
# Skip any computation if this is not the case
if o.get_id(i) != obj_count:
continue
# Useful for detecting if object fits in environment
# print("Calculating if object fits in environment")
width = distance.euclidean(o.get_left(i), o.get_right(i))
depth = distance.euclidean(o.get_front(i), o.get_back(i))
height = distance.euclidean(o.get_top(i), o.get_bottom(i))
# print("Width:", width)
# print("Depth:", depth)
# ("Height:", height)
# Useful for avoiding object overlap
# print("Calculating Object Bounds")
center_to_top = distance.euclidean(o.get_center(i), o.get_top(i))
center_to_bottom = distance.euclidean(o.get_center(i), o.get_bottom(i))
center_to_left = distance.euclidean(o.get_center(i), o.get_left(i))
center_to_right = distance.euclidean(o.get_center(i), o.get_right(i))
center_to_front = distance.euclidean(o.get_center(i), o.get_front(i))
center_to_back = distance.euclidean(o.get_center(i), o.get_back(i))
# Max object radius (center to diagonal of bounding box)
obj_radius = \
max(math.sqrt(center_to_top ** 2 + center_to_left ** 2 + center_to_front ** 2),
math.sqrt(center_to_top ** 2 + center_to_right ** 2 + center_to_front ** 2),
math.sqrt(center_to_top ** 2 + center_to_left ** 2 + center_to_back ** 2),
math.sqrt(center_to_top ** 2 + center_to_right ** 2 + center_to_back ** 2),
math.sqrt(center_to_bottom ** 2 + center_to_left ** 2 + center_to_front ** 2),
math.sqrt(center_to_bottom ** 2 + center_to_right ** 2 + center_to_front ** 2),
math.sqrt(center_to_bottom ** 2 + center_to_left ** 2 + center_to_back ** 2),
math.sqrt(center_to_bottom ** 2 + center_to_right ** 2 + center_to_back ** 2))
# print("Obj_Radius:", obj_radius)
# Set sweeping radius, based on scene plane dimensions
l_radius = random.uniform(0, min(0.5 * scene_dimensions[1][0] / 2,
0.5 * scene_dimensions[1][2] / 2))
# Checking that object fits in scene viewing
if (width > min(0.7 * scene_dimensions[1][0], 0.7 * scene_dimensions[1][2]) or
depth > min(0.7 * scene_dimensions[1][0], 0.7 * scene_dimensions[1][2]) or
height > 0.7 * scene_dimensions[1][1]):
print("Object does not fit in scene")
self.communicate([{"$type": "send_images",
"frequency": "never"},
{"$type": "destroy_object",
"id": obj_count}])
# Ensures next attempt to load in item is not the same item as before
random.shuffle(objects)
break
# Not possible to find valid object position -- too many overlapping objects
elif (not self._get_object_position(scene_dimensions=scene_dimensions,
object_positions=positions_list,
object_to_add_radius=obj_radius,
max_tries=20,
location_radius=l_radius)[0]):
print("Could not calculate valid object location")
self.communicate([{"$type": "send_images",
"frequency": "never"},
{"$type": "destroy_object",
"id": obj_count}])
obj_overlaps += 1
# Ensures next attempt to load in item is not the same item as before
random.shuffle(objects)
break
# Find appropriate, non-overlapping object position
# Reset object position to the valid position
else:
print("Object fits in scene")
# Check if object fits on table, chair, couch, etc.
# Add object if it fits, place it somewhere on top of the surface
for surface_id in surface_object_ids.keys():
print("Surface ID:", surface_id)
# Skip placement feasibility if the object is already a surface-type object
# Ex. no chair on top of a table
if objects[obj_count][0] in surface_categories:
print("Object: %s is already a surface object" % objects[obj_count][0])
break
# Check how many objects are on surface
if surface_object_ids[surface_id] >= 3:
print("Too many objects on surface")
print("From surface objects dict:", surface_object_ids[surface_id])
continue
surface_bounds = self.get_bounds_data(surface_id)
surface_area = distance.euclidean(surface_bounds.get_left(0),
surface_bounds.get_right(0)) * \
distance.euclidean(surface_bounds.get_front(0),
surface_bounds.get_back(0))
obj_area = width * height
if obj_area < surface_area:
s_center_to_top = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_top(0))
s_center_to_bottom = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_bottom(0))
s_center_to_left = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_left(0))
s_center_to_right = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_right(0))
s_center_to_front = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_front(0))
s_center_to_back = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_back(0))
surface_radius = \
max(math.sqrt(
s_center_to_top ** 2 + s_center_to_left ** 2 + s_center_to_front ** 2),
math.sqrt(
s_center_to_top ** 2 + s_center_to_right ** 2 + s_center_to_front ** 2),
math.sqrt(
s_center_to_top ** 2 + s_center_to_left ** 2 + s_center_to_back ** 2),
math.sqrt(
s_center_to_top ** 2 + s_center_to_right ** 2 + s_center_to_back ** 2),
math.sqrt(
s_center_to_bottom ** 2 + s_center_to_left ** 2 + s_center_to_front ** 2),
math.sqrt(
s_center_to_bottom ** 2 + s_center_to_right ** 2 + s_center_to_front ** 2),
math.sqrt(
s_center_to_bottom ** 2 + s_center_to_left ** 2 + s_center_to_back ** 2),
math.sqrt(
s_center_to_bottom ** 2 + s_center_to_right ** 2 + s_center_to_back ** 2))
print("Surface-type object")
self.communicate({"$type": "destroy_object",
"id": obj_count})
# Adding the object to the top of the surface
on_pos = surface_bounds.get_top(0)
on_y = on_pos[1]
on_pos = TDWUtils.get_random_point_in_circle(np.array(on_pos),
0.7 * surface_radius)
on_pos[1] = on_y
on_pos = TDWUtils.array_to_vector3(on_pos)
on_rot = {"x": 0, "y": random.uniform(-45, 45), "z": 0}
# Add the object.
print("Model Name on Surface:", objects[obj_count][0])
record = ModelLibrarian(library="models_full.json").get_record(
objects[obj_count][0])
on_id = self.communicate({"$type": "add_object",
"name": objects[obj_count][0],
"url": record.get_url(),
"scale_factor": record.scale_factor,
"position": on_pos,
"rotation": on_rot,
"category": record.wcategory,
"id": obj_count})
obj_count += 1
surface_object_ids[surface_id] += 1
object_ids.append(obj_count)
print("Object added on top of surface")
added_to_surface = True
# Breaking out of surface objects loop
break
if added_to_surface:
print("Breaking out of object loop")
# Breaking out of object loop
break
print("Post-surface")
valid_obj_pos = self._get_object_position(scene_dimensions=scene_dimensions,
object_positions=positions_list,
object_to_add_radius=obj_radius,
max_tries=20,
location_radius=l_radius)[1]
print("Position calculated")
positions_list.append(ObjectPosition(valid_obj_pos, obj_radius))
self.communicate([{"$type": "send_images",
"frequency": "never"},
{"$type": "destroy_object",
"id": obj_count}])
added_object_id = self.communicate({"$type": "add_object",
"name": objects[obj_count][0],
"url": record.get_url(),
"scale_factor": record.scale_factor,
"position": valid_obj_pos,
"rotation": {"x": 0, "y": 0, "z": 0},
"category": record.wcategory,
"id": obj_count})
# print("Object ID:", added_object_id)
print("Regular object add")
object_ids.append(added_object_id)
# If TDW model belongs to surface categories, store id_information
if objects[obj_count][0] in surface_categories:
surface_object_ids[obj_count] = 0
# Rotate the object randomly
print("Rotating object")
self.communicate({"$type": "rotate_object_by",
"angle": random.uniform(-45, 45),
"axis": "yaw",
"id": obj_count,
"is_world": True})
# Minimal rotating for position differences
# Don't rotate the object if doing so will result in overlap into scene
if not (o.get_bottom(i)[1] < 0 or o.get_top(i)[1] > 0.9 * scene_dimensions[1][1]):
pitch_angle = random.uniform(-45, 45)
self.communicate({"$type": "rotate_object_by",
"angle": pitch_angle,
"axis": "pitch",
"id": obj_count,
"is_world": True})
roll_angle = random.uniform(-45, 45)
self.communicate({"$type": "rotate_object_by",
"angle": roll_angle,
"axis": "roll",
"id": obj_count,
"is_world": True})
# Don't need this for just changing positions
# Setting random materials/textures
# Looping through sub-objects and materials
sub_count = 0
for sub_object in record.substructure:
# Loop through materials in sub-objects
for j in range(len(sub_object)):
# Get random material and load in
material = random.choice(materials[1:])
self.load_material(material)
print("Material loaded")
# Set random material on material of sub-object
self.communicate({"$type": "set_visual_material",
"material_index": j,
"material_name": material[0],
"object_name": sub_object['name'],
"id": obj_count})
print("Material set")
sub_count += 1
if sub_count > 10:
break
break
print("Updating count")
obj_count += 1
print("Breaking out of object_id loop")
break
# Break out of buffer loop
print("Breaking out of buffer loop")
break
# Move onto next iteration of while loop (next object to load in)
print("Object added - next while loop iteration")
continue
# for i in range(200):
# self.communicate({"$type": "simulate_physics",
# "value": False})
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
# Capture scene
# NOTE: THESE SCENES GET REPLACED IN THE TARGET DIRECTORY
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": {"x": 0,
"y": scene_dimensions[0][1] / 2,
"z": 0}})
images = Images(scene_data[0])
TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=path)
print("Object ids:", object_ids)
def _get_object_position(scene_dimensions: List[Tuple], object_positions: List[ObjectPosition],
object_to_add_radius: float, max_tries: int = 1000, location_radius: float = 2) -> \
Dict[str, float]:
"""
Try to get a valid random position that doesn't interpenetrate with other objects.
:param object_positions: The positions and radii of all objects so far that will be added to the scene.
:param max_tries: Try this many times to get a valid position before giving up.
:param location_radius: The radius to pick a position in.
:return: A valid position that doesn't interpenetrate with other objects.
"""
o_pos = TDWUtils.get_random_point_in_circle(center=np.array([random.uniform(-0.9 * scene_dimensions[1][0] / 2,
0.9 * scene_dimensions[1][0] / 2),
0,
random.uniform(-0.9 * scene_dimensions[1][2] / 2,
0.9 * scene_dimensions[1][2] / 2)]),
radius=location_radius)
# Scale circular point --> elliptic point for non-square rooms
# Width > Depth of scene
if scene_dimensions[1][0] > scene_dimensions[1][2]:
o_pos[0] = o_pos[0] * scene_dimensions[1][0] / scene_dimensions[1][2]
# Depth > Width of scene
else:
o_pos[2] = o_pos[2] * scene_dimensions[1][2] / scene_dimensions[1][0]
# Start with everything on ground
# # Set height value
# o_pos[1] = random.uniform(0, 0.9 * scene_dimensions[1][1] / 3)
o_pos = TDWUtils.array_to_vector3(o_pos)
# print("O_Pos:", o_pos)
# Pick a position away from other objects.
ok = False
count = 0
while not ok and count < max_tries:
count += 1
ok = True
for o in object_positions:
# If the object is too close to another object, try another position.
# Also check that object bounds are generated within reasonable viewing window
# Check Radius of old objects as well
if (TDWUtils.get_distance(o.position, o_pos) <= 1.1 * o.obj_radius or
TDWUtils.get_distance(o.position, o_pos) <= 1.1 * object_to_add_radius or
o_pos["x"] < -0.9 * scene_dimensions[1][0] / 2 or
o_pos["x"] > 0.8 * scene_dimensions[1][0] / 2 or
o_pos["y"] < 0 or
o_pos["y"] > scene_dimensions[1][1] / 2 or
o_pos["z"] < -0.8 * scene_dimensions[1][2] / 2 or
o_pos["z"] > 0.8 * scene_dimensions[1][2] / 2):
ok = False
# o_pos = TDWUtils.get_random_point_in_circle(center=np.array([
# random.uniform(-0.9 * scene_dimensions[1][0] / 2
# , 0.9 * scene_dimensions[1][0] / 2),
# 0,
# random.uniform(-0.9 * scene_dimensions[1][2] / 2
# , 0.9 * scene_dimensions[1][2] / 2)]),
# radius=location_radius)
o_pos = TDWUtils.get_random_point_in_circle(center=np.array([0, 0, 0]),
radius=location_radius)
# Scale circular point --> elliptic point for non-square rooms
# Width > Depth of scene
if scene_dimensions[1][0] > scene_dimensions[1][2]:
o_pos[0] = o_pos[0] * scene_dimensions[1][0] / scene_dimensions[1][2]
# Depth > Width of scene
else:
o_pos[2] = o_pos[2] * scene_dimensions[1][2] / scene_dimensions[1][0]
# Don't change height for now
# o_pos[1] = random.uniform(0, 0.9 * scene_dimensions[1][1])
o_pos = TDWUtils.array_to_vector3(o_pos)
print("O_Pos:", o_pos)
return [ok, o_pos]
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": -0.8, "y": 1.6, "z": 0.6},
look_at=TDWUtils.array_to_vector3([0, 0.4, 0]),
avatar_id="avatar"))
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
table_id = self.add_object(model_name="glass_table_round",
position={"x": 0, "y": 0, "z": 0},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="b03_orange",
position={"x": 0, "y": top[1], "z": 0},
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.17)
position[1] = top[1]
self.add_object(model_name="b03_orange",
position={"x": 0, "y": top[1], "z": 0},
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.4)
position[1] = top[1]
self.add_object(model_name="b04_banana",
position=TDWUtils.array_to_vector3(position),
rotation={"x": 0, "y": 40, "z": 0},
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.4)
position[1] = top[1]
self.add_object(model_name="b04_banana",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.5)
position[1] = top[1]
self.add_object(model_name="red_apple",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.6)
position[1] = top[1]
self.add_object(model_name="vk0076_fruitfork",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([0, 0.4, 0])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "fruit_table", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior")