def run(self):
self.start()
width = 12
length = 12
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(width, length))
# Disable physics.
self.communicate({"$type": "simulate_physics", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -5,
"y": 10,
"z": -2
},
look_at=TDWUtils.VECTOR3_ZERO))
# Create the dining table.
self.dining_table()
# Create the sofa.
self.sofa(width, length)
# Bake the NavMesh.
self.communicate({"$type": "bake_nav_mesh", "carve_type": "none"})
# Get a random position on the NavMesh.
position = TDWUtils.get_random_position_on_nav_mesh(
self, width, length)
position = TDWUtils.array_to_vector3(position)
# Add a side table.
self.side_table(position)
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(10, 10))
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
self.communicate(self.get_add_hdri_skybox("autumn_hockey_4k"))
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": 3, "y": 1.3, "z": 0},
look_at=TDWUtils.array_to_vector3([0, 0.5, 0]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("grass_countryside")
print(record)
self.communicate({"$type": "add_material", "name": "grass_countryside", "url": record.get_url()})
self.communicate({"$type": "set_proc_gen_floor_material", "name": "grass_countryside"})
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
self.add_object(model_name="b05_clochild4",
position={"x": 0, "y": 0, "z": 0},
rotation={"x": 0, "y": 30, "z": 0},
library="models_full.json")
self.add_object(model_name="baseballbat",
position={"x": 0.5, "y": 0, "z": 0.3},
rotation={"x": 0, "y": 30, "z": 0},
library="models_full.json")
self.add_object(model_name="base-ball",
position={"x": 0.4, "y": 0, "z": 0.22},
rotation={"x": 0, "y": 30, "z": 10},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([0, 0.5, 0])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "kid_baseball", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def run(self):
self.start()
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(12, 12))
# Create the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 3,
"z": -4
},
look_at=TDWUtils.VECTOR3_ZERO))
# Set the pass mask to _depth only.
# Get an image.
resp = self.communicate([
self.get_add_object("rh10", object_id=0), {
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_depth"]
}, {
"$type": "send_images",
"frequency": "once"
}
])
images = Images(resp[0])
# Get the depth values of each pixel.
depth = TDWUtils.get_depth_values(images.get_image(0))
print(depth)
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(32, 32))
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": -1.5, "y": 1.5, "z": -15},
look_at=TDWUtils.array_to_vector3([-1.7, 0.7, -2.3]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("concrete_chipped_cracked")
self.communicate({"$type": "add_material", "name": "concrete_chipped_cracked", "url": record.get_url()})
self.communicate({"$type": "set_proc_gen_floor_material", "name": "concrete_chipped_cracked"})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
self.communicate({"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"})
bus = self.add_object(model_name="b06_bus_new",
position={"x": -2, "y": 0, "z": 0},
rotation={"x": 0, "y": -70, "z": 0},
library="models_full.json")
bus_bounds = self.get_bounds_data(bus)
bus_front = bus_bounds.get_front(0)
print(bus_front)
self.add_object(model_name="3dscan_man_004",
position={"x": -3.7, "y": 0, "z": -7.3},
rotation={"x": 0, "y": 0, "z": 0},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([-1.7, 0.7, -2.3])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "bus",
output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def get_scene_initialization_commands(self) -> List[dict]:
# Load the proc-gen room.
# Disable images.
return [{
"$type": "load_scene",
"scene_name": "ProcGenScene"
},
TDWUtils.create_empty_room(100, 100)]
def _get_scene_init_commands(self, scene: str = None, layout: int = None, room: int = -1) -> List[dict]:
commands = [{"$type": "load_scene",
"scene_name": "ProcGenScene"},
TDWUtils.create_empty_room(4, 4)]
self.o_id, o_commands = self._add_object("trunck",
position={"x": 0, "y": 0, "z": 1})
commands.extend(o_commands)
return commands
def run(self):
self.start()
commands = [TDWUtils.create_empty_room(self.size, self.size)]
commands.extend(
TDWUtils.create_avatar(position={
"x": 0,
"y": 1.5,
"z": 0
},
avatar_id="a"))
commands.extend([{
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img"]
}, {
"$type": "send_images",
"frequency": "always"
}])
self.communicate(commands)
# Listen for keys.
# Turn left.
self.listen(key="left",
commands={
"$type": "rotate_sensor_container_by",
"axis": "yaw",
"angle": self.angle * -1,
"sensor_name": "SensorContainer",
"avatar_id": "a"
})
# Turn right.
self.listen(key="right",
commands={
"$type": "rotate_sensor_container_by",
"axis": "yaw",
"angle": self.angle,
"sensor_name": "SensorContainer",
"avatar_id": "a"
})
# Quit the application.
self.listen(key="esc", function=self.stop)
# Continue until the quit key is pressed.
i = 0
while not self.done:
# Listen for keyboard input. Receive output data.
resp = self.communicate([])
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
# Save images.
if r_id == "imag":
TDWUtils.save_images(
images=Images(r),
output_directory=self.images_directory,
filename=TDWUtils.zero_padding(i, width=4))
# Increment the image number.
i += 1
self.communicate({"$type": "terminate"})
def run(self):
# Create the output directory.
output_directory = "example_output"
if os.path.exists(output_directory):
shutil.rmtree(output_directory)
sleep(0.5)
os.mkdir(output_directory)
print(f"Images will be saved to: {output_directory}")
self.start()
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(12, 12))
# Add the objects.
lamp_id = self.add_object("alma_floor_lamp",
position={"x": 1, "y": 0, "z": 0},
rotation={"x": 0, "y": 90, "z": 0})
self.add_object("live_edge_coffee_table",
position={"x": 1.5, "y": 0, "z": 1.5},
rotation={"x": 0, "y": 30, "z": 0})
self.add_object("small_table_green_marble",
position={"x": -0.9, "y": 0, "z": -1.35})
# Create the avatar.
self.communicate({"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"})
# Try to find a valid position on the NavMesh.
x, y, z = TDWUtils.get_random_position_on_nav_mesh(self, 12, 12)
# Teleport the avatar to the valid position.
# Apply a force to the lamp.
# Set the pass masks to _img.
# Enable image capture.
self.communicate([{"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {"x": x, "y": 1.5, "z": z}},
{"$type": "apply_force_to_object",
"force": {"x": 2, "y": 1, "z": 0},
"id": lamp_id},
{"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img", "_id"]},
{"$type": "send_images",
"frequency": "always"},
])
# Capture 100 images.
for i in range(100):
# Look at the lamp.
resp = self.communicate({"$type": "look_at",
"avatar_id": "a",
"object_id": lamp_id,
"use_centroid": True})
images = Images(resp[0])
# Save the image.
TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=output_directory)
def run(self):
self.start()
# Create the room.
self.communicate(TDWUtils.create_empty_room(20, 20, 1))
# Run a trial.
# self.trial(1.5, 0.05, 0.2, "./fall_roll_ramp_collide_output/images output/basket/run1_m1.5_s0.05_b0.2_blue", 200)
self.trial(1.25, 0.05, 0.1, "./output/images output/collision2", 200)
def create_scene(self):
"""
Initialize a blank room.
"""
self.start()
commands = [TDWUtils.create_empty_room(12, 12)]
commands.extend(TDWUtils.create_avatar(position={"x": 5, "y": 10, "z": 5}, look_at=TDWUtils.VECTOR3_ZERO))
self.communicate(commands)
def _initialize_scene(self, c: Controller) -> List[dict]:
return [{"$type": "load_scene"},
TDWUtils.create_empty_room(self._room_size[0], self._room_size[1]),
{"$type": "destroy_all_objects"},
self._reverb,
{"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"},
{"$type": "add_environ_audio_sensor"},
{"$type": "toggle_image_sensor"}]
def next(self):
"""
Starts a new trial of an experiment. Populates a room with randomly placed lamps and a single avatar.
"""
# Get a random avatar position.
ret_dict = {}
avatar_position = [random.uniform(-self.room_dim * 0.33, self.room_dim * 0.33), 0,
random.uniform(-self.room_dim * 0.33, self.room_dim * 0.33)]
config_actions = []
# Begin placing objects in the room.
current_objects = []
for good_obj in range(self.num_good_objects):
current_objects.append(
self.get_object_dict(self.find_open_position(current_objects, avatar_position), self.good_obj_name))
for bad_obj in range(self.num_bad_objects):
current_objects.append(
self.get_object_dict(self.find_open_position(current_objects, avatar_position), self.bad_obj_name))
# Add the TDW configurations to a list to be sent inside TDWGenerator.restart_trial()
# Initialize the scene.
# Set the render quality to as low as possible.
# Tell the build to return transform, rigidbody, and image data.
config_actions.append([{'$type': "load_scene", 'scene_name': "ProcGenScene"}])
config_actions.append([{"$type": "send_transforms", "frequency": "always"},
{"$type": "send_collisions", "frequency": "always"},
{"$type": "send_rigidbodies", "frequency": "always"},
{"$type": "send_images", "frequency": "always"},
{"$type": "set_render_quality", "render_quality": 0},
{"$type": "set_screen_size", "width": 256, "height": 256},
{"$type": "set_img_pass_encoding", "value": False},
{"$type": "set_post_process", "value": False},
TDWUtils.create_empty_room(self.room_dim, self.room_dim)])
for c_o in current_objects:
config_actions.append(self.get_add_object(model_name=c_o["name"],
object_id=self.get_unique_id(),
position=c_o["position"],
rotation=c_o["orientation"]))
config_actions.append([{"$type": "create_avatar", "type": self.avatar_type, "id": "a",
"position": avatar_position, "rotation": [0.0, 0.0, 0.0]},
{"$type": "teleport_avatar_to",
"position": {"x": avatar_position[0], "y": 1.0, "z": avatar_position[2]}}
])
config_actions.append([{"$type": "set_pass_masks", "pass_masks": ["_img"]}])
config_actions.append([{"$type": "send_id_pass_segmentation_colors", "frequency": "always"}])
config_actions.append([{"$type": "set_avatar_drag", "drag": 80, "angular_drag": 4}])
# Set up the environment and start up the trial
ret_dict['latents'] = self.restart_trial(config_actions)
return ret_dict
def run(self):
self.start()
depth_pass = "******"
# Create an empty room.
# Set the screen size.
commands = [TDWUtils.create_empty_room(12, 12),
{"$type": "set_screen_size",
"width": 512,
"height": 512}]
# Add the avatar.
commands.extend(TDWUtils.create_avatar(position={"x": 1.57, "y": 3, "z": 3.56}, look_at=TDWUtils.VECTOR3_ZERO))
# Add an object.
# Request images and camera matrices.
commands.extend([self.get_add_object("trunck", object_id=0),
{"$type": "set_pass_masks",
"pass_masks": [depth_pass]},
{"$type": "send_images"},
{"$type": "send_camera_matrices"}])
resp = self.communicate(commands)
depth_image = None
camera_matrix = None
images = None
for i in range(len(resp) - 1):
r_id = OutputData.get_data_type_id(resp[i])
# Get the image.
if r_id == "imag":
images = Images(resp[i])
for j in range(images.get_num_passes()):
if images.get_pass_mask(j) == depth_pass:
depth_image = images.get_image(j)
# Get the camera matrix.
elif r_id == "cama":
camera_matrix = CameraMatrices(resp[i]).get_camera_matrix()
# Save the image.
TDWUtils.save_images(images=images, output_directory="D:/depth_shader", filename="0", append_pass=True)
# Get the depth values of each pixel.
depth = TDWUtils.get_depth_values(image=depth_image, width=images.get_width(), height=images.get_height())
print(np.min(depth), np.max(depth))
print(depth)
np.save("depth", depth)
np.save("camera_matrix", camera_matrix)
# Get a point cloud and write it to disk.
point_cloud_filename = "point_cloud.txt"
print(f"Point cloud saved to: {Path(point_cloud_filename)}")
TDWUtils.get_point_cloud(depth=depth, filename=point_cloud_filename, camera_matrix=camera_matrix)
# Show the depth values.
plt.imshow(depth)
plt.show()
self.communicate({"$type": "terminate"})
def run(self):
self.start()
# Create the room.
self.communicate(TDWUtils.create_empty_room(12, 12))
# Run a series of trials.
for j in range(5):
self.trial()
# Terminate the build.
self.communicate({"$type": "terminate"})
def run(self):
self.start()
# Create the room.
self.communicate(TDWUtils.create_empty_room(12, 12))
# Run a trial.
self.trial(0.1, 0.12, 0.6, 0.8, 0.75, 0.1, ["obj"])
# Run a trial with different friction and bounciness parameters.
# Listen for environment collisions.
self.trial(0.9, 0.85, 1.0, 0.2, 0.75, 1.0, ["obj", "env"])
def run(self, drag_on_land, angular_drag_on_land):
self.start()
self.communicate(TDWUtils.create_empty_room(40, 40))
# Create the avatar.
self.communicate(TDWUtils.create_avatar(avatar_type="A_Simple_Body",
position={"x": 0, "y": 1, "z": 0}))
# When the y value of the avatar's position is this value, the avatar is on the ground.
ground_y = 1
# When the y value of the avatar's position exceeds ground_y by this much, it is in the air.
in_air_threshold = 0.05
# If true, the avatar is in the air.
is_in_air = False
# Set high default drag values.
# Set a low mass.
# Apply a force.
# Request avatar data.
resp = self.communicate([{"$type": "set_avatar_drag",
"avatar_id": "a",
"drag": 30,
"angular_drag": 80},
{"$type": "send_avatars",
"frequency": "always"},
{"$type": "apply_force_to_avatar",
"avatar_id": "a",
"direction": {"x": 0.3, "y": 0.7, "z": 0},
"magnitude": 900}])
for i in range(500):
x, y, z = AvatarSimpleBody(resp[0]).get_position()
# If the avatar just launched in to the air, set its drag values to 0.
if y > ground_y + in_air_threshold and not is_in_air:
is_in_air = True
resp = self.communicate({"$type": "set_avatar_drag",
"avatar_id": "a",
"drag": 0,
"angular_drag": 0})
print("The avatar is in the air on frame: " + str(Controller.get_frame(resp[-1])))
# If the avatar just landed on the ground, set drag values.
elif y <= ground_y + in_air_threshold and is_in_air:
resp = self.communicate({"$type": "set_avatar_drag",
"avatar_id": "a",
"drag": drag_on_land,
"angular_drag": angular_drag_on_land})
is_in_air = False
print("The avatar is on the ground on frame: " + str(Controller.get_frame(resp[-1])))
# Wait.
else:
resp = self.communicate({"$type": "do_nothing"})
def run(self):
# Create the scene.
# Set image encoding globals.
self.start()
commands = [TDWUtils.create_empty_room(12, 12),
{"$type": "set_screen_size",
"width": 128,
"height": 128},
{"$type": "set_img_pass_encoding",
"value": False}]
# Create the avatar.
commands.extend(TDWUtils.create_avatar(position={"x": 2.478, "y": 1.602, "z": 1.412},
look_at=TDWUtils.VECTOR3_ZERO,
avatar_id="a"))
# Enable all pass masks. Request an image for this frame only.
commands.extend([{"$type": "set_pass_masks",
"pass_masks": ["_img", "_id", "_category", "_mask", "_depth", "_normals", "_flow",
"_depth_simple", "_albedo"],
"avatar_id": "a"},
{"$type": "send_images",
"ids": ["a"],
"frequency": "once"}])
# Add objects.
commands.append(self.get_add_object("small_table_green_marble",
position=TDWUtils.VECTOR3_ZERO,
rotation=TDWUtils.VECTOR3_ZERO,
object_id=0))
commands.append(self.get_add_object("rh10",
position={"x": 0.7, "y": 0, "z": 0.4},
rotation={"x": 0, "y": 30, "z": 0},
object_id=1))
commands.append(self.get_add_object("jug01",
position={"x": -0.3, "y": 0.9, "z": 0.2},
rotation=TDWUtils.VECTOR3_ZERO,
object_id=3))
commands.append(self.get_add_object("jug05",
position={"x": 0.3, "y": 0.9, "z": -0.2},
rotation=TDWUtils.VECTOR3_ZERO,
object_id=4))
# Send the commands.
resp = self.communicate(commands)
# Save the images.
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
if r_id == "imag":
# Save the images.
TDWUtils.save_images(output_directory="dist", images=Images(r), filename="0")
print(f"Images saved to: {Path('dist').resolve()}")
# Stop the build.
self.communicate({"$type": "terminate"})
def run(self):
self.start()
self.communicate(TDWUtils.create_empty_room(20, 20))
self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 3, "z": -6},
look_at=TDWUtils.VECTOR3_ZERO))
model_name = "rh10"
z = -3
x = -1.5
print("With the add_object command (complex syntax but you have maximum control):")
record = ModelLibrarian().get_record(model_name)
self.communicate({"$type": "add_object",
"name": model_name,
"url": record.get_url(),
"scale_factor": record.scale_factor,
"position": {"x": x, "y": 0, "z": z},
"rotation": TDWUtils.VECTOR3_ZERO,
"category": record.wcategory,
"id": self.get_unique_id()})
x = 0
print("With the wrapper function Controller.add_object() "
"(easy to use, but you can't add additional commands to this frame):")
self.add_object(model_name=model_name,
position={"x": x, "y": 0, "z": z},
rotation=TDWUtils.VECTOR3_ZERO,
library="models_core.json")
x = 1.5
print("With the wrapper function Controller.get_add_object() "
"(harder to use, but you can add commands to this frame):")
self.communicate(self.get_add_object(model_name=model_name,
object_id=self.get_unique_id(),
position={"x": x, "y": 0, "z": 0},
rotation=TDWUtils.VECTOR3_ZERO,
library="models_core.json"))
print("With the add_object command, minus all optional parameters (the model won't scale properly!):")
self.communicate({"$type": "add_object",
"name": model_name,
"url": record.get_url(),
"id": self.get_unique_id()})
print("With the wrapper function Controller.add_object(), minus all optional parameters:")
self.add_object(model_name)
print("With the wrapper function Controller.get_add_object(), minus all optional parameters:")
self.communicate(self.get_add_object(model_name=model_name,
object_id=self.get_unique_id()))
def run(self):
# Open the image file.
with open("DT4897.jpg", "rb") as f:
image = b64encode(f.read()).decode("utf-8")
# The expected dimensions of the painting.
dimensions = {"x": 479, "y": 332}
self.start()
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Create a painting.
# Apply the texture.
painting_id = self.get_unique_id()
painting_position = {"x": 1, "y": 2, "z": 3}
self.communicate([{
"$type": "create_painting",
"position": painting_position,
"size": {
"x": 5,
"y": 3
},
"euler_angles": {
"x": 0,
"y": 30,
"z": 0
},
"id": painting_id
}, {
"$type": "set_painting_texture",
"id": painting_id,
"dimensions": dimensions,
"image": image
}])
# Create the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 5,
"z": 0
},
look_at=painting_position))
def run(self):
self.start()
self.communicate(TDWUtils.create_empty_room(12, 12))
o_id = self.add_object("rh10")
self.communicate(
TDWUtils.create_avatar(position={
"x": 1,
"y": 3,
"z": 0
}))
self.communicate([{
"$type": "set_mass",
"id": o_id,
"mass": 15
}, {
"$type": "rotate_object_by",
"axis": "pitch",
"id": o_id,
"angle": 45
}, {
"$type": "apply_force_magnitude_to_object",
"id": o_id,
"magnitude": 250.0
}])
for i in range(100):
self.communicate({
"$type": "look_at",
"avatar_id": "a",
"object_id": o_id
})
sleep(1)
# Print the recorded commands to the console.
for commands in self.record:
print(commands)
# Play back all of the commands.
self.playback()
def init_scene(self,
scene: str = None,
layout: int = None,
room: int = None,
goal_room: int = None) -> ActionStatus:
origin = np.array([0, 0, 0])
commands = [{
"$type": "load_scene",
"scene_name": "ProcGenScene"
},
TDWUtils.create_empty_room(12, 12)]
self._add_container(model_name="basket_18inx18inx12iin",
position={
"x": 0.354,
"y": 0,
"z": 0.549
},
rotation={
"x": 0,
"y": -70,
"z": 0
})
# Add target objects in a circle.
num_objects = 10
d_theta = 360 / num_objects
theta = d_theta / 2
pos = np.array([2, 0, 0])
for j in range(num_objects):
object_position = TDWUtils.rotate_position_around(origin=origin,
position=pos,
angle=theta)
self._add_target_object(
"jug05", position=TDWUtils.array_to_vector3(object_position))
theta += d_theta
commands.extend(self._get_scene_init_commands())
resp = self.communicate(commands)
self._cache_static_data(resp=resp)
# Wait for the Magnebot to reset to its neutral position.
status = self._do_arm_motion()
self._end_action()
return status
def run(self):
# Parse the substructure of fridge_large.
record = ModelLibrarian().get_record("fridge_large")
self.start()
self.communicate(TDWUtils.create_empty_room(12, 12))
# Create the object.
o_id = self.add_object("fridge_large")
# Add an avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 1.886,
"z": -0.15
},
look_at=TDWUtils.VECTOR3_ZERO))
# Disable post-processing, so the changes to the material aren't blurry.
self.communicate({"$type": "set_post_process", "value": False})
for quality in ["low", "med", "high"]:
# Set the visual materials.
self.communicate(
TDWUtils.set_visual_material(self,
record.substructure,
o_id,
"parquet_long_horizontal_clean",
quality=quality))
sleep(5)
self.communicate({
"$type": "unload_asset_bundles",
"bundle_type": "materials"
})
self.communicate({
"$type": "unload_asset_bundles",
"bundle_type": "models"
})
def sound20k_set(self) -> None:
"""
Generate a dataset analogous to Sound20K.
"""
sound20k_init_commands = [
{
"$type": "load_scene"
},
TDWUtils.create_empty_room(12, 12), {
"$type": "set_proc_gen_walls_scale",
"walls": TDWUtils.get_box(12, 12),
"scale": {
"x": 1,
"y": 4,
"z": 1
}
}, {
"$type": "set_reverb_space_simple",
"env_id": 0,
"reverb_floor_material": "parquet",
"reverb_ceiling_material": "acousticTile",
"reverb_front_wall_material": "smoothPlaster",
"reverb_back_wall_material": "smoothPlaster",
"reverb_left_wall_material": "smoothPlaster",
"reverb_right_wall_material": "smoothPlaster"
}, {
"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"
}, {
"$type": "add_environ_audio_sensor"
}, {
"$type": "toggle_image_sensor"
}
]
self.process_sub_set("Sound20K", "models_per_material_sound20k",
sound20k_init_commands, get_sound20k_scenes())
from tdw.output_data import CompositeObjects, SegmentationColors
from sticky_mitten_avatar.util import get_data
"""
Get default audio data for each sub-object of a composite object.
"""
if __name__ == "__main__":
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument("--model", default="puzzle_box_composite")
args = parser.parse_args()
c = Controller()
c.start()
resp = c.communicate([
TDWUtils.create_empty_room(12, 12),
c.get_add_object(model_name=args.model, object_id=0), {
"$type": "send_composite_objects"
}, {
"$type": "send_segmentation_colors"
}
])
c.communicate({"$type": "terminate"})
# Get the name of each object.
colors = get_data(resp=resp, d_type=SegmentationColors)
names = dict()
for i in range(colors.get_num()):
names[colors.get_object_id(i)] = colors.get_object_name(i)
# Parse the composite object data.
def get_scene_initialization_commands(self) -> List[dict]:
return [{
"$type": "load_scene",
"scene_name": "ProcGenScene"
},
TDWUtils.create_empty_room(12, 12)]
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("parquet_wood_mahogany")
self.communicate({
"$type": "add_material",
"name": "parquet_wood_mahogany",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "parquet_wood_mahogany"
})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -1,
"y": 1.4,
"z": -1
},
look_at=TDWUtils.array_to_vector3(
[-0.1, 0.7, -0.1]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table_id = self.add_object(
model_name=
"b03_restoration_hardware_pedestal_salvaged_round_tables",
position={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="club_sandwich",
position={
"x": 0.45,
"y": top[1],
"z": 0.25
},
rotation={
"x": 0,
"y": 30,
"z": 0
},
library="models_full.json")
self.add_object(model_name="coffeemug",
position={
"x": 0.03,
"y": top[1],
"z": 0.03
},
library="models_full.json")
self.add_object(model_name="knife3",
position={
"x": -0.17,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 24,
"z": 90
},
library="models_full.json")
self.add_object(model_name="notes_02",
position={
"x": 0.17,
"y": top[1],
"z": -0.12
},
rotation={
"x": 0,
"y": 24,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([-0.1, 0.7, -0.1])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"sadnwich2",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("laser_cut_white_norway_spruce")
self.communicate({
"$type": "add_material",
"name": "laser_cut_white_norway_spruce",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "laser_cut_white_norway_spruce"
})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 1.5,
"y": 1.6,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[0, 0.5, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table = self.add_object(
model_name="boconcept_lugo_dining_table_vray1.5",
position={
"x": 0,
"y": 0,
"z": 0.0
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table)
top = table_bounds.get_top(0)
self.add_object(model_name='macbook_001',
position={
"x": 0,
"y": top[1],
"z": 0.1
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
self.add_object(model_name='spunlight_designermesh_lamp',
position={
"x": -0.19,
"y": top[1],
"z": -0.4
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
self.add_object(model_name="apple_magic_mouse_2_(2015)_vray",
position={
"x": 0,
"y": top[1],
"z": 0.32
},
rotation={
"x": 0,
"y": 86,
"z": 0
},
library="models_full.json")
self.add_object(model_name="f10_apple_iphone_4",
position={
"x": 0.14,
"y": top[1],
"z": -0.3
},
rotation={
"x": 0,
"y": 12,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.5, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"phone",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
self.start()
# init_setup_commands = [{"$type": "set_screen_size",
# "width": 600,
# "height": 480},
# {"$type": "set_render_quality",
# "render_quality": 5}]
# self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 2.5,
"y": 1.5,
"z": 0.3
},
look_at=TDWUtils.array_to_vector3(
[3, 0.5, 0.5]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("gravel_white")
self.communicate({
"$type": "add_material",
"name": "gravel_white",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "gravel_white"
})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
bench = self.add_object(model_name="b04_wood_metal_park_bench",
position={
"x": 3,
"y": 0,
"z": 0.5
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
bench_bounds = self.get_bounds_data(bench)
top = bench_bounds.get_top(0)
self.add_object(model_name="b05_baseball_bat_01",
position={
"x": 2.8,
"y": top[1] - 0.35,
"z": 0
},
rotation={
"x": 0,
"y": 90,
"z": 90
},
library="models_full.json")
self.add_object(model_name="b05_baseballnew_v03_12",
position={
"x": 2.9,
"y": top[1] - 0.35,
"z": 0.4
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([3, 0.5, 0.5])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"bench_ball",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def trial(self):
"""
Select random objects and collide them to produce impact sounds.
"""
p = PyImpact(initial_amp=0.25)
# Set the environment audio materials.
floor = AudioMaterial.ceramic
wall = AudioMaterial.wood
# Create an empty room.
# Set the screen size.
# Adjust framerate so that physics is closer to realtime.
commands = [
TDWUtils.create_empty_room(12, 12), {
"$type": "destroy_all_objects"
}, {
"$type": "set_screen_size",
"width": 1024,
"height": 1024
}, {
"$type": "set_target_framerate",
"framerate": 100
}
]
# Create the avatar.
commands.extend(
TDWUtils.create_avatar(avatar_type="A_Img_Caps_Kinematic",
position={
"x": 1,
"y": 1.2,
"z": 1.2
},
look_at=TDWUtils.VECTOR3_ZERO))
# Add the audio sensor.
commands.extend([{
"$type": "add_audio_sensor",
"avatar_id": "a"
}, {
"$type": "set_focus_distance",
"focus_distance": 2
}])
# Select a random pair of objects.
obj1_names = [
"trapezoidal_table", "glass_table_round", "yellow_side_chair",
"table_square", "marble_table"
]
obj2_names = ["vase_06", "spoon1", "glass3", "jug02"]
obj1_name = random.choice(obj1_names)
obj2_name = random.choice(obj2_names)
# Get initialization data from the default audio data (which includes mass, friction values, etc.).
obj1_init_data = AudioInitData(name=obj1_name)
obj2_init_data = AudioInitData(name=obj2_name,
position={
"x": 0,
"y": 2,
"z": 0
},
rotation={
"x": 135,
"y": 0,
"z": 30
})
# Convert the initialization data to commands.
obj1_id, obj1_commands = obj1_init_data.get_commands()
obj2_id, obj2_commands = obj2_init_data.get_commands()
# Cache the IDs and names of each object for PyImpact.
object_names = {obj1_id: obj1_name, obj2_id: obj2_name}
p.set_default_audio_info(object_names=object_names)
# Add the objects.
commands.extend(obj1_commands)
commands.extend(obj2_commands)
# Apply a small force to the dropped object.
# Request collision and rigidbody output data.
commands.extend([{
"$type": "apply_force_to_object",
"force": {
"x": 0,
"y": -0.01,
"z": 0
},
"id": obj2_id
}, {
"$type": "send_collisions",
"enter": True,
"stay": False,
"exit": False,
"collision_types": ["obj", "env"]
}, {
"$type": "send_rigidbodies",
"frequency": "always",
"ids": [obj2_id, obj1_id]
}])
# Send all of the commands.
resp = self.communicate(commands)
# Iterate through 200 frames.
# Every frame, listen for collisions, and parse the output data.
for i in range(200):
# Use PyImpact to generate audio from the output data and then convert the audio to TDW commands.
# If no audio is generated, then `commands` is an empty list.
commands = p.get_audio_commands(resp=resp, floor=floor, wall=wall)
# Send the commands to TDW in order to play the audio.
resp = self.communicate(commands)
def run(self):
rng = np.random.RandomState(0)
self.model_librarian = ModelLibrarian("models_special.json")
self.start()
commands = [TDWUtils.create_empty_room(100, 100)]
# The starting height of the objects.
y = 10
# The radius of the circle of objects.
r = 7.0
# The mass of each object.
mass = 5
# Get all points within the circle defined by the radius.
p0 = np.array((0, 0))
o_id = 0
for x in np.arange(-r, r, 1):
for z in np.arange(-r, r, 1):
p1 = np.array((x, z))
dist = np.linalg.norm(p0 - p1)
if dist < r:
# Add an object.
# Set its mass, physics properties, and color.
commands.extend([self.get_add_object("prim_cone",
object_id=o_id,
position={"x": x, "y": y, "z": z},
rotation={"x": 0, "y": 0, "z": 180}),
{"$type": "set_mass",
"id": o_id,
"mass": mass},
{"$type": "set_physic_material",
"dynamic_friction": 0.8,
"static_friction": 0.7,
"bounciness": 0.5,
"id": o_id},
{"$type": "set_color",
"color": {"r": rng.random_sample(),
"g": rng.random_sample(),
"b": rng.random_sample(),
"a": 1.0},
"id": o_id}])
o_id += 1
# Request transforms per frame.
commands.extend([{"$type": "send_transforms",
"frequency": "always"}])
# Create an avatar to observe the grisly spectacle.
avatar_position = {"x": -20, "y": 8, "z": 18}
commands.extend(TDWUtils.create_avatar(position=avatar_position, look_at=TDWUtils.VECTOR3_ZERO))
commands.append({"$type": "set_focus_distance",
"focus_distance": TDWUtils.get_distance(avatar_position, TDWUtils.VECTOR3_ZERO)})
resp = self.communicate(commands)
# If an objects are this far away from (0, 0, 0) the forcefield "activates".
forcefield_radius = 5
# The forcefield will bounce objects away at this force.
forcefield_force = -10
zeros = np.array((0, 0, 0))
for i in range(1000):
transforms = Transforms(resp[0])
commands = []
for j in range(transforms.get_num()):
pos = transforms.get_position(j)
pos = np.array(pos)
# If the object is in the forcefield, apply a force.
if TDWUtils.get_distance(TDWUtils.array_to_vector3(pos), TDWUtils.VECTOR3_ZERO) <= forcefield_radius:
# Get the normalized directional vector and multiply it by the force magnitude.
d = zeros - pos
d = d / np.linalg.norm(d)
d = d * forcefield_force
commands.append({"$type": "apply_force_to_object",
"id": transforms.get_id(j),
"force": TDWUtils.array_to_vector3(d)})
resp = self.communicate(commands)