def run(self):
self.start()
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(12, 12))
# Create the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 3,
"z": -4
},
look_at=TDWUtils.VECTOR3_ZERO))
# Set the pass mask to _depth only.
# Get an image.
resp = self.communicate([
self.get_add_object("rh10", object_id=0), {
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_depth"]
}, {
"$type": "send_images",
"frequency": "once"
}
])
images = Images(resp[0])
# Get the depth values of each pixel.
depth = TDWUtils.get_depth_values(images.get_image(0))
print(depth)
def save_images(images: Images,
filename: str,
output_directory="dist",
resize_to=None,
append_pass: bool = True) -> None:
"""
Save each image in the Images object.
The name of the image will be: pass_filename.extension, e.g.: `"0000"` -> `depth_0000.png`
The images object includes the pass and extension information.
:param images: The Images object. Contains each capture pass plus metadata.
:param output_directory: The directory to write images to.
:param filename: The filename of each image, minus the extension. The image pass will be appended as a prefix.
:param resize_to: Specify a (width, height) tuple to resize the images to. This is slower than saving as-is.
:param append_pass: If false, the image pass will _not_ be appended to the filename as a prefix, e.g.: `"0000"`: -> "`0000.jpg"`
"""
if not os.path.isdir(output_directory):
os.makedirs(output_directory)
for i in range(images.get_num_passes()):
if append_pass:
fi = images.get_pass_mask(
i)[1:] + "_" + filename + "." + images.get_extension(i)
else:
fi = filename + "." + images.get_extension(i)
if resize_to:
TDWUtils.get_pil_image(images, i).resize((resize_to[0], resize_to[1]), Image.LANCZOS)\
.save(os.path.join(output_directory, fi))
else:
with open(os.path.join(output_directory, fi), "wb") as f:
f.write(images.get_image(i))
def get_shaped_depth_pass(images: Images, index: int) -> np.array:
"""
The `_depth` and `_depth_simple` passes are a 1D array of RGB values, as oppposed to a png or jpg like every other pass.
This function reshapes the array into a 2D array of RGB values.
:param images: The `Images` output data.
:param index: The index in `Images` of the depth pass. See: `Images.get_pass_mask()`.
:return: A reshaped depth pass. Shape is: `(height, width, 3)`.
"""
return np.flip(
np.reshape(images.get_image(index),
(images.get_height(), images.get_width(), 3)), 0)
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(10, 10))
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
self.communicate(self.get_add_hdri_skybox("autumn_hockey_4k"))
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": 3, "y": 1.3, "z": 0},
look_at=TDWUtils.array_to_vector3([0, 0.5, 0]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("grass_countryside")
print(record)
self.communicate({"$type": "add_material", "name": "grass_countryside", "url": record.get_url()})
self.communicate({"$type": "set_proc_gen_floor_material", "name": "grass_countryside"})
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
self.add_object(model_name="b05_clochild4",
position={"x": 0, "y": 0, "z": 0},
rotation={"x": 0, "y": 30, "z": 0},
library="models_full.json")
self.add_object(model_name="baseballbat",
position={"x": 0.5, "y": 0, "z": 0.3},
rotation={"x": 0, "y": 30, "z": 0},
library="models_full.json")
self.add_object(model_name="base-ball",
position={"x": 0.4, "y": 0, "z": 0.22},
rotation={"x": 0, "y": 30, "z": 10},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([0, 0.5, 0])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "kid_baseball", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def get_image(self, record: ModelRecord):
o_id = Controller.get_unique_id()
self.communicate({"$type": "add_object",
"name": record.name,
"url": record.get_url(),
"scale_factor": record.scale_factor,
"rotation": record.canonical_rotation,
"id": o_id})
s = TDWUtils.get_unit_scale(record) * 2
# Scale the model and get an image.
# Look at the model's centroid.
resp = self.communicate([{"$type": "scale_object",
"id": o_id,
"scale_factor": {"x": s, "y": s, "z": s}},
{"$type": "look_at",
"avatar_id": "a",
"object_id": o_id,
"use_centroid": True}])
# Destroy the model and unload the asset bundle.
self.communicate([{"$type": "destroy_object",
"id": o_id},
{"$type": "unload_asset_bundles"}])
return Images(resp[0]), resp[-1]
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(32, 32))
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": -1.5, "y": 1.5, "z": -15},
look_at=TDWUtils.array_to_vector3([-1.7, 0.7, -2.3]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("concrete_chipped_cracked")
self.communicate({"$type": "add_material", "name": "concrete_chipped_cracked", "url": record.get_url()})
self.communicate({"$type": "set_proc_gen_floor_material", "name": "concrete_chipped_cracked"})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
self.communicate({"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"})
bus = self.add_object(model_name="b06_bus_new",
position={"x": -2, "y": 0, "z": 0},
rotation={"x": 0, "y": -70, "z": 0},
library="models_full.json")
bus_bounds = self.get_bounds_data(bus)
bus_front = bus_bounds.get_front(0)
print(bus_front)
self.add_object(model_name="3dscan_man_004",
position={"x": -3.7, "y": 0, "z": -7.3},
rotation={"x": 0, "y": 0, "z": 0},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([-1.7, 0.7, -2.3])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "bus",
output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def run(self):
# Create the output directory.
output_directory = "example_output"
if os.path.exists(output_directory):
shutil.rmtree(output_directory)
sleep(0.5)
os.mkdir(output_directory)
print(f"Images will be saved to: {output_directory}")
self.start()
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(12, 12))
# Add the objects.
lamp_id = self.add_object("alma_floor_lamp",
position={"x": 1, "y": 0, "z": 0},
rotation={"x": 0, "y": 90, "z": 0})
self.add_object("live_edge_coffee_table",
position={"x": 1.5, "y": 0, "z": 1.5},
rotation={"x": 0, "y": 30, "z": 0})
self.add_object("small_table_green_marble",
position={"x": -0.9, "y": 0, "z": -1.35})
# Create the avatar.
self.communicate({"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"})
# Try to find a valid position on the NavMesh.
x, y, z = TDWUtils.get_random_position_on_nav_mesh(self, 12, 12)
# Teleport the avatar to the valid position.
# Apply a force to the lamp.
# Set the pass masks to _img.
# Enable image capture.
self.communicate([{"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {"x": x, "y": 1.5, "z": z}},
{"$type": "apply_force_to_object",
"force": {"x": 2, "y": 1, "z": 0},
"id": lamp_id},
{"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img", "_id"]},
{"$type": "send_images",
"frequency": "always"},
])
# Capture 100 images.
for i in range(100):
# Look at the lamp.
resp = self.communicate({"$type": "look_at",
"avatar_id": "a",
"object_id": lamp_id,
"use_centroid": True})
images = Images(resp[0])
# Save the image.
TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=output_directory)
def run(self):
self.start()
commands = [TDWUtils.create_empty_room(self.size, self.size)]
commands.extend(
TDWUtils.create_avatar(position={
"x": 0,
"y": 1.5,
"z": 0
},
avatar_id="a"))
commands.extend([{
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img"]
}, {
"$type": "send_images",
"frequency": "always"
}])
self.communicate(commands)
# Listen for keys.
# Turn left.
self.listen(key="left",
commands={
"$type": "rotate_sensor_container_by",
"axis": "yaw",
"angle": self.angle * -1,
"sensor_name": "SensorContainer",
"avatar_id": "a"
})
# Turn right.
self.listen(key="right",
commands={
"$type": "rotate_sensor_container_by",
"axis": "yaw",
"angle": self.angle,
"sensor_name": "SensorContainer",
"avatar_id": "a"
})
# Quit the application.
self.listen(key="esc", function=self.stop)
# Continue until the quit key is pressed.
i = 0
while not self.done:
# Listen for keyboard input. Receive output data.
resp = self.communicate([])
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
# Save images.
if r_id == "imag":
TDWUtils.save_images(
images=Images(r),
output_directory=self.images_directory,
filename=TDWUtils.zero_padding(i, width=4))
# Increment the image number.
i += 1
self.communicate({"$type": "terminate"})
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 1280,
"height": 962},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 1, "z": 0},
look_at=TDWUtils.array_to_vector3([2, 0.8, -1.5]),
avatar_id="avatar"))
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
table_id = self.add_object(model_name="glass_table",
position={"x": 2, "y": 0, "z": -1.5},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="flower_vase_set_2",
position={"x": 1.6, "y": top[1], "z": -1.5},
library="models_full.json")
self.add_object(model_name="flower_vase_set_2",
position={"x": 2, "y": top[1], "z": -1.5},
library="models_full.json")
self.add_object(model_name="flower_vase_set_2",
position={"x": 2.4, "y": top[1], "z": -1.5},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([2, 0.8, -1.5])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "flowers_row", output_directory="replicated_images")
def save_image(self, resp, record: ModelRecord, image_count: int,
root_dir: str, wnid: str, train: int,
train_count: int) -> None:
"""
Save an image.
:param resp: The raw response data.
:param record: The model record.
:param image_count: The image count.
:param root_dir: The root directory.
:param wnid: The wnid.
:param train: Number of train images so far.
:param train_count: Total number of train images to generate.
"""
# Get the directory.
directory = Path(root_dir).joinpath("train" if train < train_count else
"val").joinpath(wnid).resolve()
if not os.path.exists(directory):
# Try to make the directories. Due to threading, they might already be made.
try:
os.makedirs(directory)
except:
pass
# Save the image.
filename = f"{record.name}_{image_count:04d}"
# Save the image without resizing.
if self.screen_size == self.output_size:
TDWUtils.save_images(Images(resp[0]),
filename,
output_directory=directory)
# Resize the image and save it.
else:
TDWUtils.save_images(Images(resp[0]),
filename,
output_directory=directory,
resize_to=(self.output_size,
self.output_size))
def get_image(self, record: MaterialRecord):
self.communicate([self.get_add_material(record.name),
{"$type": "set_primitive_visual_material",
"id": self.sphere_id,
"name": record.name,
"quality": "high"},
{"$type": "set_primitive_visual_material",
"id": self.cube_id,
"name": record.name,
"quality": "high"}])
# Capture the image the following frame to allow it to initialize correctly.
resp = self.communicate({"$type": "do_nothing"})
return Images(resp[0]), resp[-1]
def get_pil_image(images: Images, index: int) -> Image:
"""
Converts Images output data to a PIL Image object.
Use this function to read and analyze an image in memory.
Do NOT use this function to save image data to disk; `save_image` is much faster.
:param images: Images data from the build.
:param index: The index of the image in Images.get_image
:return A PIL image.
"""
return Image.open(io.BytesIO(images.get_image(index)))
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
width = 8
length = 8
# Create an empty room.
self.load_streamed_scene("box_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 1.7, "z": 0},
look_at={"x": 0.3, "y": 0.8, "z": 2.2},
avatar_id="avatar"))
self.communicate({"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"})
self.add_object(model_name="b06_bikenew",
position={"x": 0.3, "y": 0, "z": 2.2},
rotation={"x": 0, "y": 180, "z": 0},
library="models_full.json")
self.add_object(model_name="animal_dog_rtsit_1280",
position={"x": 0.35, "y": 0, "z": 1.8},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": {"x": 0.3, "y": 0.8, "z": 2.2}})
images = Images(scene_data[0])
TDWUtils.save_images(images, "dog_bike", output_directory="replicated_images/interior")
def _orbit_and_capture_image(c: Controller, x: float, y: float,
z: float) -> bool:
"""
Orbit around an object and check for missing materials.
:param c: The controller.
:param x: The next x positional coordinate.
:param y: The next y positional coordinate.
:param z: The next z positional coordinate.
:return: True if there is a missing material in the image, False otherwise.
"""
# Teleport the avatar.
# Look at the model.
# Receive an image.
resp = c.communicate([{
"$type": "teleport_avatar_to",
"position": {
"x": x,
"y": y,
"z": z
}
}, {
"$type": "look_at",
"object_id": MissingMaterials.OBJECT_ID,
"use_centroid": True
}])
# Check if there are any pink pixels.
images = Images(resp[0])
img = Image.open(BytesIO(images.get_image(0)))
for c in img.getcolors(maxcolors=100000):
if c[1] == MissingMaterials.HELL_PINK:
return True
return False
def run(self):
# Create the scene.
# Set image encoding globals.
self.start()
commands = [TDWUtils.create_empty_room(12, 12),
{"$type": "set_screen_size",
"width": 128,
"height": 128},
{"$type": "set_img_pass_encoding",
"value": False}]
# Create the avatar.
commands.extend(TDWUtils.create_avatar(position={"x": 2.478, "y": 1.602, "z": 1.412},
look_at=TDWUtils.VECTOR3_ZERO,
avatar_id="a"))
# Enable all pass masks. Request an image for this frame only.
commands.extend([{"$type": "set_pass_masks",
"pass_masks": ["_img", "_id", "_category", "_mask", "_depth", "_normals", "_flow",
"_depth_simple", "_albedo"],
"avatar_id": "a"},
{"$type": "send_images",
"ids": ["a"],
"frequency": "once"}])
# Add objects.
commands.append(self.get_add_object("small_table_green_marble",
position=TDWUtils.VECTOR3_ZERO,
rotation=TDWUtils.VECTOR3_ZERO,
object_id=0))
commands.append(self.get_add_object("rh10",
position={"x": 0.7, "y": 0, "z": 0.4},
rotation={"x": 0, "y": 30, "z": 0},
object_id=1))
commands.append(self.get_add_object("jug01",
position={"x": -0.3, "y": 0.9, "z": 0.2},
rotation=TDWUtils.VECTOR3_ZERO,
object_id=3))
commands.append(self.get_add_object("jug05",
position={"x": 0.3, "y": 0.9, "z": -0.2},
rotation=TDWUtils.VECTOR3_ZERO,
object_id=4))
# Send the commands.
resp = self.communicate(commands)
# Save the images.
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
if r_id == "imag":
# Save the images.
TDWUtils.save_images(output_directory="dist", images=Images(r), filename="0")
print(f"Images saved to: {Path('dist').resolve()}")
# Stop the build.
self.communicate({"$type": "terminate"})
def communicate(self, commands: Union[dict, List[dict]]) -> List[bytes]:
"""
See `Magnebot.communicate()`.
Images are saved per-frame.
"""
resp = super().communicate(commands=commands)
if not self._debug:
# Save all images.
got_images = False
for i in range(len(resp) - 1):
r_id = OutputData.get_data_type_id(resp[i])
if r_id == "imag":
got_images = True
images = Images(resp[i])
avatar_id = images.get_avatar_id()
if avatar_id in self.image_directories:
TDWUtils.save_images(filename=TDWUtils.zero_padding(self._image_count, 8),
output_directory=self.image_directories[avatar_id],
images=images)
if got_images:
self._image_count += 1
return resp
def run(self):
self.start()
depth_pass = "******"
# Create an empty room.
# Set the screen size.
commands = [TDWUtils.create_empty_room(12, 12),
{"$type": "set_screen_size",
"width": 512,
"height": 512}]
# Add the avatar.
commands.extend(TDWUtils.create_avatar(position={"x": 1.57, "y": 3, "z": 3.56}, look_at=TDWUtils.VECTOR3_ZERO))
# Add an object.
# Request images and camera matrices.
commands.extend([self.get_add_object("trunck", object_id=0),
{"$type": "set_pass_masks",
"pass_masks": [depth_pass]},
{"$type": "send_images"},
{"$type": "send_camera_matrices"}])
resp = self.communicate(commands)
depth_image = None
camera_matrix = None
images = None
for i in range(len(resp) - 1):
r_id = OutputData.get_data_type_id(resp[i])
# Get the image.
if r_id == "imag":
images = Images(resp[i])
for j in range(images.get_num_passes()):
if images.get_pass_mask(j) == depth_pass:
depth_image = images.get_image(j)
# Get the camera matrix.
elif r_id == "cama":
camera_matrix = CameraMatrices(resp[i]).get_camera_matrix()
# Save the image.
TDWUtils.save_images(images=images, output_directory="D:/depth_shader", filename="0", append_pass=True)
# Get the depth values of each pixel.
depth = TDWUtils.get_depth_values(image=depth_image, width=images.get_width(), height=images.get_height())
print(np.min(depth), np.max(depth))
print(depth)
np.save("depth", depth)
np.save("camera_matrix", camera_matrix)
# Get a point cloud and write it to disk.
point_cloud_filename = "point_cloud.txt"
print(f"Point cloud saved to: {Path(point_cloud_filename)}")
TDWUtils.get_point_cloud(depth=depth, filename=point_cloud_filename, camera_matrix=camera_matrix)
# Show the depth values.
plt.imshow(depth)
plt.show()
self.communicate({"$type": "terminate"})
def _do_frame(self, commands: Union[List[dict],
dict]) -> AvatarStickyMitten:
"""
Send commands to the build. Receive images and sticky mitten avatar data.
Save the images.
:param commands: The commands for this frame.
:return: The avatar data.
"""
# Add to the list of commands: Look at the avatar.
frame_commands = [{
"$type": "look_at_position",
"position": {
"x": self.avatar_position[0],
"y": self.avatar_position[1] + 0.5,
"z": self.avatar_position[2]
},
"avatar_id": self.cam_id
}]
if isinstance(commands, dict):
frame_commands.append(commands)
else:
frame_commands.extend(commands)
resp = self.communicate(frame_commands)
avsm: Optional[AvatarStickyMitten] = None
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
# Save images.
if r_id == "imag":
TDWUtils.save_images(
images=Images(r),
filename=TDWUtils.zero_padding(self.frame_number, 4),
output_directory=PutObjectOnTable.OUTPUT_DIR)
elif r_id == "avsm":
avsm = AvatarStickyMitten(r)
# Update the position of the avatar and the frame number.
self.avatar_position = avsm.get_position()
self.frame_number += 1
return avsm
def run(self):
""" Generate room using COCO_TDW dataset """
objects_in_scene = 15
object_ids = []
# Get Category-Object mapping
TDW_COCO_models = TDW_relationships.get_COCO_TDW_mapping()
# print("TDWCOCO:", TDW_COCO_models)
# print("KEYS:", TDW_COCO_models.keys())
# Gets COCO categories co-occurring in a scene
# +5 is for dealing with failed object insertion attempts
COCO_configurations = msCOCO_matrix.get_max_co_occurrence(5, int(objects_in_scene + 5))
configuration_1 = COCO_configurations[0]
print("Config 1:", configuration_1)
# TDW models/objects
objects = []
for COCO_object in configuration_1:
print(COCO_object)
print(COCO_object.split())
if len(COCO_object.split()) > 1:
COCO_object = COCO_object.split()[-1]
print(COCO_object)
# Check if COCO category is a key in the COCO-TDW mapping
if COCO_object in TDW_COCO_models.keys():
# Gets random TDW model (from COCO-to-TDW map) based on COCO category key
print(TDW_COCO_models[COCO_object])
model = TDW_COCO_models[COCO_object][random.randint(0, len(TDW_COCO_models[COCO_object]) - 1)]
objects.append(model)
print("COCO to TDW objects:", objects)
# print(len(objects))
# Stores object categories that other objects can be placed upon (e.g. table, chair, couch, bed)
surface_properties_list = TDW_COCO_models['table'] + TDW_COCO_models['chair'] + \
TDW_COCO_models['bed'] + TDW_COCO_models['couch'] + \
TDW_COCO_models['bench'] + TDW_COCO_models['refrigerator']
surface_categories = []
for surface_properties in surface_properties_list:
surface_categories.append(surface_properties[0])
print("Surface Categories:", surface_categories)
# Stores the actual surface object instances/ids alongside number of objects on the surface
surface_object_ids = {}
self.start()
positions_list = [] # Stores current model locations and radii
scene_dimensions = [] # Store scene/environment dimensions
init_setup_commands = [{"$type": "set_screen_size",
"width": 640,
"height": 481},
{"$type": "set_render_quality",
"render_quality": 1}]
self.communicate(init_setup_commands)
scene_lib = SceneLibrarian()
# Disable physics when adding in new objects (objects float)
self.communicate({"$type": "simulate_physics",
"value": False})
for scene in scenes[1:]:
# Load in scene
# print("Scene", scene[0])
if scene[3] == "interior" and scene[0] == "box_room_2018":
self.start()
scene_record = scene_lib.get_record(scene[0])
self.communicate({"$type": "add_scene",
"name": scene_record.name,
"url": scene_record.get_url()})
# Gets dimensions of environments (e.g. inside, outside) in the scene
# This command returns environment data in the form of a list of serialized byte arrays
scene_bytes = self.communicate({"$type": "send_environments",
"frequency": "once"})
# Iterating through data and parsing byte array
# Ignoring the last element (the frame count)
for b in scene_bytes[:-1]:
e = Environments(b)
for i in range(e.get_num()):
center = e.get_center(i)
bounds = e.get_bounds(i)
env_id = e.get_id(i)
scene_dimensions = [center, bounds, env_id] # Center, bounds are tuples
# Must come before set_pass_masks
avatar_position = TDWUtils.array_to_vector3([0.9 * scene_dimensions[1][0] / 2,
scene_dimensions[1][1] / 2,
0])
# print("Avatar Position:", avatar_position)
self.communicate(TDWUtils.create_avatar(avatar_id="avatar",
position=avatar_position,
look_at={"x": 0,
"y": scene_dimensions[0][1] / 2,
"z": 0}))
# Set collision mode
self.communicate({"$type": "set_avatar_collision_detection_mode",
"mode": "continuous_speculative",
"avatar_id": "avatar"})
# Alter FOV
self.communicate({"$type": "set_field_of_view",
"field_of_view": 80,
"avatar_id": "avatar"})
# # Gets rid of header (Model: Category)
# objects = TDW_COCO_models[1:]
# random.shuffle(objects)
obj_count = 0
obj_overlaps = 0 # Number of failed attempts to place object due to over-dense objects area
while obj_count < objects_in_scene and obj_overlaps < 5:
# Handles if object has been added to a flat surface
added_to_surface = False
print("Object COUNT:", obj_count)
# Need to have random position for Bounds Data to return meaningful info
valid_obj_pos = {"x": random.uniform(-1 * scene_dimensions[1][0] / 2,
0.5 * scene_dimensions[1][0] / 2),
"y": scene_dimensions[1][1] / 4,
"z": random.uniform(-0.9 * scene_dimensions[1][2] / 2,
0.9 * scene_dimensions[1][2] / 2)}
print("First random position")
# Add in the object at random position
# Object will later be removed or updated accordingly after performing collision calculations
record = ModelLibrarian(library="models_full.json").get_record(objects[obj_count][0])
print("Record gotten")
print(objects[obj_count][0])
o_id = self.communicate({"$type": "add_object",
"name": objects[obj_count][0],
"url": record.get_url(),
"scale_factor": record.scale_factor,
"position": valid_obj_pos,
"rotation": TDWUtils.VECTOR3_ZERO,
"category": record.wcategory,
"id": obj_count})
print("Random first add")
# Returns bound data for added object
bounds_data = self.communicate({"$type": "send_bounds",
"frequency": "once"})
print("Bounds returned")
# Appends object, with information on position and obj_radius, to positions_list
# Length of buffer array should be 1
# print("Bounds Data:", bounds_data)
for b in bounds_data[:-1]:
# print("Buffer Loop:", b)
b_id = OutputData.get_data_type_id(b)
if b_id == "boun":
# print("BOUNDS")
o = Bounds(b)
# print("# of Objects:", o.get_num())
# print("# of Failed Attempts:", obj_overlaps)
# print("Buffer Array:", b)
# print("Bounds Object:", o)
for i in range(o.get_num()):
print("Object ID:", o.get_id(i))
print("obj_count:", obj_count)
print("Object:", objects[obj_count][0], "Category:", objects[obj_count][1])
# print("Object Center:", o.get_center(i))
# Only want to compute valid_position for object we are about to add
# Skip any computation if this is not the case
if o.get_id(i) != obj_count:
continue
# Useful for detecting if object fits in environment
# print("Calculating if object fits in environment")
width = distance.euclidean(o.get_left(i), o.get_right(i))
depth = distance.euclidean(o.get_front(i), o.get_back(i))
height = distance.euclidean(o.get_top(i), o.get_bottom(i))
# print("Width:", width)
# print("Depth:", depth)
# ("Height:", height)
# Useful for avoiding object overlap
# print("Calculating Object Bounds")
center_to_top = distance.euclidean(o.get_center(i), o.get_top(i))
center_to_bottom = distance.euclidean(o.get_center(i), o.get_bottom(i))
center_to_left = distance.euclidean(o.get_center(i), o.get_left(i))
center_to_right = distance.euclidean(o.get_center(i), o.get_right(i))
center_to_front = distance.euclidean(o.get_center(i), o.get_front(i))
center_to_back = distance.euclidean(o.get_center(i), o.get_back(i))
# Max object radius (center to diagonal of bounding box)
obj_radius = \
max(math.sqrt(center_to_top ** 2 + center_to_left ** 2 + center_to_front ** 2),
math.sqrt(center_to_top ** 2 + center_to_right ** 2 + center_to_front ** 2),
math.sqrt(center_to_top ** 2 + center_to_left ** 2 + center_to_back ** 2),
math.sqrt(center_to_top ** 2 + center_to_right ** 2 + center_to_back ** 2),
math.sqrt(center_to_bottom ** 2 + center_to_left ** 2 + center_to_front ** 2),
math.sqrt(center_to_bottom ** 2 + center_to_right ** 2 + center_to_front ** 2),
math.sqrt(center_to_bottom ** 2 + center_to_left ** 2 + center_to_back ** 2),
math.sqrt(center_to_bottom ** 2 + center_to_right ** 2 + center_to_back ** 2))
# print("Obj_Radius:", obj_radius)
# Set sweeping radius, based on scene plane dimensions
l_radius = random.uniform(0, min(0.5 * scene_dimensions[1][0] / 2,
0.5 * scene_dimensions[1][2] / 2))
# Checking that object fits in scene viewing
if (width > min(0.7 * scene_dimensions[1][0], 0.7 * scene_dimensions[1][2]) or
depth > min(0.7 * scene_dimensions[1][0], 0.7 * scene_dimensions[1][2]) or
height > 0.7 * scene_dimensions[1][1]):
print("Object does not fit in scene")
self.communicate([{"$type": "send_images",
"frequency": "never"},
{"$type": "destroy_object",
"id": obj_count}])
# Ensures next attempt to load in item is not the same item as before
random.shuffle(objects)
break
# Not possible to find valid object position -- too many overlapping objects
elif (not self._get_object_position(scene_dimensions=scene_dimensions,
object_positions=positions_list,
object_to_add_radius=obj_radius,
max_tries=20,
location_radius=l_radius)[0]):
print("Could not calculate valid object location")
self.communicate([{"$type": "send_images",
"frequency": "never"},
{"$type": "destroy_object",
"id": obj_count}])
obj_overlaps += 1
# Ensures next attempt to load in item is not the same item as before
random.shuffle(objects)
break
# Find appropriate, non-overlapping object position
# Reset object position to the valid position
else:
print("Object fits in scene")
# Check if object fits on table, chair, couch, etc.
# Add object if it fits, place it somewhere on top of the surface
for surface_id in surface_object_ids.keys():
print("Surface ID:", surface_id)
# Skip placement feasibility if the object is already a surface-type object
# Ex. no chair on top of a table
if objects[obj_count][0] in surface_categories:
print("Object: %s is already a surface object" % objects[obj_count][0])
break
# Check how many objects are on surface
if surface_object_ids[surface_id] >= 3:
print("Too many objects on surface")
print("From surface objects dict:", surface_object_ids[surface_id])
continue
surface_bounds = self.get_bounds_data(surface_id)
surface_area = distance.euclidean(surface_bounds.get_left(0),
surface_bounds.get_right(0)) * \
distance.euclidean(surface_bounds.get_front(0),
surface_bounds.get_back(0))
obj_area = width * height
if obj_area < surface_area:
s_center_to_top = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_top(0))
s_center_to_bottom = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_bottom(0))
s_center_to_left = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_left(0))
s_center_to_right = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_right(0))
s_center_to_front = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_front(0))
s_center_to_back = distance.euclidean(surface_bounds.get_center(0),
surface_bounds.get_back(0))
surface_radius = \
max(math.sqrt(
s_center_to_top ** 2 + s_center_to_left ** 2 + s_center_to_front ** 2),
math.sqrt(
s_center_to_top ** 2 + s_center_to_right ** 2 + s_center_to_front ** 2),
math.sqrt(
s_center_to_top ** 2 + s_center_to_left ** 2 + s_center_to_back ** 2),
math.sqrt(
s_center_to_top ** 2 + s_center_to_right ** 2 + s_center_to_back ** 2),
math.sqrt(
s_center_to_bottom ** 2 + s_center_to_left ** 2 + s_center_to_front ** 2),
math.sqrt(
s_center_to_bottom ** 2 + s_center_to_right ** 2 + s_center_to_front ** 2),
math.sqrt(
s_center_to_bottom ** 2 + s_center_to_left ** 2 + s_center_to_back ** 2),
math.sqrt(
s_center_to_bottom ** 2 + s_center_to_right ** 2 + s_center_to_back ** 2))
print("Surface-type object")
self.communicate({"$type": "destroy_object",
"id": obj_count})
# Adding the object to the top of the surface
on_pos = surface_bounds.get_top(0)
on_y = on_pos[1]
on_pos = TDWUtils.get_random_point_in_circle(np.array(on_pos),
0.7 * surface_radius)
on_pos[1] = on_y
on_pos = TDWUtils.array_to_vector3(on_pos)
on_rot = {"x": 0, "y": random.uniform(-45, 45), "z": 0}
# Add the object.
print("Model Name on Surface:", objects[obj_count][0])
record = ModelLibrarian(library="models_full.json").get_record(
objects[obj_count][0])
on_id = self.communicate({"$type": "add_object",
"name": objects[obj_count][0],
"url": record.get_url(),
"scale_factor": record.scale_factor,
"position": on_pos,
"rotation": on_rot,
"category": record.wcategory,
"id": obj_count})
obj_count += 1
surface_object_ids[surface_id] += 1
object_ids.append(obj_count)
print("Object added on top of surface")
added_to_surface = True
# Breaking out of surface objects loop
break
if added_to_surface:
print("Breaking out of object loop")
# Breaking out of object loop
break
print("Post-surface")
valid_obj_pos = self._get_object_position(scene_dimensions=scene_dimensions,
object_positions=positions_list,
object_to_add_radius=obj_radius,
max_tries=20,
location_radius=l_radius)[1]
print("Position calculated")
positions_list.append(ObjectPosition(valid_obj_pos, obj_radius))
self.communicate([{"$type": "send_images",
"frequency": "never"},
{"$type": "destroy_object",
"id": obj_count}])
added_object_id = self.communicate({"$type": "add_object",
"name": objects[obj_count][0],
"url": record.get_url(),
"scale_factor": record.scale_factor,
"position": valid_obj_pos,
"rotation": {"x": 0, "y": 0, "z": 0},
"category": record.wcategory,
"id": obj_count})
# print("Object ID:", added_object_id)
print("Regular object add")
object_ids.append(added_object_id)
# If TDW model belongs to surface categories, store id_information
if objects[obj_count][0] in surface_categories:
surface_object_ids[obj_count] = 0
# Rotate the object randomly
print("Rotating object")
self.communicate({"$type": "rotate_object_by",
"angle": random.uniform(-45, 45),
"axis": "yaw",
"id": obj_count,
"is_world": True})
# Minimal rotating for position differences
# Don't rotate the object if doing so will result in overlap into scene
if not (o.get_bottom(i)[1] < 0 or o.get_top(i)[1] > 0.9 * scene_dimensions[1][1]):
pitch_angle = random.uniform(-45, 45)
self.communicate({"$type": "rotate_object_by",
"angle": pitch_angle,
"axis": "pitch",
"id": obj_count,
"is_world": True})
roll_angle = random.uniform(-45, 45)
self.communicate({"$type": "rotate_object_by",
"angle": roll_angle,
"axis": "roll",
"id": obj_count,
"is_world": True})
# Don't need this for just changing positions
# Setting random materials/textures
# Looping through sub-objects and materials
sub_count = 0
for sub_object in record.substructure:
# Loop through materials in sub-objects
for j in range(len(sub_object)):
# Get random material and load in
material = random.choice(materials[1:])
self.load_material(material)
print("Material loaded")
# Set random material on material of sub-object
self.communicate({"$type": "set_visual_material",
"material_index": j,
"material_name": material[0],
"object_name": sub_object['name'],
"id": obj_count})
print("Material set")
sub_count += 1
if sub_count > 10:
break
break
print("Updating count")
obj_count += 1
print("Breaking out of object_id loop")
break
# Break out of buffer loop
print("Breaking out of buffer loop")
break
# Move onto next iteration of while loop (next object to load in)
print("Object added - next while loop iteration")
continue
# for i in range(200):
# self.communicate({"$type": "simulate_physics",
# "value": False})
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
# Capture scene
# NOTE: THESE SCENES GET REPLACED IN THE TARGET DIRECTORY
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": {"x": 0,
"y": scene_dimensions[0][1] / 2,
"z": 0}})
images = Images(scene_data[0])
TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=path)
print("Object ids:", object_ids)
def _write_frame(self, frames_grp: h5py.Group, resp: List[bytes], frame_num: int) -> \
Tuple[h5py.Group, h5py.Group, dict, bool]:
num_objects = len(self.object_ids)
# Create a group for this frame.
frame = frames_grp.create_group(TDWUtils.zero_padding(frame_num, 4))
# Create a group for images.
images = frame.create_group("images")
# Transforms data.
positions = np.empty(dtype=np.float32, shape=(num_objects, 3))
forwards = np.empty(dtype=np.float32, shape=(num_objects, 3))
rotations = np.empty(dtype=np.float32, shape=(num_objects, 4))
camera_matrices = frame.create_group("camera_matrices")
# Parse the data in an ordered manner so that it can be mapped back to the object IDs.
tr_dict = dict()
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
if r_id == "tran":
tr = Transforms(r)
for i in range(tr.get_num()):
pos = tr.get_position(i)
tr_dict.update({
tr.get_id(i): {
"pos": pos,
"for": tr.get_forward(i),
"rot": tr.get_rotation(i)
}
})
# Add the Transforms data.
for o_id, i in zip(self.object_ids, range(num_objects)):
if o_id not in tr_dict:
continue
positions[i] = tr_dict[o_id]["pos"]
forwards[i] = tr_dict[o_id]["for"]
rotations[i] = tr_dict[o_id]["rot"]
elif r_id == "imag":
im = Images(r)
# Add each image.
for i in range(im.get_num_passes()):
pass_mask = im.get_pass_mask(i)
# Reshape the depth pass array.
if pass_mask == "_depth":
image_data = TDWUtils.get_shaped_depth_pass(images=im,
index=i)
else:
image_data = im.get_image(i)
images.create_dataset(pass_mask,
data=image_data,
compression="gzip")
# Save PNGs
if pass_mask in self.save_passes:
filename = pass_mask[1:] + "_" + TDWUtils.zero_padding(
frame_num, 4) + "." + im.get_extension(i)
path = self.png_dir.joinpath(filename)
if pass_mask in ["_depth", "_depth_simple"]:
Image.fromarray(
TDWUtils.get_shaped_depth_pass(
images=im, index=i)).save(path)
else:
with open(path, "wb") as f:
f.write(im.get_image(i))
# Add the camera matrices.
elif OutputData.get_data_type_id(r) == "cama":
matrices = CameraMatrices(r)
camera_matrices.create_dataset(
"projection_matrix", data=matrices.get_projection_matrix())
camera_matrices.create_dataset(
"camera_matrix", data=matrices.get_camera_matrix())
objs = frame.create_group("objects")
objs.create_dataset("positions",
data=positions.reshape(num_objects, 3),
compression="gzip")
objs.create_dataset("forwards",
data=forwards.reshape(num_objects, 3),
compression="gzip")
objs.create_dataset("rotations",
data=rotations.reshape(num_objects, 4),
compression="gzip")
return frame, objs, tr_dict, False
def trial(self, m_o, s_o, b_o, output_directory, nimages):
"""
Collide a chair with a fridge.
:param m_c: mass of the chair.
:param s_c: The static friction of the chair.
:param b_c: The bounciness of the chair.
"""
# Destroy all objects currently in the scene.
self.communicate({"$type": "destroy_all_objects"})
# Set screen size
self.communicate({
"$type": "set_screen_size",
"height": 480,
"width": 640
})
# Create the avatar.
self.communicate({
"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"
})
# Teleport the avatar to the valid position.
# Enable image capture.
self.communicate([
{
"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {
"x": -1.5,
"y": 2.5,
"z": 7
}
},
{
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img", "_id"]
},
{
"$type": "send_images",
"frequency": "always"
},
])
# place the ramp
ramp_id = self.add_object("ramp_with_platform_30",
position={
"x": -2.1,
"y": 0,
"z": 0
})
self.communicate([{
"$type": "scale_object",
"id": ramp_id,
"scale_factor": (1, 1, 1)
}])
self.communicate([{
"$type": "set_kinematic_state",
"id": ramp_id,
"is_kinematic": True,
"use_gravity": False
}])
self.communicate([{
"$type": "set_visual_material",
"old_material_index": 1,
"new_material_name": "3d_mesh_technical_fabric",
"object_name": "ramp_with_platform_30",
"id": ramp_id
}])
# place the elevated platform
platform_id = self.add_object("fridge_box",
position={
"x": 1.5,
"y": 0,
"z": 0
})
self.communicate({
"$type": "set_kinematic_state",
"id": platform_id,
"is_kinematic": True,
"use_gravity": False
})
# place a collider object on the ramp (salt_mill_max_2013,b05_fire_extinguisher)
collider_id = self.add_object("b05_fire_extinguisher",
position={
"x": -4,
"y": 0,
"z": 0
})
# Set the scale, mass, and friction of the object.
self.communicate({
"$type": "scale_object",
"id": collider_id,
"scale_factor": {
"x": 2,
"y": 2,
"z": 2
}
})
self.communicate({"$type": "set_mass", "id": collider_id, "mass": 1})
self.communicate({
"$type": "set_physic_material",
"id": collider_id,
"static_friction": 0.5,
"bounciness": 0.5
})
#self.communicate({"$type":"set_color",
# "color":{"r":1, "g":0, "b":1, "a":1},
# "id": collider_id})
#self.communicate({"$type": "nudge_onto_surface",
# "id": collider_id,
# "is_circle": True,
# "nudge_step": .05,
# "surface_object_id": ramp_id})
# place the basket on the ramp (bucketnew, bowl_wood_a_01)
if 0:
basket_id = self.add_object("bowl_wood_a_01",
position={
"x": 0.1,
"y": 1,
"z": 0
})
self.communicate({
"$type": "set_kinematic_state",
"id": basket_id,
"is_kinematic": True,
"use_gravity": False
})
# Create the object.
# b04_geosphere001, chair_billiani_doll, b04_orange_00, b05_1_(1), b05_baseballnew_v03_12, b05_geosphere001, base-ball, prim_sphere
object_id = self.add_object("b04_geosphere001",
position={
"x": 0.2,
"y": 1,
"z": 0
})
# Set the scale, mass, and friction of the object.
self.communicate({
"$type": "scale_object",
"id": object_id,
"scale_factor": {
"x": 10,
"y": 10,
"z": 10
}
})
self.communicate({"$type": "set_mass", "id": object_id, "mass": m_o})
self.communicate({
"$type": "set_physic_material",
"id": object_id,
"static_friction": s_o,
"bounciness": b_o
})
self.communicate({
"$type": "set_color",
"color": {
"r": 0.5,
"g": 1,
"b": 0,
"a": 1
},
"id": object_id
})
# nudge the ball onto the surface of the platform
# self.communicate({"$type": "nudge_onto_surface",
# "id": object_id,
# "is_circle": False,
# "nudge_step": 1.0,
# "surface_object_id": ramp_id})
# apply force to the ball
self.communicate([{
"$type": "apply_force_to_object",
"force": {
"x": -8,
"y": -0.1,
"z": 0.5
},
"id": object_id
}])
# Create the output directory.
if os.path.exists(output_directory):
shutil.rmtree(output_directory)
sleep(0.5)
os.mkdir(output_directory)
# Capture n images.
for i in range(nimages):
# Look at the object.
resp = self.communicate({
"$type": "look_at_position",
"avatar_id": "a",
"position": {
"x": -1.5,
"y": 1,
"z": 0
}
})
images = Images(resp[0])
frame = resp[-1]
# Save the image.
TDWUtils.save_images(images,
frame,
output_directory=output_directory)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("laser_cut_white_norway_spruce")
self.communicate({
"$type": "add_material",
"name": "laser_cut_white_norway_spruce",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "laser_cut_white_norway_spruce"
})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 1.5,
"y": 1.6,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[0, 0.5, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table = self.add_object(
model_name="boconcept_lugo_dining_table_vray1.5",
position={
"x": 0,
"y": 0,
"z": 0.0
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table)
top = table_bounds.get_top(0)
self.add_object(model_name='macbook_001',
position={
"x": 0,
"y": top[1],
"z": 0.1
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
self.add_object(model_name='spunlight_designermesh_lamp',
position={
"x": -0.19,
"y": top[1],
"z": -0.4
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
self.add_object(model_name="apple_magic_mouse_2_(2015)_vray",
position={
"x": 0,
"y": top[1],
"z": 0.32
},
rotation={
"x": 0,
"y": 86,
"z": 0
},
library="models_full.json")
self.add_object(model_name="f10_apple_iphone_4",
position={
"x": 0.14,
"y": top[1],
"z": -0.3
},
rotation={
"x": 0,
"y": 12,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.5, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"phone",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("parquet_wood_mahogany")
self.communicate({
"$type": "add_material",
"name": "parquet_wood_mahogany",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "parquet_wood_mahogany"
})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -1,
"y": 1.4,
"z": -1
},
look_at=TDWUtils.array_to_vector3(
[-0.1, 0.7, -0.1]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table_id = self.add_object(
model_name=
"b03_restoration_hardware_pedestal_salvaged_round_tables",
position={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="club_sandwich",
position={
"x": 0.45,
"y": top[1],
"z": 0.25
},
rotation={
"x": 0,
"y": 30,
"z": 0
},
library="models_full.json")
self.add_object(model_name="coffeemug",
position={
"x": 0.03,
"y": top[1],
"z": 0.03
},
library="models_full.json")
self.add_object(model_name="knife3",
position={
"x": -0.17,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 24,
"z": 90
},
library="models_full.json")
self.add_object(model_name="notes_02",
position={
"x": 0.17,
"y": top[1],
"z": -0.12
},
rotation={
"x": 0,
"y": 24,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([-0.1, 0.7, -0.1])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"sadnwich2",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
# Create the output directory.
output_directory = "example_output"
if os.path.exists(output_directory):
shutil.rmtree(output_directory)
sleep(0.5)
os.mkdir(output_directory)
self.start()
# Load a streamed scene. This scene covers a relatively small area, and is optimized for use with HDRI maps.
self.load_streamed_scene(scene="building_site")
# Add the object.
lamp_id = self.add_object("alma_floor_lamp",
position={
"x": 0,
"y": 0,
"z": 0
},
rotation={
"x": 0,
"y": 90,
"z": 0
})
# Create the avatar.
self.communicate({
"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"
})
# Change the skybox.
self.communicate(self.get_add_hdri_skybox("bergen_4k"))
# Teleport the avatar to a suitable position.
# Set the pass masks to _img.
# Enable image capture.
self.communicate([
{
"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {
"x": -4.28,
"y": 0.85,
"z": 4.27
}
},
{
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img"]
},
{
"$type": "send_images",
"frequency": "always"
},
])
# Create the recommended post-processing setup for HDRI.
self.communicate([{
"$type": "set_post_exposure",
"post_exposure": 0.6
}, {
"$type": "set_contrast",
"contrast": -20
}, {
"$type": "set_saturation",
"saturation": 10
}, {
"$type": "set_screen_space_reflections",
"enabled": False
}, {
"$type": "set_vignette",
"enabled": False
}])
# Set the shadow strength to maximum.
self.communicate({"$type": "set_shadow_strength", "strength": 1.0})
# Capture 48 images.
for i in range(48):
# Look at the lamp.
resp = self.communicate({
"$type": "look_at",
"avatar_id": "a",
"object_id": lamp_id,
"use_centroid": True
})
# Rotate the skybox by 15 degrees.
self.communicate({"$type": "rotate_hdri_skybox_by", "angle": 15})
images = Images(resp[0])
# Save the image.
TDWUtils.save_images(images,
TDWUtils.zero_padding(i),
output_directory=output_directory)
def run(self):
self.start()
# init_setup_commands = [{"$type": "set_screen_size",
# "width": 600,
# "height": 480},
# {"$type": "set_render_quality",
# "render_quality": 5}]
# self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 2.5,
"y": 1.5,
"z": 0.3
},
look_at=TDWUtils.array_to_vector3(
[3, 0.5, 0.5]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("gravel_white")
self.communicate({
"$type": "add_material",
"name": "gravel_white",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "gravel_white"
})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
bench = self.add_object(model_name="b04_wood_metal_park_bench",
position={
"x": 3,
"y": 0,
"z": 0.5
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
bench_bounds = self.get_bounds_data(bench)
top = bench_bounds.get_top(0)
self.add_object(model_name="b05_baseball_bat_01",
position={
"x": 2.8,
"y": top[1] - 0.35,
"z": 0
},
rotation={
"x": 0,
"y": 90,
"z": 90
},
library="models_full.json")
self.add_object(model_name="b05_baseballnew_v03_12",
position={
"x": 2.9,
"y": top[1] - 0.35,
"z": 0.4
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([3, 0.5, 0.5])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"bench_ball",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 2.0,
"y": 0.8,
"z": -0.55
},
look_at=TDWUtils.array_to_vector3(
[3, 0.6, 0.35]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table = self.add_object(model_name="b05_table_new",
position={
"x": 3,
"y": 0,
"z": 0.35
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
# record = ModelLibrarian(library='models_full.json').get_record("b03_simpsons_london_-_round_hudson_mirror")
# self.communicate({"$type": "add_object",
# "name": "b05_tv1970",
# "url": record.get_url(),
# "scale_factor": 0.5,
# "position": {"x": 3, "y": 0.6, "z": 0.35},
# "rotation": {"x": 0, "y": -90, "z": 0},
# "category": record.wcategory,
# "id": self.get_unique_id()})
table_bounds = self.get_bounds_data(table)
top = table_bounds.get_top(0)
self.add_object(model_name="b03_tv_controller_2013__vray",
position={
"x": 3,
"y": top[1],
"z": 0.7
},
rotation={
"x": 0,
"y": 0,
"z": 90
},
library="models_full.json")
#
# self.add_object(model_name='cgaxis_models_65_06_vray',
# position={"x": 3, "y": top[1], "z": 0.25},
# rotation={"x": 0, "y": 0, "z": 0},
# library="models_full.json")
self.add_object(model_name="b04_phone",
position={
"x": 3,
"y": top[1],
"z": 0.25
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
self.add_object(model_name="books_collection__1_2013",
position={
"x": 3.3,
"y": top[1],
"z": -0.2
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([3, 0.5, 0.35])
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"remote1",
output_directory="replicated_images")
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": -0.8, "y": 1.6, "z": 0.6},
look_at=TDWUtils.array_to_vector3([0, 0.4, 0]),
avatar_id="avatar"))
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
table_id = self.add_object(model_name="glass_table_round",
position={"x": 0, "y": 0, "z": 0},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="b03_orange",
position={"x": 0, "y": top[1], "z": 0},
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.17)
position[1] = top[1]
self.add_object(model_name="b03_orange",
position={"x": 0, "y": top[1], "z": 0},
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.4)
position[1] = top[1]
self.add_object(model_name="b04_banana",
position=TDWUtils.array_to_vector3(position),
rotation={"x": 0, "y": 40, "z": 0},
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.4)
position[1] = top[1]
self.add_object(model_name="b04_banana",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.5)
position[1] = top[1]
self.add_object(model_name="red_apple",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.6)
position[1] = top[1]
self.add_object(model_name="vk0076_fruitfork",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([0, 0.4, 0])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "fruit_table", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior")
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -1,
"y": 0.9,
"z": -1
},
look_at=TDWUtils.array_to_vector3(
[-0.1, 0.5, -0.1]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table_id = self.add_object(model_name="coffee_table_glass_round",
position={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="am151_043_00",
position={
"x": 0.15,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 30,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b04_tea_cup",
position={
"x": 0.3,
"y": top[1],
"z": 0.1
},
library="models_full.json")
self.add_object(model_name="cgaxis_models_65_06_vray",
position={
"x": -0.17,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 24,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([-0.1, 0.45, -0.1])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"sadnwich1",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
# init_setup_commands = [{"$type": "set_screen_size",
# "width": 600,
# "height": 480},
# {"$type": "set_render_quality",
# "render_quality": 5}]
# self.communicate(init_setup_commands)
# Load the streamed scene.
self.load_streamed_scene(scene="tdw_room_2018")
# Add the objects.
self.add_object("live_edge_coffee_table",
position={"x": -12.8, "y": 0.96, "z": -5.47},
rotation={"x": 0, "y": -90, "z": 0})
self.add_object(model_name="03_106",
position={"x": -14.394, "y": 0.96, "z": -7.06},
rotation={"x": 0, "y": 21.35, "z": 0})
self.add_object("chair_billiani_doll",
position={"x": -15.15, "y": 0.96, "z": -6.8},
rotation={"x": 0, "y": 63.25, "z": 0})
# self.add_object("zenblocks",
# position={"x": -12.7, "y": 1.288, "z": -5.55},
# rotation={"x": 0, "y": 90, "z": 0})
# Organize all setup commands into a single list.
# We want a high-quality result, so set 1080P screen resolution / aspect ratio
# and maximum render quality.
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create the avatar and adjust its field of view for a wider camera angle.
init_setup_commands.extend([{"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"},
{"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "a"}])
# Adjust post-processing parameters to achieve a suitable depth of field and exposure, and disable vignette.
# Also adjust the ambient occlusion parameters for realistic shadowing in corners and under furniture objects.
init_setup_commands.extend([{"$type": "set_aperture",
"aperture": 1.6},
{"$type": "set_focus_distance",
"focus_distance": 2.25},
{"$type": "set_post_exposure",
"post_exposure": 0.4},
{"$type": "set_vignette",
"enabled": False},
{"$type": "set_ambient_occlusion_intensity",
"intensity": 0.175},
{"$type": "set_ambient_occlusion_thickness_modifier",
"thickness": 3.5}])
# Set shadow strength to full.
init_setup_commands.append({"$type":
"set_shadow_strength",
"strength": 1.0})
# Execute the setup commands.
self.communicate(init_setup_commands)
# Teleport the avatar to the desired position.
# Set the pass masks to _img.
# Enable image capture.
resp = self.communicate([{"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {"x": -10.48, "y": 1.81, "z": -6.583}},
{"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img"]},
{"$type": "send_images",
"frequency": "always"},
{"$type": "look_at_position",
"position": {"x": -12.873, "y": 1.85, "z": -5.75},
"avatar_id": "a"}])
# Parse the output image data.
images = Images(resp[0])
# Save the image.
TDWUtils.save_images(images, "photoreal_living", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior")
"$type": "look_at",
"avatar_id": avatar_id,
"object_id": box_id
}, {
"$type": "set_pass_masks",
"avatar_id": avatar_id,
"pass_masks": ["_img"]
}, {
"$type": "send_images",
"frequency": "once",
"avatar_id": avatar_id
}])
# Get the image.
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
# Find the image data.
if r_id == "imag":
img = Images(r)
# Usually, you'll want to use one of these functions, but not both of them:
# Use this to save a .jpg
TDWUtils.save_images(img, filename="test_img")
print(f"Image saved to: {Path('dist/test_img.jpg').resolve()}")
# Use this to convert the image to a PIL image, which can be processed by a ML system at runtime.
# The index is 0 because we know that there is only one pass ("_img").
pil_img = TDWUtils.get_pil_image(img, index=0)
