def _write_frame(self, frames_grp: h5py.Group, resp: List[bytes], frame_num: int) -> \
Tuple[h5py.Group, h5py.Group, dict, bool]:
num_objects = len(self.object_ids)
# Create a group for this frame.
frame = frames_grp.create_group(TDWUtils.zero_padding(frame_num, 4))
# Create a group for images.
images = frame.create_group("images")
# Transforms data.
positions = np.empty(dtype=np.float32, shape=(num_objects, 3))
forwards = np.empty(dtype=np.float32, shape=(num_objects, 3))
rotations = np.empty(dtype=np.float32, shape=(num_objects, 4))
camera_matrices = frame.create_group("camera_matrices")
# Parse the data in an ordered manner so that it can be mapped back to the object IDs.
tr_dict = dict()
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
if r_id == "tran":
tr = Transforms(r)
for i in range(tr.get_num()):
pos = tr.get_position(i)
tr_dict.update({
tr.get_id(i): {
"pos": pos,
"for": tr.get_forward(i),
"rot": tr.get_rotation(i)
}
})
# Add the Transforms data.
for o_id, i in zip(self.object_ids, range(num_objects)):
if o_id not in tr_dict:
continue
positions[i] = tr_dict[o_id]["pos"]
forwards[i] = tr_dict[o_id]["for"]
rotations[i] = tr_dict[o_id]["rot"]
elif r_id == "imag":
im = Images(r)
# Add each image.
for i in range(im.get_num_passes()):
pass_mask = im.get_pass_mask(i)
# Reshape the depth pass array.
if pass_mask == "_depth":
image_data = TDWUtils.get_shaped_depth_pass(images=im,
index=i)
else:
image_data = im.get_image(i)
images.create_dataset(pass_mask,
data=image_data,
compression="gzip")
# Add the camera matrices.
elif OutputData.get_data_type_id(r) == "cama":
matrices = CameraMatrices(r)
camera_matrices.create_dataset(
"projection_matrix", data=matrices.get_projection_matrix())
camera_matrices.create_dataset(
"camera_matrix", data=matrices.get_camera_matrix())
objs = frame.create_group("objects")
objs.create_dataset("positions",
data=positions.reshape(num_objects, 3),
compression="gzip")
objs.create_dataset("forwards",
data=forwards.reshape(num_objects, 3),
compression="gzip")
objs.create_dataset("rotations",
data=rotations.reshape(num_objects, 4),
compression="gzip")
return frame, objs, tr_dict, False
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate({
"$type": "create_empty_environment",
"center": {
"x": 0,
"y": 0,
"z": 0
},
"bounds": {
"x": 8,
"y": 8,
"z": 8
}
})
self.communicate({"$type": "set_gravity", "value": False})
cube_record = ModelLibrarian("models_special.json").get_record(
"prim_cube")
self.communicate({
"$type": "add_object",
"name": "prim_cube",
"url": cube_record.get_url(),
"scale_factor": cube_record.scale_factor,
"position": {
"x": 0,
"y": 0,
"z": 0
},
"rotation": {
"x": 0,
"y": 0,
"z": 0
},
"category": cube_record.wcategory,
"id": 1
})
self.communicate({
"$type": "scale_object",
"scale_factor": {
"x": 30,
"y": 0.0001,
"z": 30
},
"id": 1
})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 1.6,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[0.5, 1.5, 0]),
avatar_id="avatar"))
self.communicate(self.get_add_hdri_skybox("table_mountain_1_4k"))
self.communicate({"$type": "rotate_hdri_skybox_by", "angle": 90})
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("iceland_black_sand_beach")
self.communicate({
"$type": "add_material",
"name": "iceland_black_sand_beach",
"url": record.get_url()
})
self.communicate(
TDWUtils.set_visual_material(c=self,
substructure=cube_record.substructure,
object_id=1,
material="iceland_black_sand_beach"))
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
board = self.add_object(model_name="surfboard_3",
position={
"x": 3.5,
"y": 0,
"z": -0.5
},
rotation={
"x": 0,
"y": 30,
"z": -90
},
library="models_full.json")
self.communicate({
"$type": "scale_object",
"scale_factor": {
"x": 1.5,
"y": 1.5,
"z": 1.5
},
"id": board
})
board_bounds = self.get_bounds_data(board)
top = board_bounds.get_top(0)
self.add_object(model_name="animal_dog_rtsit_1282",
position={
"x": top[0],
"y": top[1],
"z": top[2]
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
self.add_object(model_name="3dscan_man_003",
position={
"x": 3.3,
"y": 0,
"z": 0.7
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0.5, 1.5, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"surfboard1",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def do_panorama(self, model_name):
# Create the model.
object_id = self.add_object(model_name)
# Teleport the avatar.
self.communicate({
"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {
"x": 5,
"y": 2.5,
"z": 0
}
})
# Set the starting parameters of the rotation.
d_theta = 15
rotations = int(360 / d_theta)
theta = 0
x = 3
y = 2.5
z = 0
c_x = 0
c_z = 0
# Capture images.
self.communicate({"$type": "send_images", "frequency": "always"})
for i in range(rotations):
# Get the new position.
rad = radians(theta)
x_rot = cos(rad) * (x - c_x) - sin(rad) * (z - c_z) + c_x
z_rot = sin(rad) * (x - c_x) + cos(rad) * (z - c_z) + c_z
# Teleport the avatar to the next position.
# Rotate the avatar's camera to look at the object.
# Receive an image.
resp = self.communicate([{
"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {
"x": x_rot,
"y": y,
"z": z_rot
}
}, {
"$type": "look_at",
"avatar_id": "a",
"object_id": object_id
}])
# Save the images.
for r in resp[:-1]:
TDWUtils.save_images(Images(r), str(TDWUtils.zero_padding(i)),
str(self.output_directory.resolve()))
# Increment the angle.
theta += d_theta
# Stop image capture.
# Destroy the target object.
self.communicate([{
"$type": "send_images",
"frequency": "never"
}, {
"$type": "destroy_object",
"id": object_id
}])
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": 0, "y": 1, "z": 0},
look_at=TDWUtils.array_to_vector3([3, 0.5, 0.5]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("iceland_grassy_cliff")
self.communicate({"$type": "add_material", "name": "iceland_grassy_cliff", "url": record.get_url()})
self.communicate({"$type": "set_proc_gen_floor_material", "name": "iceland_grassy_cliff"})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
bench = self.add_object(model_name="b04_wood_metal_park_bench",
position={"x": 3, "y": 0, "z": 0.5},
rotation={"x": 0, "y": 0, "z": 0},
library="models_full.json")
bench_bounds = self.get_bounds_data(bench)
top = bench_bounds.get_top(0)
self.add_object(model_name="animal_dog_rtsit_1282",
position={"x": 2.83, "y": top[1], "z": 0.52},
rotation={"x": 0, "y": 110, "z": 0},
library="models_full.json")
self.add_object(model_name="shar_pei",
position={"x": 2.6, "y": top[1] - 0.35, "z": 0.25},
rotation={"x": 0, "y": 0, "z": 0},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([3, 0.5, 0.5])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "bench_dog",
output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate({
"$type": "create_empty_environment",
"center": {
"x": 0,
"y": 0,
"z": 0
},
"bounds": {
"x": 15,
"y": 15,
"z": 15
}
})
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 5,
"y": 1.4,
"z": 0.6
},
look_at=TDWUtils.array_to_vector3(
[0, 0.5, 0.5]),
avatar_id="avatar"))
self.communicate(self.get_add_hdri_skybox("sky_white"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
cube_record = ModelLibrarian("models_special.json").get_record(
"prim_cube")
self.communicate({
"$type": "add_object",
"name": "prim_cube",
"url": cube_record.get_url(),
"scale_factor": cube_record.scale_factor,
"position": {
"x": 0,
"y": 0,
"z": 0
},
"rotation": {
"x": 0,
"y": 0,
"z": 0
},
"category": cube_record.wcategory,
"id": 1
})
self.communicate({
"$type": "scale_object",
"scale_factor": {
"x": 30,
"y": 0.0001,
"z": 30
},
"id": 1
})
grass_record = MaterialLibrarian("materials_high.json").get_record(
"iceland_mossy_meadow")
self.communicate({
"$type": "add_material",
"name": "iceland_mossy_meadow",
"url": grass_record.get_url()
})
self.communicate(
TDWUtils.set_visual_material(c=self,
substructure=cube_record.substructure,
object_id=1,
material="iceland_mossy_meadow"))
bench = self.add_object(model_name="b04_wood_metal_park_bench",
position={
"x": 3,
"y": 0,
"z": 0.5
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
bench_bounds = self.get_bounds_data(bench)
top = bench_bounds.get_top(0)
self.add_object(model_name="3dscan_man_004",
position={
"x": 2.5,
"y": 0,
"z": 0
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
hummingbird = self.add_object(model_name="b04_pigeon",
position={
"x": 3,
"y": top[1] - 0.05,
"z": 0.75
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b03_realistic_pigeon_max",
position={
"x": 3,
"y": top[1] - 0.05,
"z": 0.35
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.5, 0.5])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"bird1",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def get_physics_quality(self) -> float:
"""
Get the physics quality of the model.
First, capture multiple images at different angles.
Then, obscure the object with its own colliders and capture the same images.
Compare the _id capture pass pixel count between each set of images.
If the hull colliders obscure the entirety of the object in every image, physics_quality = 1.0
"""
self.start()
self.communicate([{"$type": "simulate_physics",
"value": False},
{"$type": "create_empty_environment"}])
avatar_id = "A"
delta_theta = 15
print("Writing the physics quality to the record.")
scale = TDWUtils.get_unit_scale(self.record)
# Add the object.
object_id = self.get_unique_id()
self.communicate({"$type": "add_object",
"name": self.record.name,
"url": self.asset_bundle_path,
"scale_factor": self.record.scale_factor,
"id": object_id})
# Scale the object to unit size.
self.communicate({"$type": "scale_object",
"id": object_id,
"scale_factor": {"x": scale, "y": scale, "z": scale}})
# Create the avatar.
# Set the pass masks.
self.communicate(TDWUtils.create_avatar(avatar_id=avatar_id))
self.communicate({"$type": "set_pass_masks",
"avatar_id": avatar_id,
"pass_masks": ["_id"]})
id_pass = []
pink_pass = []
for show_collider_hulls in [False, True]:
x = 1.75
y = 0.5
z = 0
# Show collider hulls.
if show_collider_hulls:
self.communicate({"$type": "show_collider_hulls",
"id": object_id})
# Reset the avatar.
resp = self.communicate([{"$type": "teleport_avatar_to",
"avatar_id": avatar_id,
"position": {"x": x, "y": y, "z": z}},
{"$type": "look_at",
"avatar_id": avatar_id,
"object_id": object_id,
"use_centroid": True},
{"$type": "send_images",
"frequency": "always"}])
# Equatorial orbit.
theta = 0
while theta < 360:
# Get the number of pixels that aren't black.
img = np.array(Image.open(BytesIO(Images(resp[0]).get_image(0))))
grade = (256 * 256) - np.sum(np.all(img == np.array([0, 0, 0]), axis=2))
if show_collider_hulls:
pink_pass.append(grade)
else:
id_pass.append(grade)
theta += delta_theta
# Get the new position.
rad = radians(theta)
x1 = cos(rad) * x - sin(rad) * z
z1 = sin(rad) * x + cos(rad) * z
# Teleport the avatar.
# Look at the object.
resp = self.communicate([{"$type": "teleport_avatar_to",
"avatar_id": avatar_id,
"position": {"x": x1, "y": y, "z": z1}},
{"$type": "look_at",
"avatar_id": avatar_id,
"object_id": object_id,
"use_centroid": True}
])
# Reset the avatar.
resp = self.communicate([{"$type": "teleport_avatar_to",
"avatar_id": avatar_id,
"position": {"x": x, "y": y, "z": z}},
{"$type": "look_at",
"avatar_id": avatar_id,
"object_id": object_id,
"use_centroid": True},
{"$type": "send_images",
"frequency": "always"}])
# Polar orbit.
theta = 0
while theta < 360:
# Get the number of pixels that aren't black.
img = np.array(Image.open(BytesIO(Images(resp[0]).get_image(0))))
grade = (256 * 256) - np.sum(np.all(img == np.array([0, 0, 0]), axis=2))
if show_collider_hulls:
pink_pass.append(grade)
else:
id_pass.append(grade)
theta += delta_theta
# Get the new position.
rad = radians(theta)
x1 = cos(rad) * x - sin(rad) * z
y1 = (sin(rad) * x + cos(rad) * z) + y
# Teleport the avatar.
# Look at the object.
resp = self.communicate([{"$type": "teleport_avatar_to",
"avatar_id": avatar_id,
"position": {"x": x1, "y": y1, "z": 0}},
{"$type": "look_at",
"avatar_id": avatar_id,
"object_id": object_id,
"use_centroid": True}
])
grades = [1 - (float(h) / float(i)) for h, i in zip(pink_pass, id_pass)]
physics_quality = float(sum(grades)) / len(grades)
print("Physics quality: " + str(physics_quality))
# Kill the build.
self.kill_build()
return physics_quality
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 1280,
"height": 962},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": 2, "y": 1.5, "z": 0},
look_at=TDWUtils.array_to_vector3([2, 0.8, -4]),
avatar_id="avatar"))
self.communicate({"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"})
self.add_object(model_name="kenmore_refr_74049",
position={"x": 3, "y": 0, "z": -4},
rotation={"x": 0, "y": 90, "z": 0},
library="models_full.json")
record = ModelLibrarian(library='models_full.json').get_record("b03_bosch_cbg675bs1b_2013__vray_composite")
self.communicate({"$type": "add_object",
"name": "b03_bosch_cbg675bs1b_2013__vray_composite",
"url": record.get_url(),
"scale_factor": 1.5,
"position": {"x": 2, "y": 0, "z": -4},
"category": record.wcategory,
"id": self.get_unique_id()})
table_id = self.add_object(model_name="b03_restoration_hardware_pedestal_salvaged_round_tables",
position={"x": 1, "y": 0, "z": -4},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="cgaxis_models_65_06_vray",
position={"x": 1.2, "y": top[1], "z": -4.2},
library="models_full.json")
self.add_object(model_name="b04_heineken_beer_vray",
position={"x": 0.77, "y": top[1], "z": -3.9},
library="models_full.json")
self.add_object(model_name="b03_backpack",
position={"x": 1.3, "y": 0, "z": -3.7},
rotation={"x": 0, "y": 20, "z": 0},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([2, 0.8, -4])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "fridge", output_directory="replicated_images")
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 1280,
"height": 962
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
width = 12
length = 12
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(width, length))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 1.7,
"z": -4
},
look_at={
"x": 0,
"y": 0.8,
"z": 0
},
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 78.0,
"avatar_id": "avatar"
})
# Create the dining table.
self.dining_table()
# Create the sofa.
self.sofa(width, length)
# Bake the NavMesh.
self.communicate({"$type": "bake_nav_mesh"})
print('mesh')
# Get a random position on the NavMesh.
position = TDWUtils.get_random_position_on_nav_mesh(
self, width, length)
position = TDWUtils.array_to_vector3(position)
# Add a side table.
self.side_table(position)
self.add_object(model_name="samsung_refr_rf28k9380sg",
position={
"x": -1,
"y": 0,
"z": 2.5
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type": "look_at_position",
"avatar_id": "avatar",
"position": {
"x": 0,
"y": 0.8,
"z": 0
}
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"kitchen2",
output_directory="replicated_images")
def run(self):
# Setting up image directory
output_directory = "floating_bowl_images"
parent_dir = '/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/floating_bowl'
path = os.path.join(parent_dir, output_directory)
print(path)
if os.path.exists(path):
shutil.rmtree(path)
time.sleep(0.5)
os.mkdir(path)
# Initialize scene
self.start()
# Load in scene
scene_lib = SceneLibrarian()
scene_record = scene_lib.get_record("tdw_room_2018")
self.communicate({"$type": "add_scene",
"name": scene_record.name,
"url": scene_record.get_url()})
# Resize screen; note render_quality 5 is best
self.communicate([{"$type": "set_screen_size",
"width": 1280,
"height": 962},
{"$type": "set_render_quality",
"render_quality": 2}])
# Toggle physics
physics_on = True
self.communicate({"$type": "simulate_physics",
"value": physics_on})
# Load in avatar
self.communicate(TDWUtils.create_avatar(avatar_id="avatar",
position={"x": 3, "y": 1.5, "z": 3},
look_at={"x": 0, "y": 0.8, "z": 0}))
# Necessary commands to avoid error in calling save_images()
# Second command enables image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
# Capture images
for i in range(20):
bowl_id = self.add_object(model_name="b04_bowl_smooth",
position={"x": 0, "y": i / 10, "z": 0},
rotation={"x": i * 30 / 10, "y": i / 10, "z": 0},
library="models_full.json")
resp = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": {"x": 0, "y": 0.8, "z": 0}})
images = Images(resp[0])
TDWUtils.save_images(images, TDWUtils.zero_padding(i), output_directory=path)
self.communicate({"$type": "destroy_object",
"id": bowl_id})
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -0.8,
"y": 2,
"z": 0.9
},
look_at=TDWUtils.array_to_vector3(
[0, 0.8, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table_id = self.add_object(
model_name="enzo_industrial_loft_pine_metal_round_dining_table",
position={
"x": 0,
"y": 0,
"z": 0
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="b04_cake_022(vray_2013)",
position={
"x": -0.2,
"y": top[1],
"z": 0.3
},
library="models_full.json")
self.add_object(model_name="yellow_cheesecake",
position={
"x": 0.3,
"y": top[1],
"z": -0.13
},
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.55)
position[1] = top[1]
self.add_object(model_name="b03_donut_max2012",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
position = TDWUtils.get_random_point_in_circle([0, 0, 0], 0.5)
position[1] = top[1]
self.add_object(model_name="b04_bottle_20ml",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.4, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"desserts_table",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
width = 8
length = 8
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(width, length))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 1.7,
"z": 0
},
look_at={
"x": 0.3,
"y": 0.8,
"z": 2.2
},
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
bed = self.add_object(model_name="cgaxis_models_49_38_vray",
position={
"x": 0.3,
"y": 0,
"z": 2.2
},
rotation={
"x": 0,
"y": 180,
"z": 0
},
library="models_full.json")
bed_bounds = self.get_bounds_data(bed)
top = bed_bounds.get_top(0)
# Create the dining table.
# self.dining_table()
# Create the sofa.
# self.sofa(width, length)
# Bake the NavMesh.
self.communicate({"$type": "bake_nav_mesh"})
print('mesh')
# Get a random position on the NavMesh.
position = {"x": -0.8, "y": 0, "z": 2}
# Add a side table.
self.side_table(position)
position = [
0.2 + np.random.uniform(-0.1, 0.1), 0.8,
1.8 + np.random.uniform(-0.1, 0.1)
]
self.add_object(model_name="b04_dog",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type": "look_at_position",
"avatar_id": "avatar",
"position": {
"x": 0.3,
"y": 0.8,
"z": 2.2
}
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"dog_bed",
output_directory="replicated_images/interior")
"$type": "look_at",
"avatar_id": avatar_id,
"object_id": box_id
}, {
"$type": "set_pass_masks",
"avatar_id": avatar_id,
"pass_masks": ["_img"]
}, {
"$type": "send_images",
"frequency": "once",
"avatar_id": avatar_id
}])
# Get the image.
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
# Find the image data.
if r_id == "imag":
img = Images(r)
# Usually, you'll want to use one of these functions, but not both of them:
# Use this to save a .jpg
TDWUtils.save_images(img, filename="test_img")
print(f"Image saved to: {Path('dist/test_img.jpg').resolve()}")
# Use this to convert the image to a PIL image, which can be processed by a ML system at runtime.
# The index is 0 because we know that there is only one pass ("_img").
pil_img = TDWUtils.get_pil_image(img, index=0)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0.75,
"y": 1,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[3, 0.7, 0]),
avatar_id="avatar"))
# self.communicate({"$type": "set_field_of_view",
# "field_of_view": 68.0,
# "avatar_id": "avatar"})
self.add_object(model_name="12_01_001",
position={
"x": 3,
"y": 0,
"z": -0.35
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
sink = self.add_object(model_name="cgaxis_models_22_33_vray",
position={
"x": 3,
"y": 0,
"z": 0.35
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
record = ModelLibrarian(library='models_full.json').get_record(
"b03_simpsons_london_-_round_hudson_mirror")
self.communicate({
"$type": "add_object",
"name": "b05_tv1970",
"url": record.get_url(),
"scale_factor": 0.5,
"position": {
"x": 3,
"y": 0.6,
"z": 0.35
},
"rotation": {
"x": 0,
"y": -90,
"z": 0
},
"category": record.wcategory,
"id": self.get_unique_id()
})
sink_bounds = self.get_bounds_data(sink)
top = sink_bounds.get_top(0)
self.add_object(model_name="808409_toothbrush",
position={
"x": 3,
"y": top[1] - 0.217,
"z": 0.7
},
rotation={
"x": -20,
"y": -40,
"z": 90
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([3, 0.7, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"bathroom1",
output_directory="replicated_images")
def run(self):
# Create the output directory.
output_directory = Path("photoreal")
if not output_directory.exists():
output_directory.mkdir()
# Load the streamed scene.
self.load_streamed_scene(scene="archviz_house_2018")
# Add the objects.
self.add_object("live_edge_coffee_table",
position={
"x": -12.8,
"y": 0.96,
"z": -5.47
},
rotation={
"x": 0,
"y": -90,
"z": 0
})
self.add_object("chista_slice_of_teak_table",
position={
"x": -14.394,
"y": 0.96,
"z": -7.06
},
rotation={
"x": 0,
"y": 21.35,
"z": 0
})
self.add_object("chair_billiani_doll",
position={
"x": -15.15,
"y": 0.96,
"z": -6.8
},
rotation={
"x": 0,
"y": 63.25,
"z": 0
})
self.add_object("zenblocks",
position={
"x": -12.7,
"y": 1.288,
"z": -5.55
},
rotation={
"x": 0,
"y": 90,
"z": 0
})
# Organize all setup commands into a single list.
# We want a high-quality result, so set 1080P screen resolution / aspect ratio
# and maximum render quality.
init_setup_commands = [{
"$type": "set_screen_size",
"width": 1920,
"height": 1080
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
# Create the avatar and adjust its field of view for a wider camera angle.
init_setup_commands.extend([{
"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"
}, {
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "a"
}])
# Adjust post-processing parameters to achieve a suitable depth of field and exposure, and disable vignette.
# Also adjust the ambient occlusion parameters for realistic shadowing in corners and under furniture objects.
init_setup_commands.extend([{
"$type": "set_aperture",
"aperture": 1.6
}, {
"$type": "set_focus_distance",
"focus_distance": 2.25
}, {
"$type": "set_post_exposure",
"post_exposure": 0.4
}, {
"$type": "set_vignette",
"enabled": False
}, {
"$type": "set_ambient_occlusion_intensity",
"intensity": 0.175
}, {
"$type": "set_ambient_occlusion_thickness_modifier",
"thickness": 3.5
}])
# Set shadow strength to full.
init_setup_commands.append({
"$type": "set_shadow_strength",
"strength": 1.0
})
# Execute the setup commands.
self.communicate(init_setup_commands)
# Teleport the avatar to the desired position.
# Set the pass masks to _img.
# Enable image capture.
resp = self.communicate([{
"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {
"x": -10.48,
"y": 1.81,
"z": -6.583
}
}, {
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img"]
}, {
"$type": "send_images",
"frequency": "always"
}, {
"$type": "look_at_position",
"position": {
"x": -12.873,
"y": 1.85,
"z": -5.75
},
"avatar_id": "a"
}])
# Parse the output image data.
images = Images(resp[0])
# Save the image.
TDWUtils.save_images(images,
TDWUtils.zero_padding(0),
output_directory=str(output_directory.resolve()))
print(f"Image saved to: {output_directory.resolve()}")
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(16, 16))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -7.5,
"y": 1.5,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[0, 0.7, 0]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("concrete_asphalt_rolled")
self.communicate({
"$type": "add_material",
"name": "concrete_asphalt_rolled",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "concrete_asphalt_rolled"
})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
self.add_object(model_name="b05_2018_chevy_tahoe_rst",
position={
"x": 3,
"y": 0,
"z": 0.5
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b05_chevrolet_camaro_3ds_max_2010_v-ray",
position={
"x": -2,
"y": 0,
"z": -3
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b04_harley_davidson_3dsmax2014",
position={
"x": -4.2,
"y": 0,
"z": 2
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
self.add_object(model_name="audia7_2018_vray",
position={
"x": -6.5,
"y": 0,
"z": -3
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.7, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"cars_bikes",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def run(self):
self.start()
init_setup_commands = [{"$type": "set_screen_size",
"width": 600,
"height": 480},
{"$type": "set_render_quality",
"render_quality": 5}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(10, 10))
# Disable physics.
self.communicate({"$type": "set_gravity",
"value": False})
# Add the avatar.
self.communicate(TDWUtils.create_avatar(position={"x": -4, "y": 1.7, "z": 0},
look_at=TDWUtils.array_to_vector3([0, 1.0, 0]),
avatar_id="avatar"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("grass_countryside")
print(record)
self.communicate({"$type": "add_material", "name": "grass_countryside", "url": record.get_url()})
self.communicate({"$type": "set_proc_gen_floor_material", "name": "grass_countryside"})
self.communicate({"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"})
self.add_object(model_name="129601_sheep",
position={"x": 0.2, "y": 0, "z": 2.0},
library="models_full.json")
self.add_object(model_name="b03_cow",
position={"x": 2, "y": 0, "z": 0},
rotation={"x": 0, "y": 30, "z": 0},
library="models_full.json")
self.add_object(model_name="mouflon",
position={"x": -1, "y": 0, "z": -0.7},
rotation={"x": 0, "y": 30, "z": 0},
library="models_full.json")
self.add_object(model_name="b05_sheep",
position={"x": 0.3, "y": 0, "z": -1.5},
rotation={"x": 0, "y": 85, "z": 0},
library="models_full.json")
# Enable image capture
self.communicate({"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]})
self.communicate({"$type": "send_images",
"frequency": "always"})
scene_data = self.communicate({"$type": "look_at_position",
"avatar_id": "avatar",
"position": TDWUtils.array_to_vector3([0, 1.0, 0])})
images = Images(scene_data[0])
TDWUtils.save_images(images, "sheep_cow", output_directory="/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior")
def run(self, c: Controller):
"""
Run the trial and save the output.
:param c: The controller.
"""
print(f"Images will be saved to: {self.output_dir}")
# Initialize the scene.
resp = c.communicate(self.init_commands)
# Get a map of the segmentation colors.
segm = SegmentationColors(resp[0])
for i in range(segm.get_num()):
for obj in self.moving_objects:
if obj.object_id == segm.get_object_id(i):
obj.possibility.segmentation_color = segm.get_object_color(
i)
# Request scene data and images per frame.
frame_data: List[dict] = []
resp = c.communicate([{
"$type": "send_images",
"frequency": "always"
}, {
"$type": "send_transforms",
"ids": self.m_ids,
"frequency": "always"
}, {
"$type": "send_rigidbodies",
"ids": self.m_ids,
"frequency": "always"
}])
# Run the trial.
for frame in range(self.num_frames):
colors: Dict[int, Tuple[int, int, int]] = {}
transforms: Dict[int, Tuple[float, float, float]] = {}
transform_data = None
rigidbody_data = None
# Parse the output data.
for r in resp[:-1]:
r_id = OutputData.get_data_type_id(r)
# Record all Transforms data.
if r_id == "tran":
transform_data = Transforms(r)
for i in range(transform_data.get_num()):
transforms.update({
transform_data.get_id(i):
transform_data.get_position(i)
})
# Record all Rigidbodies data.
elif r_id == "rigi":
rigidbody_data = Rigidbodies(r)
# Save the images.
elif r_id == "imag":
images = Images(r)
for p in range(images.get_num_passes()):
if images.get_pass_mask(p) == "_id":
image_colors = TDWUtils.get_pil_image(
images, p).getcolors()
for ic in image_colors:
color = ic[1]
for obj in self.moving_objects:
if obj.possibility.segmentation_color == color:
colors.update({obj.object_id: color})
TDWUtils.save_images(Images(r),
TDWUtils.zero_padding(frame),
output_directory=self.output_dir)
# Append frame data.
frame_data.append(
Trial._get_frame_state(transform_data, rigidbody_data, frame))
# Build the frame state.
state = State(colors, transforms, frame)
# Apply object actions.
commands = []
for o in self.occluders:
commands.extend(o.get_frame_commands(state))
for mo in self.moving_objects:
commands.extend(mo.get_frame_commands(state))
if len(commands) == 0:
commands = [{"$type": "do_nothing"}]
# Send the commands and update the state.
resp = c.communicate(commands)
# Cleanup.
c.communicate([{
"$type": "destroy_all_objects"
}, {
"$type": "unload_asset_bundles"
}, {
"$type": "send_images",
"frequency": "never"
}, {
"$type": "send_transforms",
"ids": self.m_ids,
"frequency": "never"
}, {
"$type": "send_rigidbodies",
"ids": self.m_ids,
"frequency": "never"
}])
print("\tGenerated images.")
# Output the scene metadata.
Path(self.output_dir).joinpath("state.json").write_text(
json.dumps({"frames": frame_data}), encoding="utf-8")
print("\tWrote state file.")
# Get _id passes with randomized colors.
self._randomize_segmentation_colors()
print("\tCreated random segmentation colors.")
# Organize the images.
self._organize_output()
print("\tOrganized files")
def dining_table(self):
"""
1. Create a dining table in the center of the room.
2. Create four chairs, one on each side of the table.
3. Create "table settings" in front of each chair on the table.
4. Create a centerpiece on the table.
"""
# Create a dining table in the center of the room.
dining_table_id = self.add_object("willisau_varion_w3_table")
scene_data = self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
# Get the table bounds.
dining_table_bounds = self.get_bounds_data(dining_table_id)
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.8, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"apple_1",
output_directory="../replicated_images/segmentation")
top = dining_table_bounds.get_top(0)
# Add apple
apple = self.add_object(model_name="apple07(8_vray)",
position={
"x": top[0],
"y": top[1],
"z": top[2]
},
library="models_full.json")
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.8, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"apple_2",
output_directory="../replicated_images/segmentation")
# Add vase 1
position = [top[0] - 0.125, 0, top[2] + 0.134]
position[1] = top[1]
vase_1 = self.add_object(model_name="vase_03",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.8, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"apple_3",
output_directory="../replicated_images/segmentation")
# Add vase 2
position = [top[0] + 0.245, 0, top[2] + 0.234]
position[1] = top[1]
vase_2 = self.add_object(model_name="vase_06",
position=TDWUtils.array_to_vector3(position),
library="models_full.json")
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.8, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"apple_4",
output_directory="../replicated_images/segmentation")
# IMG required when all objects added in
scene_data = self.communicate({
"$type": "set_pass_masks",
"pass_masks": ["_img"],
"avatar_id": "avatar"
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"apple",
output_directory="../replicated_images/segmentation")
""" Destroying select objects for generating segmentations """
""" ---------------------------- """
def trial(self, output_directory, nimages):
"""
Collide a chair with a fridge.
:param m_c: mass of the chair.
:param s_c: The static friction of the chair.
:param b_c: The bounciness of the chair.
"""
# Destroy all objects currently in the scene.
self.communicate({"$type": "destroy_all_objects"})
# Set screen size
self.communicate({
"$type": "set_screen_size",
"height": 480,
"width": 640
})
# Create the avatar.
self.communicate({
"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"
})
# Teleport the avatar to the valid position.
# Enable image capture.
self.communicate([
{
"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {
"x": 4,
"y": 2.5,
"z": 5
}
},
{
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img", "_id"]
},
{
"$type": "send_images",
"frequency": "always"
},
])
# Create the fridge and the chair.
# b04_kenmore_refr_70419, b03_ka90ivi20r_2013__vray, b05_amati_red_violin
static_id = self.add_object("b05_amati_red_violin",
position={
"x": -3,
"y": 0,
"z": 1
})
throw_id = self.add_object("b03_dumbbell_vray",
position={
"x": 1,
"y": 1.5,
"z": -1
})
# Set the mass of the fridge and the chair.
self.communicate([{
"$type": "set_mass",
"id": static_id,
"mass": 40
}, {
"$type": "set_mass",
"id": throw_id,
"mass": 20
}])
# Apply a force to the chair.
self.communicate({
"$type": "apply_force_to_object",
"force": {
"x": -250,
"y": -10,
"z": 0
},
"id": throw_id
})
# Create the output directory.
if os.path.exists(output_directory):
shutil.rmtree(output_directory)
sleep(0.5)
os.mkdir(output_directory)
# Capture n images.
for i in range(nimages):
# Look at the object.
resp = self.communicate({
"$type": "look_at_position",
"avatar_id": "a",
"position": {
"x": -1.5,
"y": 1,
"z": 0
}
})
images = Images(resp[0])
frame = resp[-1]
# Save the image.
TDWUtils.save_images(images,
frame,
output_directory=output_directory)
def trial(self, output_directory, nimages):
"""
Collide a chair with a fridge.
:param m_c: mass of the chair.
:param s_c: The static friction of the chair.
:param b_c: The bounciness of the chair.
"""
# Destroy all objects currently in the scene.
self.communicate({"$type": "destroy_all_objects"})
# Set screen size
self.communicate({
"$type": "set_screen_size",
"height": 480,
"width": 640
})
# Create the avatar.
self.communicate({
"$type": "create_avatar",
"type": "A_Img_Caps_Kinematic",
"id": "a"
})
# Teleport the avatar to the valid position.
# Enable image capture.
self.communicate([
{
"$type": "teleport_avatar_to",
"avatar_id": "a",
"position": {
"x": -1.5,
"y": 2.5,
"z": 7
}
},
{
"$type": "set_pass_masks",
"avatar_id": "a",
"pass_masks": ["_img", "_id"]
},
{
"$type": "send_images",
"frequency": "always"
},
])
# place the elevated platform
platform_id = self.add_object("fridge_box",
position={
"x": 1,
"y": 0,
"z": 0
})
self.communicate({
"$type": "set_kinematic_state",
"id": platform_id,
"is_kinematic": True,
"use_gravity": False
})
# place a table
table_id = self.add_object("quatre_dining_table",
position={
"x": -4,
"y": 0,
"z": -1
})
self.communicate({
"$type": "rotate_object_by",
"angle": -45,
"id": table_id
})
# place an object on the ground (b05_microsoft_surface_rt,big_glass_bowl_water,b05_seamodel5_(2015))
collider_id = self.add_object("b05_seamodel5_(2015)",
position={
"x": -1,
"y": 0,
"z": 0
})
self.communicate({
"$type": "scale_object",
"id": collider_id,
"scale_factor": {
"x": 1,
"y": 1,
"z": 1
}
})
# Create the falling object. (piano_max2017, villaverde_-_murano_luna_chandelier)
object_id = self.add_object("piano_max2017",
position={
"x": -1,
"y": 2.5,
"z": 0
})
# Set the scale, mass, and friction of the object.
self.communicate({
"$type": "scale_object",
"id": object_id,
"scale_factor": {
"x": 1,
"y": 1,
"z": 1
}
})
# self.communicate({"$type":"set_color",
# "color":{"r":0, "g":0, "b":1, "a":1},
# "id": object_id})
# Create the output directory.
if os.path.exists(output_directory):
shutil.rmtree(output_directory)
sleep(0.5)
os.mkdir(output_directory)
# Capture n images.
for i in range(nimages):
# Look at the object.
resp = self.communicate({
"$type": "look_at_position",
"avatar_id": "a",
"position": {
"x": -1.5,
"y": 1,
"z": 0
}
})
images = Images(resp[0])
frame = resp[-1]
# Save the image.
TDWUtils.save_images(images,
frame,
output_directory=output_directory)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty scene (no walls)
self.communicate({
"$type": "create_empty_environment",
"center": {
"x": 0,
"y": 0,
"z": 0
},
"bounds": {
"x": 30,
"y": 30,
"z": 30
}
})
self.communicate(self.get_add_hdri_skybox("pink_sunrise_4k"))
cube = self.communicate({
"$type": "load_primitive_from_resources",
"primitive_type": "Cube",
"id": 1,
"position": {
"x": 0,
"y": 0,
"z": 0
},
"orientation": {
"x": 0,
"y": 0,
"z": 0
}
})
self.communicate({
"$type": "scale_object",
"scale_factor": {
"x": 30,
"y": 0.0001,
"z": 30
},
"id": 1
})
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("concrete_chipped_cracked")
self.communicate({
"$type": "add_material",
"name": "concrete_chipped_cracked",
"url": record.get_url()
})
# self.communicate({"$type": "set_visual_material",
# "material_index": 1,
# "material_name": "concrete_chipped_cracked",
# "object_name": "cube",
# "id": 1})
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -13,
"y": 1.5,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[5, 1.5, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
self.add_object(model_name="spitfire_2012",
position={
"x": 0,
"y": 0,
"z": 2
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b03_topkicktruck",
position={
"x": -7,
"y": 0,
"z": -5
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
self.add_object(model_name="3dscan_man_012",
position={
"x": -4,
"y": 0,
"z": 3
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([5, 1.5, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"airplane",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 1280,
"height": 962
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -2,
"y": 1.5,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[4, 0.8, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
self.add_object(model_name="b04_wood_metal_park_bench",
position={
"x": 3,
"y": 0,
"z": -1.5
},
library="models_full.json")
self.add_object(model_name="b04_wood_metal_park_bench",
position={
"x": 3,
"y": 0,
"z": 1.5
},
library="models_full.json")
self.add_object(model_name="b05_table_new",
position={
"x": 2,
"y": 0,
"z": -1.5
},
rotation=TDWUtils.array_to_vector3([0, 90, 0]),
library="models_full.json")
self.add_object(model_name="b05_table_new",
position={
"x": 2,
"y": 0,
"z": 1.5
},
rotation=TDWUtils.array_to_vector3([0, 90, 0]),
library="models_full.json")
self.add_object(model_name="chair_eames_plastic_armchair",
position={
"x": 1,
"y": 0,
"z": -1.5
},
rotation=TDWUtils.array_to_vector3([0, 90, 0]),
library="models_full.json")
self.add_object(model_name="chair_eames_plastic_armchair",
position={
"x": 1,
"y": 0,
"z": 1.5
},
rotation=TDWUtils.array_to_vector3([0, 90, 0]),
library="models_full.json")
# self.communicate({"$type": "rotate_object_by",
# "angle": 180,
# "axis": "yaw",
# "id": chair_id,
# "is_world": True})
# record = ModelLibrarian(library='models_full.json').get_record("b05_tv1970")
# self.communicate({"$type": "add_object",
# "name": "b05_tv1970",
# "url": record.get_url(),
# "scale_factor": 30,
# "position": {"x": 0, "y": 0, "z": 3.5},
# "category": record.wcategory,
# "id": self.get_unique_id()})
#
#
# side_table = self.add_object(model_name="side_table_wood",
# position={"x": 1.5, "y": 0, "z": 4},
# library="models_full.json")
# side_table_bounds = self.get_bounds_data(side_table)
#
# top = side_table_bounds.get_top(0)
# self.add_object(model_name="acacia_table_lamp_jamie_young",
# position={"x": 1.5, "y": top[1], "z": 4},
# library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([4, 0.8, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"two_tables",
output_directory="replicated_images")
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -0.7,
"y": 0.45,
"z": -0.7
},
look_at=TDWUtils.array_to_vector3(
[-0.1, 0.45, -0.1]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table_id = self.add_object(model_name="coffee_table_glass_round",
position={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
book_1 = self.add_object(model_name="cgaxis_models_23_06_vray",
position={
"x": 0.15,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 30,
"z": 0
},
library="models_full.json")
book_1_bounds = self.get_bounds_data(book_1)
book_1_top = book_1_bounds.get_top(0)
self.add_object(model_name="cgaxis_models_23_15_vray",
position={
"x": 0.26,
"y": book_1_top[1],
"z": 0.31
},
library="models_full.json")
self.add_object(model_name="b04_whiskeybottle",
position={
"x": -0.2,
"y": top[1],
"z": -0.32
},
library="models_full.json")
self.add_object(
model_name="b04_cup_nijniy_novgorod_gamma1.0_2013__vray",
position={
"x": -0.17,
"y": top[1],
"z": 0.15
},
rotation={
"x": 0,
"y": 24,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([-0.1, 0.45, -0.1])
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"books_whiskey",
output_directory="replicated_images")
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate({
"$type": "create_empty_environment",
"center": {
"x": 0,
"y": 0,
"z": 0
},
"bounds": {
"x": 30,
"y": 30,
"z": 30
}
})
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -8,
"y": 1.7,
"z": 0
},
look_at=TDWUtils.array_to_vector3([0, 1,
0]),
avatar_id="avatar"))
self.communicate(self.get_add_hdri_skybox("table_mountain_1_4k"))
self.communicate({"$type": "rotate_hdri_skybox_by", "angle": 90})
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
cube_record = ModelLibrarian("models_special.json").get_record(
"prim_cube")
self.communicate({
"$type": "add_object",
"name": "prim_cube",
"url": cube_record.get_url(),
"scale_factor": cube_record.scale_factor,
"position": {
"x": 0,
"y": 0,
"z": 0
},
"rotation": {
"x": 0,
"y": 0,
"z": 0
},
"category": cube_record.wcategory,
"id": 1
})
self.communicate({
"$type": "scale_object",
"scale_factor": {
"x": 30,
"y": 0.0001,
"z": 30
},
"id": 1
})
grass_record = MaterialLibrarian("materials_high.json").get_record(
"grass_countryside")
self.communicate({
"$type": "add_material",
"name": "grass_countryside",
"url": grass_record.get_url()
})
self.communicate(
TDWUtils.set_visual_material(c=self,
substructure=cube_record.substructure,
object_id=1,
material="grass_countryside"))
self.add_object(model_name="b03_giraffe",
position={
"x": 1.2,
"y": 0.0001,
"z": 2.4
},
rotation={
"x": 0,
"y": 193,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b03_zebra",
position={
"x": -3.3,
"y": 0.0001,
"z": -1.6
},
rotation={
"x": 0,
"y": 13,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b03_zebra",
position={
"x": 0,
"y": 0.001,
"z": 0.7
},
rotation={
"x": 0,
"y": -190,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 1, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"giraffe_zebra",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(16, 16))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -7.5,
"y": 1.5,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[0, 0.7, 0]),
avatar_id="avatar"))
self.communicate(self.get_add_hdri_skybox("industrial_sunset_4k"))
lib = MaterialLibrarian(library="materials_med.json")
record = lib.get_record("concrete_asphalt_rolled")
self.communicate({
"$type": "add_material",
"name": "concrete_asphalt_rolled",
"url": record.get_url()
})
self.communicate({
"$type": "set_proc_gen_floor_material",
"name": "concrete_asphalt_rolled"
})
# self.communicate({"$type": "set_proc_gen_walls_material", "name": "concrete"})
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
self.add_object(model_name="b05_model_sell",
position={
"x": 2.5,
"y": 0,
"z": 4
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
self.add_object(model_name="b03_fire_hydrant",
position={
"x": 2,
"y": 0,
"z": -1
},
rotation={
"x": 0,
"y": 0,
"z": 0
},
library="models_full.json")
# record = ModelLibrarian(library='models_full.json').get_record("bigben_clock")
# self.communicate({"$type": "add_object",
# "name": "bigben_clock",
# "url": record.get_url(),
# "scale_factor": record.scale_factor * 10,
# "position": {"x": 1.7, "y": 0, "z": -3},
# "rotation": TDWUtils.VECTOR3_ZERO,
# "category": record.wcategory,
# "id": self.get_unique_id()})
self.add_object(model_name="bench",
position={
"x": 1.7,
"y": 0,
"z": -3
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.7, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"hydrant_clock",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 1280,
"height": 962
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.load_streamed_scene(scene="tdw_room_2018")
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 1.7,
"z": 0
},
look_at=TDWUtils.array_to_vector3([4, 0,
0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
self.add_object(model_name="linbrazil_diz_armchair",
position={
"x": 3.5,
"y": 0,
"z": 1.5
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
table_id = self.add_object(model_name="03_106",
position={
"x": 2.5,
"y": 0,
"z": 0
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table_id)
top = table_bounds.get_top(0)
self.add_object(model_name="ficus",
position={
"x": 3.5,
"y": 0,
"z": -1.5
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
self.add_object(model_name="vm_v2_015",
position={
"x": 2.5,
"y": top[1],
"z": 0.5
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([4, 0, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(images,
"fan_room",
output_directory="replicated_images")
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 640,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
self.communicate(TDWUtils.create_empty_room(8, 8))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": 0,
"y": 1.4,
"z": 3
},
look_at=TDWUtils.array_to_vector3(
[0, 0.5, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
table_record = ModelLibrarian(library="models_full.json").get_record(
"restoration_hardware_salvaged_tables")
table = self.add_object(
model_name="restoration_hardware_salvaged_tables",
position={
"x": 0,
"y": 0,
"z": 0
},
rotation={
"x": 0,
"y": -90,
"z": 0
},
library="models_full.json")
table_bounds = self.get_bounds_data(table)
left = table_bounds.get_left(0)
self.add_object(model_name="3dscan_man_002",
position={
"x": 1.6,
"y": 0,
"z": left[2]
},
rotation={
"x": 0,
"y": 124,
"z": 0
},
library="models_full.json")
top = table_bounds.get_top(0)
cup_record = ModelLibrarian(
library="models_full.json").get_record("cup")
self.communicate({
"$type": "add_object",
"name": "cup",
"url": cup_record.get_url(),
"scale_factor": cup_record.scale_factor,
"position": {
"x": 0,
"y": top[1],
"z": -0.07
},
"rotation": {
"x": 0,
"y": -90,
"z": 0
},
"category": cup_record.wcategory,
"id": 1
})
# record = MaterialLibrarian(library='materials_high.json').get_record("polystyrene_foam")
self.communicate({
"$type":
"add_material",
"name":
"polystyrene_foam",
"url":
"http://s3.amazonaws.com/tdw_test_dev/materials/osx/high/polystyrene_foam"
})
for sub_object in cup_record.substructure:
for j in range(len(sub_object)):
self.communicate({
"$type": "set_visual_material",
"material_index": j,
"material_name": "polystyrene_foam",
"object_name": sub_object["name"],
"id": 1
})
self.communicate({
"$type": "add_object",
"name": "cup",
"url": cup_record.get_url(),
"scale_factor": cup_record.scale_factor,
"position": {
"x": -0.3,
"y": top[1],
"z": 0.1
},
"rotation": {
"x": 0,
"y": -90,
"z": 0
},
"category": cup_record.wcategory,
"id": 2
})
for sub_object in cup_record.substructure:
for j in range(len(sub_object)):
self.communicate({
"$type": "set_visual_material",
"material_index": j,
"material_name": "polystyrene_foam",
"object_name": sub_object["name"],
"id": 2
})
self.add_object(model_name='neopolitan_pizza',
position={
"x": 0.7,
"y": top[1],
"z": 0
},
rotation={
"x": 0,
"y": -22,
"z": 0
},
library="models_full.json")
self.add_object(model_name='stua_onda_stool_max2012',
position={
"x": -1.4,
"y": 0,
"z": 0
},
rotation={
"x": 0,
"y": 90,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 0.5, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"pizza2",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/interior"
)
def run(self):
self.start()
init_setup_commands = [{
"$type": "set_screen_size",
"width": 600,
"height": 480
}, {
"$type": "set_render_quality",
"render_quality": 5
}]
self.communicate(init_setup_commands)
# Create an empty room.
self.communicate(TDWUtils.create_empty_room(10, 10))
# Disable physics.
self.communicate({"$type": "set_gravity", "value": False})
# Add the avatar.
self.communicate(
TDWUtils.create_avatar(position={
"x": -4,
"y": 1.7,
"z": 0
},
look_at=TDWUtils.array_to_vector3(
[0, 1.0, 0]),
avatar_id="avatar"))
self.communicate({
"$type": "set_field_of_view",
"field_of_view": 68.0,
"avatar_id": "avatar"
})
self.add_object(model_name="b05_zebra",
position={
"x": 0.2,
"y": 0,
"z": 2.0
},
library="models_full.json")
self.add_object(model_name="735737_zebra",
position={
"x": 2,
"y": 0,
"z": 0
},
rotation={
"x": 0,
"y": 30,
"z": 0
},
library="models_full.json")
# Enable image capture
self.communicate({
"$type": "set_pass_masks",
"avatar_id": "avatar",
"pass_masks": ["_img", "_id"]
})
self.communicate({"$type": "send_images", "frequency": "always"})
scene_data = self.communicate({
"$type":
"look_at_position",
"avatar_id":
"avatar",
"position":
TDWUtils.array_to_vector3([0, 1.0, 0])
})
images = Images(scene_data[0])
TDWUtils.save_images(
images,
"zebras",
output_directory=
"/Users/leonard/Desktop/TDWBase-1.5.0/Python/Leonard/compare_COCO_TDW/replicated_images/exterior"
)
def __init__(self, resp: List[bytes], objects: Dict[int, StaticObjectInfo],
avatar: Avatar):
"""
:param resp: The response from the build.
:param objects: Static object info per object. Key = the ID of the object in the scene.
:param avatar: The avatar in the scene.
"""
self._frame_count = Controller.get_frame(resp[-1])
self.audio: List[Tuple[Base64Sound, int]] = list()
collisions, env_collisions, rigidbodies = FrameData._P.get_collisions(
resp=resp)
# Record avatar collisions.
if avatar is not None:
self.avatar_object_collisions = avatar.collisions
self.avatar_env_collisions = avatar.env_collisions
self.held_objects = {
Arm.left: avatar.frame.get_held_left(),
Arm.right: avatar.frame.get_held_right()
}
else:
self.avatar_object_collisions = None
self.avatar_env_collisions = None
self.held_objects = None
# Get the object transform data.
self.object_transforms: Dict[int, Transform] = dict()
tr = get_data(resp=resp, d_type=Transforms)
for i in range(tr.get_num()):
o_id = tr.get_id(i)
self.object_transforms[o_id] = Transform(
position=np.array(tr.get_position(i)),
rotation=np.array(tr.get_rotation(i)),
forward=np.array(tr.get_forward(i)))
# Get camera matrix data.
matrices = get_data(resp=resp, d_type=CameraMatrices)
self.projection_matrix = matrices.get_projection_matrix()
self.camera_matrix = matrices.get_camera_matrix()
# Get the transform data of the avatar.
self.avatar_transform = Transform(
position=np.array(avatar.frame.get_position()),
rotation=np.array(avatar.frame.get_rotation()),
forward=np.array(avatar.frame.get_forward()))
self.avatar_body_part_transforms: Dict[int, Transform] = dict()
for i in range(avatar.frame.get_num_body_parts()):
self.avatar_body_part_transforms[avatar.frame.get_body_part_id(
i)] = Transform(
position=np.array(avatar.frame.get_body_part_position(i)),
rotation=np.array(avatar.frame.get_body_part_rotation(i)),
forward=np.array(avatar.frame.get_body_part_forward(i)))
# Get the audio of each collision.
for coll in collisions:
if not FrameData._P.is_valid_collision(coll):
continue
collider_id = coll.get_collider_id()
collidee_id = coll.get_collidee_id()
collider_info: Optional[ObjectInfo] = None
collidee_info: Optional[ObjectInfo] = None
if collider_id in objects:
collider_info = objects[collider_id].audio
# Check if the object is a body part.
else:
if collider_id in avatar.body_parts_static:
collider_info = avatar.body_parts_static[collider_id].audio
if collidee_id in objects:
collidee_info = objects[collidee_id].audio
# Check if the object is a body part.
else:
if collidee_id in avatar.body_parts_static:
collidee_info = avatar.body_parts_static[collidee_id].audio
# If either object isn't a cached object, don't try to add audio.
if collider_info is None or collidee_info is None:
continue
if collider_info.mass < collidee_info.mass:
target_id = collider_id
target_amp = collider_info.amp
target_mat = collider_info.material.name
other_id = collidee_id
other_amp = collidee_info.amp
other_mat = collider_info.material.name
else:
target_id = collidee_id
target_amp = collidee_info.amp
target_mat = collidee_info.material.name
other_id = collider_id
other_amp = collider_info.amp
other_mat = collider_info.material.name
rel_amp = other_amp / target_amp
audio = FrameData._P.get_sound(coll, rigidbodies, other_id,
other_mat, target_id, target_mat,
rel_amp)
self.audio.append((audio, target_id))
# Get the audio of each environment collision.
for coll in env_collisions:
collider_id = coll.get_object_id()
if collider_id not in objects:
continue
v = FrameData._get_velocity(rigidbodies, collider_id)
if (v is not None) and (v > 0):
collider_info = objects[collider_id].audio
audio = FrameData._P.get_sound(
coll, rigidbodies, 1, FrameData._SURFACE_MATERIAL.name,
collider_id, collider_info.material.name, 0.01)
self.audio.append((audio, collider_id))
# Get the image data.
self.id_pass: Optional[np.array] = None
self.depth_pass: Optional[np.array] = None
self.image_pass: Optional[np.array] = None
for i in range(0, len(resp) - 1):
if OutputData.get_data_type_id(resp[i]) == "imag":
images = Images(resp[i])
for j in range(images.get_num_passes()):
if images.get_pass_mask(j) == "_id":
self.id_pass = images.get_image(j)
elif images.get_pass_mask(j) == "_depth_simple":
self.depth_pass = images.get_image(j)
elif images.get_pass_mask(j) == "_img":
self.image_pass = images.get_image(j)