def do_trial(self):
# Initialize PyImpact and pass in the "master gain" amplitude value. This value must be betweem 0 and 1.
# The relative amplitudes of all scene objects involved in collisions will be scaled relative to this value.
# Note -- if this value is too high, waveform clipping can occur and the resultant audio will be distorted.
# For this reason, the value used here is considerably smaller than the corresponding value used in the
# impact_sounds.py example controller. Here we have a large number of closely-occuring collisions resulting in
# a rapid series of "clustered" impact sounds, as opposed to a single object falling from a height;
# using a higher value such as the 0.5 used in the example controller will definitely result in unpleasant
# distortion of the audio.
# Note that logging is also enabled.
# Keep track of trial number.
self.current_trial_num += 1
# Create folder for this trial's logging info.
dest_dir = self.root_dest_dir.joinpath(str(self.current_trial_num))
if not dest_dir.exists():
dest_dir.mkdir(parents=True)
dest_dir_str = str(dest_dir.resolve())
p = PyImpact(0.25, logging=True)
self.add_all_objects()
# "Aim" the ball at the monkey and apply the force.
# Note that this force value was arrived at through a number of trial-and-error iterations.
resp = self.communicate([{"$type": "object_look_at_position",
"id": 0,
"position": {"x": -12.95, "y": 1.591, "z": -5.1}},
{"$type": "apply_force_magnitude_to_object",
"id": 0,
"magnitude": 98.0}])
for i in range(400):
collisions, environment_collision, rigidbodies = PyImpact.get_collisions(resp)
# Sort the objects by mass.
masses: Dict[int, float] = {}
for j in range(rigidbodies.get_num()):
masses.update({rigidbodies.get_id(j): rigidbodies.get_mass(j)})
# If there was a collision, create an impact sound.
if len(collisions) > 0 and PyImpact.is_valid_collision(collisions[0]):
collider_id = collisions[0].get_collider_id()
collidee_id = collisions[0].get_collidee_id()
# The "target" object should have less mass than the "other" object.
if masses[collider_id] < masses[collidee_id]:
target_id = collider_id
other_id = collidee_id
else:
target_id = collidee_id
other_id = collider_id
target_name = self.obj_name_dict[target_id]
other_name = self.obj_name_dict[other_id]
impact_sound_command = p.get_impact_sound_command(
collision=collisions[0],
rigidbodies=rigidbodies,
target_id=target_id,
target_mat=self.object_audio_data[target_name].material,
other_id=other_id,
other_mat=self.object_audio_data[other_name].material,
target_amp=self.object_audio_data[target_name].amp,
other_amp=self.object_audio_data[other_name].amp,
resonance=self.object_audio_data[target_name].resonance)
resp = self.communicate(impact_sound_command)
# Continue to run the trial.
else:
resp = self.communicate([])
# Get the logging info for this trial and write it out.
log = p.get_log()
json_dest = dest_dir.joinpath("mode_properties_log.json")
json_dest.write_text(json.dumps(log, indent=2))
for obj_setup in self.object_setups:
self.communicate({"$type": "destroy_object", "id": obj_setup.id})
def trial(self):
"""
Select random objects and collide them to produce impact sounds.
"""
p = PyImpact(initial_amp=0.25)
# Set the environment audio materials.
floor = AudioMaterial.ceramic
wall = AudioMaterial.wood
# Create an empty room.
# Set the screen size.
# Adjust framerate so that physics is closer to realtime.
commands = [
TDWUtils.create_empty_room(12, 12), {
"$type": "destroy_all_objects"
}, {
"$type": "set_screen_size",
"width": 1024,
"height": 1024
}, {
"$type": "set_target_framerate",
"framerate": 100
}
]
# Create the avatar.
commands.extend(
TDWUtils.create_avatar(avatar_type="A_Img_Caps_Kinematic",
position={
"x": 1,
"y": 1.2,
"z": 1.2
},
look_at=TDWUtils.VECTOR3_ZERO))
# Add the audio sensor.
commands.extend([{
"$type": "add_audio_sensor",
"avatar_id": "a"
}, {
"$type": "set_focus_distance",
"focus_distance": 2
}])
# Select a random pair of objects.
obj1_names = [
"trapezoidal_table", "glass_table_round", "yellow_side_chair",
"table_square", "marble_table"
]
obj2_names = ["vase_06", "spoon1", "glass3", "jug02"]
obj1_name = random.choice(obj1_names)
obj2_name = random.choice(obj2_names)
# Get initialization data from the default audio data (which includes mass, friction values, etc.).
obj1_init_data = AudioInitData(name=obj1_name)
obj2_init_data = AudioInitData(name=obj2_name,
position={
"x": 0,
"y": 2,
"z": 0
},
rotation={
"x": 135,
"y": 0,
"z": 30
})
# Convert the initialization data to commands.
obj1_id, obj1_commands = obj1_init_data.get_commands()
obj2_id, obj2_commands = obj2_init_data.get_commands()
# Cache the IDs and names of each object for PyImpact.
object_names = {obj1_id: obj1_name, obj2_id: obj2_name}
p.set_default_audio_info(object_names=object_names)
# Add the objects.
commands.extend(obj1_commands)
commands.extend(obj2_commands)
# Apply a small force to the dropped object.
# Request collision and rigidbody output data.
commands.extend([{
"$type": "apply_force_to_object",
"force": {
"x": 0,
"y": -0.01,
"z": 0
},
"id": obj2_id
}, {
"$type": "send_collisions",
"enter": True,
"stay": False,
"exit": False,
"collision_types": ["obj", "env"]
}, {
"$type": "send_rigidbodies",
"frequency": "always",
"ids": [obj2_id, obj1_id]
}])
# Send all of the commands.
resp = self.communicate(commands)
# Iterate through 200 frames.
# Every frame, listen for collisions, and parse the output data.
for i in range(200):
# Use PyImpact to generate audio from the output data and then convert the audio to TDW commands.
# If no audio is generated, then `commands` is an empty list.
commands = p.get_audio_commands(resp=resp, floor=floor, wall=wall)
# Send the commands to TDW in order to play the audio.
resp = self.communicate(commands)
def __init__(self,
output_dir: Path = Path("D:/audio_dataset"),
total: int = 28602,
port: int = 1071):
"""
:param output_dir: The output directory for the files.
:param port: The socket port.
:param total: The total number of files per sub-set.
"""
self.total = total
self.output_dir = output_dir
if not self.output_dir.exists():
self.output_dir.mkdir(parents=True)
db_path = self.output_dir.joinpath('results.db')
self.conn = sqlite3.connect(str(db_path.resolve()))
self.db_c = self.conn.cursor()
# Sound20K table.
if self.db_c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='sound20k'").\
fetchone() is None:
self.db_c.execute(
"CREATE TABLE sound20k (path text, scene integer, cam_x real, cam_y real, cam_z real,"
"obj_x real, obj_y real, obj_z real, mass real, static_friction real, dynamic_friction "
"real, yaw real, pitch real, roll real, force real)")
# Scenes table.
if self.db_c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='scenes'").\
fetchone() is None:
self.db_c.execute(
"CREATE TABLE scenes (id integer, commands text)")
self.py_impact = PyImpact()
self.object_info = PyImpact.get_object_info()
sound20k_object_info = PyImpact.get_object_info(
Path("models/object_info.csv"))
for obj_info in sound20k_object_info:
if obj_info in self.object_info:
continue
else:
self.object_info.update(
{obj_info: sound20k_object_info[obj_info]})
self.libs: Dict[str, ModelLibrarian] = {}
# Load all model libraries into memory.
for lib_name in ModelLibrarian.get_library_filenames():
self.libs.update({lib_name: ModelLibrarian(lib_name)})
# Add the custom model library.
self.libs.update({
"models/models.json":
ModelLibrarian(str(Path("models/models.json").resolve()))
})
super().__init__(port=port)
# Global settings.
self.communicate([{
"$type": "set_screen_size",
"width": 256,
"height": 256
}, {
"$type": "set_time_step",
"time_step": 0.02
}, {
"$type": "set_target_framerate",
"framerate": 60
}, {
"$type": "set_physics_solver_iterations",
"iterations": 20
}])
def trial(self):
"""
Select random objects and collide them to produce impact sounds.
"""
p = PyImpact(initial_amp=0.5)
# Destroy all objects currently in the scene.
# Set the screen size.
# Adjust physics timestep for more real-time physics behavior.
commands = [{
"$type": "destroy_all_objects"
}, {
"$type": "set_screen_size",
"width": 1024,
"height": 1024
}, {
"$type": "set_time_step",
"time_step": 0.02
}]
# Create the avatar.
commands.extend(
TDWUtils.create_avatar(avatar_type="A_Img_Caps_Kinematic",
position={
"x": 1,
"y": 1.2,
"z": 1.2
},
look_at=TDWUtils.VECTOR3_ZERO))
# Add the audio sensor.
# Set the target framerate.
# Make sure that post-processing is enabled and render quality is set to max.
commands.extend([{
"$type": "add_audio_sensor",
"avatar_id": "a"
}, {
"$type": "set_target_framerate",
"framerate": 60
}, {
"$type": "set_post_process",
"value": True
}, {
"$type": "set_focus_distance",
"focus_distance": 2
}, {
"$type": "set_render_quality",
"render_quality": 5
}])
# Select a random pair of objects.
objects = PyImpact.get_object_info()
obj1_names = [
"trapezoidal_table", "glass_table_round", "yellow_side_chair",
"table_square", "marble_table"
]
obj2_names = ["vase_06", "spoon1", "glass3", "jug02"]
obj1_name = random.choice(obj1_names)
obj2_name = random.choice(obj2_names)
obj1_id = 0
obj2_id = 1
# Add the objects.
# Set their masses from the audio info data.
# Set a physics material for the second object.
# Apply a force to the second object.
# Listen for collisions, and object properties.
commands.extend([
self.get_add_object(model_name=obj1_name,
object_id=obj1_id,
library=objects[obj1_name].library), {
"$type": "set_mass",
"id": obj1_id,
"mass": objects[obj1_name].mass
},
self.get_add_object(model_name=obj2_name,
object_id=obj2_id,
library=objects[obj2_name].library,
rotation={
"x": 135,
"y": 0,
"z": 30
},
position={
"x": 0,
"y": 2,
"z": 0
}), {
"$type": "set_mass",
"id": obj2_id,
"mass": objects[obj2_name].mass
}, {
"$type": "set_physic_material",
"id": obj2_id,
"bounciness":
objects[obj2_name].bounciness,
"dynamic_friction": 0.8
}, {
"$type": "apply_force_to_object",
"force": {
"x": 0,
"y": -0.01,
"z": 0
},
"id": obj2_id
}, {
"$type": "send_collisions",
"enter": True,
"stay": False,
"exit": False
}, {
"$type": "send_rigidbodies",
"frequency": "always",
"ids": [obj2_id, obj1_id]
}
])
# Send all of the commands.
resp = self.communicate(commands)
# Iterate through 200 frames.
# Every frame, listen for collisions, and parse the output data.
for i in range(200):
collisions, environment_collision, rigidbodies = PyImpact.get_collisions(
resp)
# If there was a collision, create an impact sound.
if len(collisions) > 0 and PyImpact.is_valid_collision(
collisions[0]):
impact_sound_command = p.get_impact_sound_command(
collision=collisions[0],
rigidbodies=rigidbodies,
target_id=obj2_id,
target_mat=objects[obj2_name].material.name,
target_amp=objects[obj2_name].amp,
other_id=obj1_id,
other_amp=objects[obj1_name].amp,
other_mat=objects[obj1_name].material.name,
resonance=objects[obj1_name].resonance)
resp = self.communicate(impact_sound_command)
# Continue to run the trial.
else:
resp = self.communicate([])
# Stop listening for collisions and rigidbodies.
self.communicate([{
"$type": "send_collisions",
"frequency": "never"
}, {
"$type": "send_rigidbodies",
"frequency": "never"
}])
class FrameData:
"""
Per-frame data that an avatar can use to decide what action to do next.
Access this data from the [StickyMittenAvatarController](sma_controller.md):
```python
from sticky_mitten_avatar import StickyMittenAvatarController, Arm
c = StickyMittenAvatarController()
c.init_scene()
# Look towards the left arm.
c.rotate_camera_by(pitch=70, yaw=-45)
c.reach_for_target(target={"x": -0.2, "y": 0.21, "z": 0.385}, arm=Arm.left)
# Save each image from the start of the most recent API action to the end.
for frame in c.frames:
frame.save_images(output_directory="dist")
c.end()
```
***
## Fields
### Visual
- `image_pass` Rendered image of the scene as a numpy array.
- `id_pass` Image pass of object color segmentation as a numpy array.
- `depth_pass` Image pass of depth values per pixel as a numpy array.
- `projection_matrix` The [camera projection matrix](https://github.com/threedworld-mit/tdw/blob/master/Documentation/api/output_data.md#cameramatrices) of the avatar's camera as a numpy array.
- `camera_matrix` The [camera matrix](https://github.com/threedworld-mit/tdw/blob/master/Documentation/api/output_data.md#cameramatrices) of the avatar's camera as a numpy array.
### Audio
- `audio` A list of tuples of audio generated by impacts. The first element in the tuple is a [`Base64Sound` object](https://github.com/threedworld-mit/tdw/blob/master/Documentation/python/py_impact.md#base64sound).
The second element is the ID of the "target" (smaller) object.
```python
for frame in c.frames:
for audio in frame.audio:
# Get the audio data.
wav_data = audio[0].bytes
# Get the position of the object that generated the audio data.
object_id = audio[1]
position = frame.object_transforms[object_id].position
```
### Objects
- `object_transforms` The dictionary of object [transform data](transform.md). Key = the object ID.
```python
# Print the position of each object per frame.
for frame in c.frames:
for object_id in frame.object_transforms:
print(frame.object_transforms[object_id].position)
```
### Avatar
- `avatar_transform` The [transform data](transform.md) of the avatar.
```python
for frame in c.frames:
avatar_position = frame.avatar_transform.position
```
- `avatar_body_part_transforms` The [transform data](transform.md) of each body part of the avatar. Key = body part ID.
```python
for frame in c.frames:
# Get the position and segmentation color of each body part.
for body_part_id in frame.avatar_body_part_transforms:
position = frame.avatar_body_part_transforms[body_part_id]
segmentation_color = c.static_avatar_data[body_part_id]
```
- `avatar_object_collisions` A dictionary of objects the avatar collided with. Key = body part ID. Value = A list of object IDs.
```python
for frame in c.frames:
for body_part_id in frame.avatar_object_collisions:
body_part = c.static_avatar_info[body_part_id]
object_ids = frame.avatar_object_collisions[body_part_id]
for object_id in object_ids:
print(body_part.name + " collided with object " + str(object_id))
```
- `avatar_env_collisions` A list of body part IDs that collided with the environment (such as a wall).
```python
for frame in c.frames:
for body_part_id in frame.avatar_env_collisions:
body_part = c.static_avatar_info[body_part_id]
print(body_part.name + " collided with the environment.")
```
- `held_objects` A dictionary of IDs of objects held in each mitten. Key = arm:
```python
from sticky_mitten_avatar import StickyMittenAvatarController, Arm
c = StickyMittenAvatarController()
# Your code here.
# Prints all objects held by the left mitten at the last frame.
print(c.frames[-1].held_objects[Arm.left])
```
***
## Functions
"""
_P = PyImpact(initial_amp=0.01)
_SURFACE_MATERIAL: AudioMaterial = AudioMaterial.hardwood
def __init__(self, resp: List[bytes], objects: Dict[int, StaticObjectInfo],
avatar: Avatar):
"""
:param resp: The response from the build.
:param objects: Static object info per object. Key = the ID of the object in the scene.
:param avatar: The avatar in the scene.
"""
self._frame_count = Controller.get_frame(resp[-1])
self.audio: List[Tuple[Base64Sound, int]] = list()
collisions, env_collisions, rigidbodies = FrameData._P.get_collisions(
resp=resp)
# Record avatar collisions.
if avatar is not None:
self.avatar_object_collisions = avatar.collisions
self.avatar_env_collisions = avatar.env_collisions
self.held_objects = {
Arm.left: avatar.frame.get_held_left(),
Arm.right: avatar.frame.get_held_right()
}
else:
self.avatar_object_collisions = None
self.avatar_env_collisions = None
self.held_objects = None
# Get the object transform data.
self.object_transforms: Dict[int, Transform] = dict()
tr = get_data(resp=resp, d_type=Transforms)
for i in range(tr.get_num()):
o_id = tr.get_id(i)
self.object_transforms[o_id] = Transform(
position=np.array(tr.get_position(i)),
rotation=np.array(tr.get_rotation(i)),
forward=np.array(tr.get_forward(i)))
# Get camera matrix data.
matrices = get_data(resp=resp, d_type=CameraMatrices)
self.projection_matrix = matrices.get_projection_matrix()
self.camera_matrix = matrices.get_camera_matrix()
# Get the transform data of the avatar.
self.avatar_transform = Transform(
position=np.array(avatar.frame.get_position()),
rotation=np.array(avatar.frame.get_rotation()),
forward=np.array(avatar.frame.get_forward()))
self.avatar_body_part_transforms: Dict[int, Transform] = dict()
for i in range(avatar.frame.get_num_body_parts()):
self.avatar_body_part_transforms[avatar.frame.get_body_part_id(
i)] = Transform(
position=np.array(avatar.frame.get_body_part_position(i)),
rotation=np.array(avatar.frame.get_body_part_rotation(i)),
forward=np.array(avatar.frame.get_body_part_forward(i)))
# Get the audio of each collision.
for coll in collisions:
if not FrameData._P.is_valid_collision(coll):
continue
collider_id = coll.get_collider_id()
collidee_id = coll.get_collidee_id()
collider_info: Optional[ObjectInfo] = None
collidee_info: Optional[ObjectInfo] = None
if collider_id in objects:
collider_info = objects[collider_id].audio
# Check if the object is a body part.
else:
if collider_id in avatar.body_parts_static:
collider_info = avatar.body_parts_static[collider_id].audio
if collidee_id in objects:
collidee_info = objects[collidee_id].audio
# Check if the object is a body part.
else:
if collidee_id in avatar.body_parts_static:
collidee_info = avatar.body_parts_static[collidee_id].audio
# If either object isn't a cached object, don't try to add audio.
if collider_info is None or collidee_info is None:
continue
if collider_info.mass < collidee_info.mass:
target_id = collider_id
target_amp = collider_info.amp
target_mat = collider_info.material.name
other_id = collidee_id
other_amp = collidee_info.amp
other_mat = collider_info.material.name
else:
target_id = collidee_id
target_amp = collidee_info.amp
target_mat = collidee_info.material.name
other_id = collider_id
other_amp = collider_info.amp
other_mat = collider_info.material.name
rel_amp = other_amp / target_amp
audio = FrameData._P.get_sound(coll, rigidbodies, other_id,
other_mat, target_id, target_mat,
rel_amp)
self.audio.append((audio, target_id))
# Get the audio of each environment collision.
for coll in env_collisions:
collider_id = coll.get_object_id()
if collider_id not in objects:
continue
v = FrameData._get_velocity(rigidbodies, collider_id)
if (v is not None) and (v > 0):
collider_info = objects[collider_id].audio
audio = FrameData._P.get_sound(
coll, rigidbodies, 1, FrameData._SURFACE_MATERIAL.name,
collider_id, collider_info.material.name, 0.01)
self.audio.append((audio, collider_id))
# Get the image data.
self.id_pass: Optional[np.array] = None
self.depth_pass: Optional[np.array] = None
self.image_pass: Optional[np.array] = None
for i in range(0, len(resp) - 1):
if OutputData.get_data_type_id(resp[i]) == "imag":
images = Images(resp[i])
for j in range(images.get_num_passes()):
if images.get_pass_mask(j) == "_id":
self.id_pass = images.get_image(j)
elif images.get_pass_mask(j) == "_depth_simple":
self.depth_pass = images.get_image(j)
elif images.get_pass_mask(j) == "_img":
self.image_pass = images.get_image(j)
@staticmethod
def set_surface_material(surface_material: AudioMaterial) -> None:
"""
Set the surface material of the scene.
:param surface_material: The floor's [audio material](https://github.com/threedworld-mit/tdw/blob/master/Documentation/python/py_impact.md#audiomaterialenum).
"""
FrameData._SURFACE_MATERIAL = surface_material
def save_images(self, output_directory: Union[str, Path]) -> None:
"""
Save the ID pass (segmentation colors) and the depth pass to disk.
Images will be named: `[frame_number]_[pass_name].[extension]`
For example, the depth pass on the first frame will be named: `00000000_depth.png`
The image pass is a jpg file and the other passes are png files.
:param output_directory: The directory that the images will be saved to.
"""
if isinstance(output_directory, str):
output_directory = Path(output_directory)
if not output_directory.exists():
output_directory.mkdir(parents=True)
prefix = TDWUtils.zero_padding(self._frame_count, 8)
# Save each image.
for image, pass_name, ext in zip(
[self.image_pass, self.id_pass, self.depth_pass],
["img", "id", "depth"], ["jpg", "png", "png"]):
p = output_directory.joinpath(f"{prefix}_{pass_name}.{ext}")
with p.open("wb") as f:
f.write(image)
def get_pil_images(self) -> dict:
"""
Convert each image pass to PIL images.
:return: A dictionary of PIL images. Key = the name of the pass (img, id, depth)
"""
print(type(Image.open(BytesIO(self.image_pass))))
return {
"img": Image.open(BytesIO(self.image_pass)),
"id": Image.open(BytesIO(self.id_pass)),
"depth": Image.open(BytesIO(self.depth_pass))
}
@staticmethod
def _get_velocity(rigidbodies: Rigidbodies, o_id: int) -> float:
"""
:param rigidbodies: The rigidbody data.
:param o_id: The ID of the object.
:return: The velocity magnitude of the object.
"""
for i in range(rigidbodies.get_num()):
if rigidbodies.get_id(i) == o_id:
return np.linalg.norm(rigidbodies.get_velocity(i))