def get_cuboid_image_space(obj_id, camera_name='camera'):
"""
reproject the 3d points into the image space for a given object.
It assumes you already added the cuboid to the object
:obj_id: string for the name of the object of interest
:camera_name: string representing the camera name in visii
:return: cubdoid + centroid projected to the image, values [0..1]
"""
cam_matrix = visii.entity.get(
camera_name).get_transform().get_world_to_local_matrix()
cam_proj_matrix = visii.entity.get(
camera_name).get_camera().get_projection()
points = []
points_cam = []
for i_t in range(9):
trans = visii.transform.get(f"{obj_id}_cuboid_{i_t}")
mat_trans = trans.get_local_to_world_matrix()
pos_m = visii.vec4(mat_trans[3][0], mat_trans[3][1], mat_trans[3][2],
1)
p_cam = cam_matrix * pos_m
p_image = cam_proj_matrix * (cam_matrix * pos_m)
p_image = visii.vec2(p_image) / p_image.w
p_image = p_image * visii.vec2(1, -1)
p_image = (p_image + visii.vec2(1, 1)) * 0.5
points.append([p_image[0], p_image[1]])
points_cam.append([p_cam[0], p_cam[1], p_cam[2]])
return points, points_cam
def interact():
global speed_camera
global cursor
global rot
# visii camera matrix
cam_matrix = camera.get_transform().get_local_to_world_matrix()
dt = visii.vec4(0,0,0,0)
# translation
if visii.is_button_held("W"): dt[2] = -speed_camera
if visii.is_button_held("S"): dt[2] = speed_camera
if visii.is_button_held("A"): dt[0] = -speed_camera
if visii.is_button_held("D"): dt[0] = speed_camera
if visii.is_button_held("Q"): dt[1] = -speed_camera
if visii.is_button_held("E"): dt[1] = speed_camera
# control the camera
if visii.length(dt) > 0.0:
w_dt = cam_matrix * dt
camera.get_transform().add_position(visii.vec3(w_dt))
# camera rotation
cursor[2] = cursor[0]
cursor[3] = cursor[1]
cursor[0] = visii.get_cursor_pos().x
cursor[1] = visii.get_cursor_pos().y
if visii.is_button_held("MOUSE_LEFT"):
visii.set_cursor_mode("DISABLED")
rotate_camera = True
else:
visii.set_cursor_mode("NORMAL")
rotate_camera = False
if rotate_camera:
rot.x -= (cursor[0] - cursor[2]) * 0.001
rot.y -= (cursor[1] - cursor[3]) * 0.001
init_rot = visii.angleAxis(visii.pi() * .5, (1,0,0))
yaw = visii.angleAxis(rot.x, (0,1,0))
pitch = visii.angleAxis(rot.y, (1,0,0))
camera.get_transform().set_rotation(init_rot * yaw * pitch)
# change speed movement
if visii.is_button_pressed("UP"):
speed_camera *= 0.5
print('decrease speed camera', speed_camera)
if visii.is_button_pressed("DOWN"):
speed_camera /= 0.5
print('increase speed camera', speed_camera)
def get_add(obj1, obj2, pos1, pos2, quat1, quat2):
obj1.get_transform().set_position(visii.vec3(pos1[0],pos1[1],pos1[2]))
obj1.get_transform().set_rotation(visii.quat(quat1[0],quat1[1],quat1[2],quat1[3]))
obj2.get_transform().set_position(visii.vec3(pos2[0],pos2[1],pos2[2]))
obj2.get_transform().set_rotation(visii.quat(quat2[0],quat2[1],quat2[2],quat2[3]))
dist = []
vertices = obj1.get_mesh().get_vertices()
for i in range(len(vertices)):
v = visii.vec4(vertices[i][0],vertices[i][1],vertices[i][2],1)
p0 = obj1.get_transform().get_local_to_world_matrix() * v
p1 = obj2.get_transform().get_local_to_world_matrix() * v
dist.append(visii.distance(p0, p1))
dist = np.mean(dist)
return dist
def export_to_ndds_file(
filename="tmp.json", #this has to include path as well
obj_names=[], # this is a list of ids to load and export
height=500,
width=500,
camera_name='camera',
camera_struct=None,
visibility_percentage=False,
):
"""
Method that exports the meta data like NDDS. This includes all the scene information in one
scene.
:filename: string for the json file you want to export, you have to include the extension
:obj_names: [string] each entry is a visii entity that has the cuboids attached to, these
are the objects that are going to be exported.
:height: int height of the image size
:width: int width of the image size
:camera_name: string for the camera name visii entity
:camera_struct: dictionary of the camera look at information. Expecting the following
entries: 'at','eye','up'. All three has to be floating arrays of three entries.
This is an optional export.
:visibility_percentage: bool if you want to export the visibility percentage of the object.
Careful this can be costly on a scene with a lot of objects.
:return nothing:
"""
import simplejson as json
# assume we only use the view camera
cam_matrix = visii.entity.get(
camera_name).get_transform().get_world_to_local_matrix()
cam_matrix_export = []
for row in cam_matrix:
cam_matrix_export.append([row[0], row[1], row[2], row[3]])
cam_world_location = visii.entity.get(
camera_name).get_transform().get_position()
cam_world_quaternion = visii.entity.get(
camera_name).get_transform().get_rotation()
# cam_world_quaternion = visii.quat_cast(cam_matrix)
cam_intrinsics = visii.entity.get(
camera_name).get_camera().get_intrinsic_matrix(width, height)
if camera_struct is None:
camera_struct = {
'at': [
0,
0,
0,
],
'eye': [
0,
0,
0,
],
'up': [
0,
0,
0,
]
}
dict_out = {
"camera_data": {
"width":
width,
'height':
height,
'camera_look_at': {
'at': [
camera_struct['at'][0],
camera_struct['at'][1],
camera_struct['at'][2],
],
'eye': [
camera_struct['eye'][0],
camera_struct['eye'][1],
camera_struct['eye'][2],
],
'up': [
camera_struct['up'][0],
camera_struct['up'][1],
camera_struct['up'][2],
]
},
'camera_view_matrix':
cam_matrix_export,
'location_world': [
cam_world_location[0],
cam_world_location[1],
cam_world_location[2],
],
'quaternion_world_xyzw': [
cam_world_quaternion[0],
cam_world_quaternion[1],
cam_world_quaternion[2],
cam_world_quaternion[3],
],
'intrinsics': {
'fx': cam_intrinsics[0][0],
'fy': cam_intrinsics[1][1],
'cx': cam_intrinsics[2][0],
'cy': cam_intrinsics[2][1]
}
},
"objects": []
}
# Segmentation id to export
id_keys_map = visii.entity.get_name_to_id_map()
for obj_name in obj_names:
projected_keypoints, _ = get_cuboid_image_space(
obj_name, camera_name=camera_name)
# put them in the image space.
for i_p, p in enumerate(projected_keypoints):
projected_keypoints[i_p] = [p[0] * width, p[1] * height]
# Get the location and rotation of the object in the camera frame
trans = visii.transform.get(obj_name)
quaternion_xyzw = visii.inverse(
cam_world_quaternion) * trans.get_rotation()
object_world = visii.vec4(trans.get_position()[0],
trans.get_position()[1],
trans.get_position()[2], 1)
pos_camera_frame = cam_matrix * object_world
#check if the object is visible
visibility = -1
bounding_box = [-1, -1, -1, -1]
segmentation_mask = visii.render_data(
width=int(width),
height=int(height),
start_frame=0,
frame_count=1,
bounce=int(0),
options="entity_id",
)
segmentation_mask = np.array(segmentation_mask).reshape(
width, height, 4)[:, :, 0]
if visibility_percentage == True and int(
id_keys_map[obj_name]) in np.unique(
segmentation_mask.astype(int)):
transforms_to_keep = {}
for name in id_keys_map.keys():
if 'camera' in name.lower() or obj_name in name:
continue
trans_to_keep = visii.entity.get(name).get_transform()
transforms_to_keep[name] = trans_to_keep
visii.entity.get(name).clear_transform()
# Percentage visibility through full segmentation mask.
segmentation_unique_mask = visii.render_data(
width=int(width),
height=int(height),
start_frame=0,
frame_count=1,
bounce=int(0),
options="entity_id",
)
segmentation_unique_mask = np.array(
segmentation_unique_mask).reshape(width, height, 4)[:, :, 0]
values_segmentation = np.where(
segmentation_mask == int(id_keys_map[obj_name]))[0]
values_segmentation_full = np.where(
segmentation_unique_mask == int(id_keys_map[obj_name]))[0]
visibility = len(values_segmentation) / float(
len(values_segmentation_full))
# set back the objects from remove
for entity_name in transforms_to_keep.keys():
visii.entity.get(entity_name).set_transform(
transforms_to_keep[entity_name])
else:
if int(id_keys_map[obj_name]) in np.unique(
segmentation_mask.astype(int)):
#
visibility = 1
y, x = np.where(
segmentation_mask == int(id_keys_map[obj_name]))
bounding_box = [
int(min(x)),
int(max(x)), height - int(max(y)), height - int(min(y))
]
else:
visibility = 0
# Final export
dict_out['objects'].append({
'class':
obj_name.split('_')[0],
'name':
obj_name,
'provenance':
'visii',
# TODO check the location
'location':
[pos_camera_frame[0], pos_camera_frame[1], pos_camera_frame[2]],
'quaternion_xyzw': [
quaternion_xyzw[0],
quaternion_xyzw[1],
quaternion_xyzw[2],
quaternion_xyzw[3],
],
'quaternion_xyzw_world': [
trans.get_rotation()[0],
trans.get_rotation()[1],
trans.get_rotation()[2],
trans.get_rotation()[3]
],
'projected_cuboid':
projected_keypoints[0:8],
'projected_cuboid_centroid':
projected_keypoints[8],
'segmentation_id':
id_keys_map[obj_name],
'visibility_image':
visibility,
'bounding_box': {
'top_left': [
bounding_box[0],
bounding_box[2],
],
'bottom_right': [
bounding_box[1],
bounding_box[3],
],
},
})
with open(filename, 'w+') as fp:
json.dump(dict_out, fp, indent=4, sort_keys=True)
# elif 'light' in s.get_name().lower():
# print(s.get_name())
# s.set_light(visii.light.create('light' + str(i_s)))
# s.get_light().set_intensity(20)
# s.get_light().set_temperature(5000)
# if 'tire' in s.get_name().lower():
# s.get_transform().set_angular_velocity(visii.angleAxis(3.14 * .05, visii.vec3(1,0,0)))
# visii.entity.get("sdbcarShell_1").get_material().set_base_color(visii.vec3(1,0,0))
# visii.render_to_png(1024, 1024, 1000, "motion_blur_3")
if opt.control:
camera.get_transform().clear_motion()
cursor = visii.vec4()
speed_camera = 4.0
rot = visii.vec2(visii.pi() * 1.25, 0)
def interact():
global speed_camera
global cursor
global rot
# visii camera matrix
cam_matrix = camera.get_transform().get_local_to_world_matrix()
dt = visii.vec4(0,0,0,0)
# translation
if visii.is_button_held("W"): dt[2] = -speed_camera
if visii.is_button_held("S"): dt[2] = speed_camera
if visii.is_button_held("A"): dt[0] = -speed_camera