def get_cuboid_image_space(obj_id, camera_name='camera'): """ reproject the 3d points into the image space for a given object. It assumes you already added the cuboid to the object :obj_id: string for the name of the object of interest :camera_name: string representing the camera name in visii :return: cubdoid + centroid projected to the image, values [0..1] """ cam_matrix = visii.entity.get( camera_name).get_transform().get_world_to_local_matrix() cam_proj_matrix = visii.entity.get( camera_name).get_camera().get_projection() points = [] points_cam = [] for i_t in range(9): trans = visii.transform.get(f"{obj_id}_cuboid_{i_t}") mat_trans = trans.get_local_to_world_matrix() pos_m = visii.vec4(mat_trans[3][0], mat_trans[3][1], mat_trans[3][2], 1) p_cam = cam_matrix * pos_m p_image = cam_proj_matrix * (cam_matrix * pos_m) p_image = visii.vec2(p_image) / p_image.w p_image = p_image * visii.vec2(1, -1) p_image = (p_image + visii.vec2(1, 1)) * 0.5 points.append([p_image[0], p_image[1]]) points_cam.append([p_cam[0], p_cam[1], p_cam[2]]) return points, points_cam
def interact(): global speed_camera global cursor global rot # visii camera matrix cam_matrix = camera.get_transform().get_local_to_world_matrix() dt = visii.vec4(0,0,0,0) # translation if visii.is_button_held("W"): dt[2] = -speed_camera if visii.is_button_held("S"): dt[2] = speed_camera if visii.is_button_held("A"): dt[0] = -speed_camera if visii.is_button_held("D"): dt[0] = speed_camera if visii.is_button_held("Q"): dt[1] = -speed_camera if visii.is_button_held("E"): dt[1] = speed_camera # control the camera if visii.length(dt) > 0.0: w_dt = cam_matrix * dt camera.get_transform().add_position(visii.vec3(w_dt)) # camera rotation cursor[2] = cursor[0] cursor[3] = cursor[1] cursor[0] = visii.get_cursor_pos().x cursor[1] = visii.get_cursor_pos().y if visii.is_button_held("MOUSE_LEFT"): visii.set_cursor_mode("DISABLED") rotate_camera = True else: visii.set_cursor_mode("NORMAL") rotate_camera = False if rotate_camera: rot.x -= (cursor[0] - cursor[2]) * 0.001 rot.y -= (cursor[1] - cursor[3]) * 0.001 init_rot = visii.angleAxis(visii.pi() * .5, (1,0,0)) yaw = visii.angleAxis(rot.x, (0,1,0)) pitch = visii.angleAxis(rot.y, (1,0,0)) camera.get_transform().set_rotation(init_rot * yaw * pitch) # change speed movement if visii.is_button_pressed("UP"): speed_camera *= 0.5 print('decrease speed camera', speed_camera) if visii.is_button_pressed("DOWN"): speed_camera /= 0.5 print('increase speed camera', speed_camera)
def get_add(obj1, obj2, pos1, pos2, quat1, quat2): obj1.get_transform().set_position(visii.vec3(pos1[0],pos1[1],pos1[2])) obj1.get_transform().set_rotation(visii.quat(quat1[0],quat1[1],quat1[2],quat1[3])) obj2.get_transform().set_position(visii.vec3(pos2[0],pos2[1],pos2[2])) obj2.get_transform().set_rotation(visii.quat(quat2[0],quat2[1],quat2[2],quat2[3])) dist = [] vertices = obj1.get_mesh().get_vertices() for i in range(len(vertices)): v = visii.vec4(vertices[i][0],vertices[i][1],vertices[i][2],1) p0 = obj1.get_transform().get_local_to_world_matrix() * v p1 = obj2.get_transform().get_local_to_world_matrix() * v dist.append(visii.distance(p0, p1)) dist = np.mean(dist) return dist
def export_to_ndds_file( filename="tmp.json", #this has to include path as well obj_names=[], # this is a list of ids to load and export height=500, width=500, camera_name='camera', camera_struct=None, visibility_percentage=False, ): """ Method that exports the meta data like NDDS. This includes all the scene information in one scene. :filename: string for the json file you want to export, you have to include the extension :obj_names: [string] each entry is a visii entity that has the cuboids attached to, these are the objects that are going to be exported. :height: int height of the image size :width: int width of the image size :camera_name: string for the camera name visii entity :camera_struct: dictionary of the camera look at information. Expecting the following entries: 'at','eye','up'. All three has to be floating arrays of three entries. This is an optional export. :visibility_percentage: bool if you want to export the visibility percentage of the object. Careful this can be costly on a scene with a lot of objects. :return nothing: """ import simplejson as json # assume we only use the view camera cam_matrix = visii.entity.get( camera_name).get_transform().get_world_to_local_matrix() cam_matrix_export = [] for row in cam_matrix: cam_matrix_export.append([row[0], row[1], row[2], row[3]]) cam_world_location = visii.entity.get( camera_name).get_transform().get_position() cam_world_quaternion = visii.entity.get( camera_name).get_transform().get_rotation() # cam_world_quaternion = visii.quat_cast(cam_matrix) cam_intrinsics = visii.entity.get( camera_name).get_camera().get_intrinsic_matrix(width, height) if camera_struct is None: camera_struct = { 'at': [ 0, 0, 0, ], 'eye': [ 0, 0, 0, ], 'up': [ 0, 0, 0, ] } dict_out = { "camera_data": { "width": width, 'height': height, 'camera_look_at': { 'at': [ camera_struct['at'][0], camera_struct['at'][1], camera_struct['at'][2], ], 'eye': [ camera_struct['eye'][0], camera_struct['eye'][1], camera_struct['eye'][2], ], 'up': [ camera_struct['up'][0], camera_struct['up'][1], camera_struct['up'][2], ] }, 'camera_view_matrix': cam_matrix_export, 'location_world': [ cam_world_location[0], cam_world_location[1], cam_world_location[2], ], 'quaternion_world_xyzw': [ cam_world_quaternion[0], cam_world_quaternion[1], cam_world_quaternion[2], cam_world_quaternion[3], ], 'intrinsics': { 'fx': cam_intrinsics[0][0], 'fy': cam_intrinsics[1][1], 'cx': cam_intrinsics[2][0], 'cy': cam_intrinsics[2][1] } }, "objects": [] } # Segmentation id to export id_keys_map = visii.entity.get_name_to_id_map() for obj_name in obj_names: projected_keypoints, _ = get_cuboid_image_space( obj_name, camera_name=camera_name) # put them in the image space. for i_p, p in enumerate(projected_keypoints): projected_keypoints[i_p] = [p[0] * width, p[1] * height] # Get the location and rotation of the object in the camera frame trans = visii.transform.get(obj_name) quaternion_xyzw = visii.inverse( cam_world_quaternion) * trans.get_rotation() object_world = visii.vec4(trans.get_position()[0], trans.get_position()[1], trans.get_position()[2], 1) pos_camera_frame = cam_matrix * object_world #check if the object is visible visibility = -1 bounding_box = [-1, -1, -1, -1] segmentation_mask = visii.render_data( width=int(width), height=int(height), start_frame=0, frame_count=1, bounce=int(0), options="entity_id", ) segmentation_mask = np.array(segmentation_mask).reshape( width, height, 4)[:, :, 0] if visibility_percentage == True and int( id_keys_map[obj_name]) in np.unique( segmentation_mask.astype(int)): transforms_to_keep = {} for name in id_keys_map.keys(): if 'camera' in name.lower() or obj_name in name: continue trans_to_keep = visii.entity.get(name).get_transform() transforms_to_keep[name] = trans_to_keep visii.entity.get(name).clear_transform() # Percentage visibility through full segmentation mask. segmentation_unique_mask = visii.render_data( width=int(width), height=int(height), start_frame=0, frame_count=1, bounce=int(0), options="entity_id", ) segmentation_unique_mask = np.array( segmentation_unique_mask).reshape(width, height, 4)[:, :, 0] values_segmentation = np.where( segmentation_mask == int(id_keys_map[obj_name]))[0] values_segmentation_full = np.where( segmentation_unique_mask == int(id_keys_map[obj_name]))[0] visibility = len(values_segmentation) / float( len(values_segmentation_full)) # set back the objects from remove for entity_name in transforms_to_keep.keys(): visii.entity.get(entity_name).set_transform( transforms_to_keep[entity_name]) else: if int(id_keys_map[obj_name]) in np.unique( segmentation_mask.astype(int)): # visibility = 1 y, x = np.where( segmentation_mask == int(id_keys_map[obj_name])) bounding_box = [ int(min(x)), int(max(x)), height - int(max(y)), height - int(min(y)) ] else: visibility = 0 # Final export dict_out['objects'].append({ 'class': obj_name.split('_')[0], 'name': obj_name, 'provenance': 'visii', # TODO check the location 'location': [pos_camera_frame[0], pos_camera_frame[1], pos_camera_frame[2]], 'quaternion_xyzw': [ quaternion_xyzw[0], quaternion_xyzw[1], quaternion_xyzw[2], quaternion_xyzw[3], ], 'quaternion_xyzw_world': [ trans.get_rotation()[0], trans.get_rotation()[1], trans.get_rotation()[2], trans.get_rotation()[3] ], 'projected_cuboid': projected_keypoints[0:8], 'projected_cuboid_centroid': projected_keypoints[8], 'segmentation_id': id_keys_map[obj_name], 'visibility_image': visibility, 'bounding_box': { 'top_left': [ bounding_box[0], bounding_box[2], ], 'bottom_right': [ bounding_box[1], bounding_box[3], ], }, }) with open(filename, 'w+') as fp: json.dump(dict_out, fp, indent=4, sort_keys=True)
# elif 'light' in s.get_name().lower(): # print(s.get_name()) # s.set_light(visii.light.create('light' + str(i_s))) # s.get_light().set_intensity(20) # s.get_light().set_temperature(5000) # if 'tire' in s.get_name().lower(): # s.get_transform().set_angular_velocity(visii.angleAxis(3.14 * .05, visii.vec3(1,0,0))) # visii.entity.get("sdbcarShell_1").get_material().set_base_color(visii.vec3(1,0,0)) # visii.render_to_png(1024, 1024, 1000, "motion_blur_3") if opt.control: camera.get_transform().clear_motion() cursor = visii.vec4() speed_camera = 4.0 rot = visii.vec2(visii.pi() * 1.25, 0) def interact(): global speed_camera global cursor global rot # visii camera matrix cam_matrix = camera.get_transform().get_local_to_world_matrix() dt = visii.vec4(0,0,0,0) # translation if visii.is_button_held("W"): dt[2] = -speed_camera if visii.is_button_held("S"): dt[2] = speed_camera if visii.is_button_held("A"): dt[0] = -speed_camera