def render_segmentation_data(self, img_file):
segmentation_array = nvisii.render_data(width=int(self.width),
height=int(self.height),
start_frame=0,
frame_count=1,
bounce=int(0),
options="entity_id",
seed=1)
segmentation_array = np.array(segmentation_array).reshape(
self.height, self.width, 4)[:, :, 0]
segmentation_array[segmentation_array > 3.4028234663852886e+37] = 0
segmentation_array[segmentation_array < 3.4028234663852886e-37] = 0
segmentation_array = np.flipud(segmentation_array)
rgb_data = self.segmentation_to_rgb(
segmentation_array.astype(dtype=np.uint8))
from PIL import Image
rgb_img = Image.fromarray(rgb_data)
rgb_img.save(img_file)
save_image(t0_array, f"{opt.outf}/t0.png")
nvisii.sample_time_interval((1.0, 1.0)) # only sample at t = 1
t1_array = nvisii.render(width=opt.width,
height=opt.height,
samples_per_pixel=8,
seed=1)
t1_array = np.array(t1_array).reshape(opt.height, opt.width, 4)
save_image(t1_array, f"{opt.outf}/t1.png")
# Next, let's obtain segmentation data for both
# these timesteps to do the reprojection
nvisii.sample_time_interval((0.0, 0.0))
t0_base_colors_array = nvisii.render_data(width=opt.width,
height=opt.height,
start_frame=0,
frame_count=1,
bounce=0,
options="base_color")
t0_base_colors_array = np.array(t0_base_colors_array).reshape(
opt.height, opt.width, 4)
save_image(t0_base_colors_array, f"{opt.outf}/t0_base_color.png")
nvisii.sample_time_interval((1.0, 1.0))
t1_base_colors_array = nvisii.render_data(width=opt.width,
height=opt.height,
start_frame=0,
frame_count=1,
bounce=int(0),
options="base_color")
t1_base_colors_array = np.array(t1_base_colors_array).reshape(
opt.height, opt.width, 4)
y_sample_interval=(.5, .5))
visii.sample_time_interval((1, 1))
visii.render_data_to_file(
width=opt.width,
height=opt.height,
start_frame=0,
frame_count=1,
bounce=int(0),
options="entity_id",
file_path=f"{opt.outf}/{str(i_render).zfill(5)}.seg.exr")
segmentation_array = visii.render_data(
width=int(opt.width),
height=int(opt.height),
start_frame=0,
frame_count=1,
bounce=int(0),
options="entity_id",
)
export_to_ndds_file(
f"{opt.outf}/{str(i_render).zfill(5)}.json",
obj_names=names_to_export,
width=opt.width,
height=opt.height,
camera_name='camera',
# cuboids = cuboids,
camera_struct=random_camera_movement,
segmentation_mask=np.array(segmentation_array).reshape(
opt.width, opt.height, 4)[:, :, 0],
visibility_percentage=False,
)
def render_data_to_file(self, img_file):
if self.vision_modalities == "depth" and self.img_cntr != 1:
depth_data = nvisii.render_data(
width=self.width,
height=self.height,
start_frame=0,
frame_count=1,
bounce=int(0),
options=self.vision_modalities,
)
depth_data = np.array(depth_data).reshape(self.height, self.width,
4)
depth_data = np.flipud(depth_data)[:, :, [0, 1, 2]]
# normalize depths
depth_data[:, :, 0] = (depth_data[:, :, 0] - np.min(
depth_data[:, :, 0])) / (np.max(depth_data[:, :, 0]) -
np.min(depth_data[:, :, 0]))
depth_data[:, :, 1] = (depth_data[:, :, 1] - np.min(
depth_data[:, :, 1])) / (np.max(depth_data[:, :, 1]) -
np.min(depth_data[:, :, 1]))
depth_data[:, :, 2] = (depth_data[:, :, 2] - np.min(
depth_data[:, :, 2])) / (np.max(depth_data[:, :, 2]) -
np.min(depth_data[:, :, 2]))
from PIL import Image
depth_image = Image.fromarray(
((1 - depth_data) * 255).astype(np.uint8))
depth_image.save(img_file)
elif self.vision_modalities == "normal" and self.img_cntr != 1:
normal_data = nvisii.render_data(
width=self.width,
height=self.height,
start_frame=0,
frame_count=1,
bounce=int(0),
options="screen_space_normal",
)
normal_data = np.array(normal_data).reshape(
self.height, self.width, 4)
normal_data = np.flipud(normal_data)[:, :, [0, 1, 2]]
normal_data[:, :, 0] = (normal_data[:, :, 0] + 1) / 2 * 255 # R
normal_data[:, :, 1] = (normal_data[:, :, 1] + 1) / 2 * 255 # G
normal_data[:, :,
2] = 255 - ((normal_data[:, :, 2] + 1) / 2 * 255) # B
from PIL import Image
normal_image = Image.fromarray((normal_data).astype(np.uint8))
normal_image.save(img_file)
else:
nvisii.render_data_to_file(width=self.width,
height=self.height,
start_frame=0,
frame_count=1,
bounce=int(0),
options=self.vision_modalities,
file_path=img_file)
nvisii.set_dome_light_intensity(0)
# # # # # # # # # # # # # # # # # # # # # # # # #
# nvisii offers different ways to export meta data
# these are exported as raw arrays of numbers
# for many segmentations, it might be beneficial to only
# sample pixel centers instead of the whole pixel area.
# to do so, call this function
nvisii.sample_pixel_area(x_sample_interval=(.5, .5),
y_sample_interval=(.5, .5))
depth_array = nvisii.render_data(width=int(opt.width),
height=int(opt.height),
start_frame=0,
frame_count=1,
bounce=int(0),
options="depth")
depth_array = np.array(depth_array).reshape(opt.height, opt.width, 4)
depth_array = np.flipud(depth_array)
# save the segmentation image
def convert_from_uvd(u, v, d, fx, fy, cx, cy):
# d *= self.pxToMetre
x_over_z = (cx - u) / fx
y_over_z = (cy - v) / fy
z = d / np.sqrt(1. + x_over_z**2 + y_over_z**2)
x = x_over_z * z
y = y_over_z * z
return x, y, z
def export_to_ndds_file(
filename="tmp.json", #this has to include path as well
obj_names=[], # this is a list of ids to load and export
height=500,
width=500,
camera_name='camera',
camera_struct=None,
visibility_percentage=False,
):
"""
Method that exports the meta data like NDDS. This includes all the scene information in one
scene.
:filename: string for the json file you want to export, you have to include the extension
:obj_names: [string] each entry is a nvisii entity that has the cuboids attached to, these
are the objects that are going to be exported.
:height: int height of the image size
:width: int width of the image size
:camera_name: string for the camera name nvisii entity
:camera_struct: dictionary of the camera look at information. Expecting the following
entries: 'at','eye','up'. All three has to be floating arrays of three entries.
This is an optional export.
:visibility_percentage: bool if you want to export the visibility percentage of the object.
Careful this can be costly on a scene with a lot of objects.
:return nothing:
"""
import simplejson as json
# assume we only use the view camera
cam_matrix = nvisii.entity.get(
camera_name).get_transform().get_world_to_local_matrix()
cam_matrix_export = []
for row in cam_matrix:
cam_matrix_export.append([row[0], row[1], row[2], row[3]])
cam_world_location = nvisii.entity.get(
camera_name).get_transform().get_position()
cam_world_quaternion = nvisii.entity.get(
camera_name).get_transform().get_rotation()
# cam_world_quaternion = nvisii.quat_cast(cam_matrix)
cam_intrinsics = nvisii.entity.get(
camera_name).get_camera().get_intrinsic_matrix(width, height)
if camera_struct is None:
camera_struct = {
'at': [
0,
0,
0,
],
'eye': [
0,
0,
0,
],
'up': [
0,
0,
0,
]
}
dict_out = {
"camera_data": {
"width":
width,
'height':
height,
'camera_look_at': {
'at': [
camera_struct['at'][0],
camera_struct['at'][1],
camera_struct['at'][2],
],
'eye': [
camera_struct['eye'][0],
camera_struct['eye'][1],
camera_struct['eye'][2],
],
'up': [
camera_struct['up'][0],
camera_struct['up'][1],
camera_struct['up'][2],
]
},
'camera_view_matrix':
cam_matrix_export,
'location_world': [
cam_world_location[0],
cam_world_location[1],
cam_world_location[2],
],
'quaternion_world_xyzw': [
cam_world_quaternion[0],
cam_world_quaternion[1],
cam_world_quaternion[2],
cam_world_quaternion[3],
],
'intrinsics': {
'fx': cam_intrinsics[0][0],
'fy': cam_intrinsics[1][1],
'cx': cam_intrinsics[2][0],
'cy': cam_intrinsics[2][1]
}
},
"objects": []
}
# Segmentation id to export
id_keys_map = nvisii.entity.get_name_to_id_map()
for obj_name in obj_names:
projected_keypoints, _ = get_cuboid_image_space(
obj_name, camera_name=camera_name)
# put them in the image space.
for i_p, p in enumerate(projected_keypoints):
projected_keypoints[i_p] = [p[0] * width, p[1] * height]
# Get the location and rotation of the object in the camera frame
trans = nvisii.transform.get(obj_name)
quaternion_xyzw = nvisii.inverse(
cam_world_quaternion) * trans.get_rotation()
object_world = nvisii.vec4(trans.get_position()[0],
trans.get_position()[1],
trans.get_position()[2], 1)
pos_camera_frame = cam_matrix * object_world
#check if the object is visible
visibility = -1
bounding_box = [-1, -1, -1, -1]
segmentation_mask = nvisii.render_data(
width=int(width),
height=int(height),
start_frame=0,
frame_count=1,
bounce=int(0),
options="entity_id",
)
segmentation_mask = np.array(segmentation_mask).reshape(
width, height, 4)[:, :, 0]
if visibility_percentage == True and int(
id_keys_map[obj_name]) in np.unique(
segmentation_mask.astype(int)):
transforms_to_keep = {}
for name in id_keys_map.keys():
if 'camera' in name.lower() or obj_name in name:
continue
trans_to_keep = nvisii.entity.get(name).get_transform()
transforms_to_keep[name] = trans_to_keep
nvisii.entity.get(name).clear_transform()
# Percentage visibility through full segmentation mask.
segmentation_unique_mask = nvisii.render_data(
width=int(width),
height=int(height),
start_frame=0,
frame_count=1,
bounce=int(0),
options="entity_id",
)
segmentation_unique_mask = np.array(
segmentation_unique_mask).reshape(width, height, 4)[:, :, 0]
values_segmentation = np.where(
segmentation_mask == int(id_keys_map[obj_name]))[0]
values_segmentation_full = np.where(
segmentation_unique_mask == int(id_keys_map[obj_name]))[0]
visibility = len(values_segmentation) / float(
len(values_segmentation_full))
# set back the objects from remove
for entity_name in transforms_to_keep.keys():
nvisii.entity.get(entity_name).set_transform(
transforms_to_keep[entity_name])
else:
if int(id_keys_map[obj_name]) in np.unique(
segmentation_mask.astype(int)):
#
visibility = 1
y, x = np.where(
segmentation_mask == int(id_keys_map[obj_name]))
bounding_box = [
int(min(x)),
int(max(x)), height - int(max(y)), height - int(min(y))
]
else:
visibility = 0
# Final export
dict_out['objects'].append({
'class':
obj_name.split('_')[0],
'name':
obj_name,
'provenance':
'nvisii',
# TODO check the location
'location':
[pos_camera_frame[0], pos_camera_frame[1], pos_camera_frame[2]],
'quaternion_xyzw': [
quaternion_xyzw[0],
quaternion_xyzw[1],
quaternion_xyzw[2],
quaternion_xyzw[3],
],
'quaternion_xyzw_world': [
trans.get_rotation()[0],
trans.get_rotation()[1],
trans.get_rotation()[2],
trans.get_rotation()[3]
],
'projected_cuboid':
projected_keypoints[0:8],
'projected_cuboid_centroid':
projected_keypoints[8],
'segmentation_id':
id_keys_map[obj_name],
'visibility_image':
visibility,
'bounding_box': {
'top_left': [
bounding_box[0],
bounding_box[2],
],
'bottom_right': [
bounding_box[1],
bounding_box[3],
],
},
})
with open(filename, 'w+') as fp:
json.dump(dict_out, fp, indent=4, sort_keys=True)
file_path=opt.outf + "20_frame1.png"
)
obj1.get_transform().set_position(obj1.get_transform().get_position(),previous=True)
obj1.get_transform().add_position(nvisii.vec3(0,0.5,0))
obj2.get_transform().set_position(obj2.get_transform().get_position(),previous=True)
obj2.get_transform().add_position(nvisii.vec3(0,0,0.5))
obj3.get_transform().set_rotation(obj3.get_transform().get_rotation(),previous=True)
obj3.get_transform().add_rotation(nvisii.quat(0,-1,0,0))
motion_vectors_array = nvisii.render_data(
width=int(opt.width),
height=int(opt.height),
start_frame=0,
frame_count=1,
bounce=int(0),
options="diffuse_motion_vectors"
)
motion_vectors_array = np.array(motion_vectors_array).reshape(opt.height,opt.width,4) * -1
motion_vectors_array = np.flipud(motion_vectors_array)
image = generate_image_from_motion_vector(motion_vectors_array)
cv2.imwrite(opt.outf + "20_motion_from_1_to_2.png",image*255)
# frame now has to be set at 1 to have the current image, e.g., the transformed one
nvisii.sample_time_interval((1,1))
nvisii.render_to_file(
width=int(opt.width),
height=int(opt.height),
