def render_big_gallery(results_dir,
nb=30,
pts_colors=[0.5, 0.5, 0.5],
draw_text=False):
'''
pts_colors: [0,0,0]
return np array of a big image
'''
cam = PerspectiveCamera(yfov=(YFOV))
cam_pose = CAM_POSE
point_l = PointLight(color=np.ones(3), intensity=POINT_LIGHT_INTENSITY)
scene = Scene(bg_color=np.array([1, 1, 1, 0]))
# cam and light
_ = scene.add(cam, pose=cam_pose)
_ = scene.add(point_l, pose=cam_pose)
input_ply_filenames = get_all_filnames(results_dir, nb)
r = OffscreenRenderer(viewport_width=640 * 2,
viewport_height=480 * 2,
point_size=POINT_SIZE)
pc_pose = PC_POSE
images = []
for _, input_pf in enumerate(input_ply_filenames):
input_pc = read_ply_xyz(input_pf)
colors = np.array(pts_colors)
colors = np.tile(colors, (input_pc.shape[0], 1))
input_pc_node = add_point_cloud_mesh_to_scene(input_pc, scene, pc_pose,
colors)
renderred_color, _ = r.render(scene)
scene.remove_node(input_pc_node)
if draw_text:
im_here = Image.fromarray(renderred_color)
d = ImageDraw.Draw(im_here)
fnt = ImageFont.truetype(
font='/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',
size=100)
d.text((0, 0),
input_pf.split('/')[-1],
fill=(0, 0, 0, 255),
font=fnt)
renderred_color = np.array(im_here)
images.append(renderred_color)
big_gallery = np.concatenate(images, axis=0)
r.delete()
return big_gallery
def render_big_gallery_overlay(dir_1,
dir_2,
pts_color_1=[0.5, 0.5, 0.5],
pts_color_2=[0.5, 0.5, 0.5],
nb=30):
'''
return np array of a big image
'''
cam = PerspectiveCamera(yfov=(YFOV))
cam_pose = CAM_POSE
point_l = PointLight(color=np.ones(3), intensity=POINT_LIGHT_INTENSITY)
scene = Scene(bg_color=np.array([1, 1, 1, 0]))
# cam and light
_ = scene.add(cam, pose=cam_pose)
_ = scene.add(point_l, pose=cam_pose)
input_ply_filenames_1 = get_all_filnames(dir_1, nb)
input_ply_filenames_2 = get_all_filnames(dir_2, nb)
r = OffscreenRenderer(viewport_width=640 * 2,
viewport_height=480 * 2,
point_size=POINT_SIZE)
pc_pose = PC_POSE
images = []
for idx, input_pf in enumerate(input_ply_filenames_1):
input_pc_1 = read_ply_xyz(input_pf)
input_pc_2 = read_ply_xyz(input_ply_filenames_2[idx])
color_1 = np.array(pts_color_1)
color_1 = np.tile(color_1, (input_pc_1.shape[0], 1))
color_2 = np.array(pts_color_2)
color_2 = np.tile(color_2, (input_pc_2.shape[0], 1))
input_pc_node_1 = add_point_cloud_mesh_to_scene(
input_pc_1, scene, pc_pose, color_1)
input_pc_node_2 = add_point_cloud_mesh_to_scene(
input_pc_2, scene, pc_pose, color_2)
renderred_color, _ = r.render(scene)
scene.remove_node(input_pc_node_1)
scene.remove_node(input_pc_node_2)
images.append(renderred_color)
big_gallery = np.concatenate(images, axis=0)
r.delete()
return big_gallery
def __init__(self):
"""
"""
self.scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02, 1.0]))
self.scene.add(PointLight(color=[0.5, 0.2, 0.3], intensity=2.0))
self.scene.add(
SpotLight(color=[0.1, 0.6, 0.3],
intensity=2.0,
innerConeAngle=0.05,
outerConeAngle=0.5))
self.scene.add(
DirectionalLight(color=[0.33, 0.33, 0.33], intensity=2.0))
self.root_node = None
def test_point_light():
s = PointLight()
assert s.name is None
assert np.all(s.color == 1.0)
assert s.intensity == 1.0
assert s.range is None
with pytest.raises(ValueError):
s.range = -1.0
with pytest.raises(ValueError):
s.range = 0.0
s.range = 5.0
with pytest.raises(NotImplementedError):
s._generate_shadow_texture()
with pytest.raises(NotImplementedError):
s._get_shadow_camera(scene_scale=5.0)
def render_sensor(
point_set,
render_sensor_path="/Users/macbookpro15/Documents/mujoco_hand_exps/data/sensor_render"
):
"""
pointset: it is collectiono of sensor points for all timesteps
"""
# first take one of the point, subtract the center from it which
# I know is the 0-th position out of the 220 points
# form the mesh from this
if not os.path.exists(render_sensor_path):
os.makedirs(render_sensor_path)
time_steps = len(point_set)
for t in range(time_steps):
sensor = trimesh.load_mesh(
f'../data/mesh_dir/mesh_{t}_out/mc_mesh_out.ply')
sensor_mesh = Mesh.from_trimesh(sensor)
# Light for the scene
direc_l = DirectionalLight(color=np.ones(3), intensity=1.0)
spot_l = SpotLight(color=np.ones(3),
intensity=10.0,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6)
point_l = PointLight(color=np.ones(3), intensity=10.0)
# add camera to the scene
cam = PerspectiveCamera(yfov=(np.pi / 3.0))
cam_pose = np.array([[0.0, -np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.5], [1.0, 0.0, 0.0, 0.0],
[0.0, np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.4], [0.0, 0.0, 0.0, 1.0]])
# create the scene
scene = Scene(ambient_light=np.array([0.02, 0.02, 0.02, 1.0]))
point_mesh_node = scene.add(sensor_mesh)
direc_l_node = scene.add(direc_l, pose=cam_pose)
spot_l_node = scene.add(spot_l, pose=cam_pose)
cam_node = scene.add(cam, pose=cam_pose)
print('rendering the scene offline')
r = OffscreenRenderer(viewport_width=640, viewport_height=480)
color, depth = r.render(scene)
r.delete()
plt.figure()
plt.imshow(color)
plt.savefig(f'{render_sensor_path}/img_{t}.jpg')
def render_mesh(mesh, h=256, w=256):
"""https://pyrender.readthedocs.io/en/latest/examples/quickstart.html"""
mesh = pyrender.Mesh.from_trimesh(mesh.trimesh())
scene = Scene()
scene.add(mesh)
# z-axis away from the scene, x-axis right, y-axis up
pose = np.eye(4)
pose[2, 3] = 250
# add camera
camera = PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0)
scene.add(camera, pose=pose)
# add light
# light = DirectionalLight(color=np.ones(3), intensity=5.0)
light = PointLight(color=[1.0, 1.0, 1.0], intensity=2.0)
scene.add(light, pose=pose)
r = OffscreenRenderer(h, w)
color, depth = r.render(scene)
return color
# Creating meshes from point clouds
# ------------------------------------------------------------------------------
points = trimesh.creation.icosphere(radius=0.05).vertices
point_colors = np.random.uniform(size=points.shape)
points_mesh = Mesh.from_points(points, colors=point_colors)
# ==============================================================================
# Light creation
# ==============================================================================
direc_l = DirectionalLight(color=np.ones(3), intensity=1.0)
spot_l = SpotLight(color=np.ones(3),
intensity=10.0,
innerConeAngle=np.pi / 16,
outerConeAngle=np.pi / 6)
point_l = PointLight(color=np.ones(3), intensity=10.0)
# ==============================================================================
# Camera creation
# ==============================================================================
cam = PerspectiveCamera(yfov=(np.pi / 3.0))
cam_pose = np.array([[0.0, -np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.5], [1.0, 0.0, 0.0, 0.0],
[0.0, np.sqrt(2) / 2,
np.sqrt(2) / 2, 0.4], [0.0, 0.0, 0.0, 1.0]])
# ==============================================================================
# Scene creation
# ==============================================================================
def test_scenes():
# Basics
s = Scene()
assert np.allclose(s.bg_color, np.ones(4))
assert np.allclose(s.ambient_light, np.zeros(3))
assert len(s.nodes) == 0
assert s.name is None
s.name = 'asdf'
s.bg_color = None
s.ambient_light = None
assert np.allclose(s.bg_color, np.ones(4))
assert np.allclose(s.ambient_light, np.zeros(3))
assert s.nodes == set()
assert s.cameras == set()
assert s.lights == set()
assert s.point_lights == set()
assert s.spot_lights == set()
assert s.directional_lights == set()
assert s.meshes == set()
assert s.camera_nodes == set()
assert s.light_nodes == set()
assert s.point_light_nodes == set()
assert s.spot_light_nodes == set()
assert s.directional_light_nodes == set()
assert s.mesh_nodes == set()
assert s.main_camera_node is None
assert np.all(s.bounds == 0)
assert np.all(s.centroid == 0)
assert np.all(s.extents == 0)
assert np.all(s.scale == 0)
# From trimesh scene
tms = trimesh.load('tests/data/WaterBottle.glb')
s = Scene.from_trimesh_scene(tms)
assert len(s.meshes) == 1
assert len(s.mesh_nodes) == 1
# Test bg color formatting
s = Scene(bg_color=[0, 1.0, 0])
assert np.allclose(s.bg_color, np.array([0.0, 1.0, 0.0, 1.0]))
# Test constructor for nodes
n1 = Node()
n2 = Node()
n3 = Node()
nodes = [n1, n2, n3]
s = Scene(nodes=nodes)
n1.children.append(n2)
s = Scene(nodes=nodes)
n3.children.append(n2)
with pytest.raises(ValueError):
s = Scene(nodes=nodes)
n3.children = []
n2.children.append(n3)
n3.children.append(n2)
with pytest.raises(ValueError):
s = Scene(nodes=nodes)
# Test node accessors
n1 = Node()
n2 = Node()
n3 = Node()
nodes = [n1, n2]
s = Scene(nodes=nodes)
assert s.has_node(n1)
assert s.has_node(n2)
assert not s.has_node(n3)
# Test node poses
for n in nodes:
assert np.allclose(s.get_pose(n), np.eye(4))
with pytest.raises(ValueError):
s.get_pose(n3)
with pytest.raises(ValueError):
s.set_pose(n3, np.eye(4))
tf = np.eye(4)
tf[:3, 3] = np.ones(3)
s.set_pose(n1, tf)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), np.eye(4))
nodes = [n1, n2, n3]
tf2 = np.eye(4)
tf2[:3, :3] = np.diag([-1, -1, 1])
n1.children.append(n2)
n1.matrix = tf
n2.matrix = tf2
s = Scene(nodes=nodes)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), tf.dot(tf2))
assert np.allclose(s.get_pose(n3), np.eye(4))
n1 = Node()
n2 = Node()
n3 = Node()
n1.children.append(n2)
s = Scene()
s.add_node(n1)
with pytest.raises(ValueError):
s.add_node(n2)
s.set_pose(n1, tf)
assert np.allclose(s.get_pose(n1), tf)
assert np.allclose(s.get_pose(n2), tf)
s.set_pose(n2, tf2)
assert np.allclose(s.get_pose(n2), tf.dot(tf2))
# Test node removal
n1 = Node()
n2 = Node()
n3 = Node()
n1.children.append(n2)
n2.children.append(n3)
s = Scene(nodes=[n1, n2, n3])
s.remove_node(n2)
assert len(s.nodes) == 1
assert n1 in s.nodes
assert len(n1.children) == 0
assert len(n2.children) == 1
s.add_node(n2, parent_node=n1)
assert len(n1.children) == 1
n1.matrix = tf
n3.matrix = tf2
assert np.allclose(s.get_pose(n3), tf.dot(tf2))
# Now test ADD function
s = Scene()
m = Mesh([], name='m')
cp = PerspectiveCamera(yfov=2.0)
co = OrthographicCamera(xmag=1.0, ymag=1.0)
dl = DirectionalLight()
pl = PointLight()
sl = SpotLight()
n1 = s.add(m, name='mn')
assert n1.mesh == m
assert len(s.nodes) == 1
assert len(s.mesh_nodes) == 1
assert n1 in s.mesh_nodes
assert len(s.meshes) == 1
assert m in s.meshes
assert len(s.get_nodes(node=n2)) == 0
n2 = s.add(m, pose=tf)
assert len(s.nodes) == len(s.mesh_nodes) == 2
assert len(s.meshes) == 1
assert len(s.get_nodes(node=n1)) == 1
assert len(s.get_nodes(node=n1, name='mn')) == 1
assert len(s.get_nodes(name='mn')) == 1
assert len(s.get_nodes(obj=m)) == 2
assert len(s.get_nodes(obj=m, obj_name='m')) == 2
assert len(s.get_nodes(obj=co)) == 0
nsl = s.add(sl, name='sln')
npl = s.add(pl, parent_name='sln')
assert nsl.children[0] == npl
ndl = s.add(dl, parent_node=npl)
assert npl.children[0] == ndl
nco = s.add(co)
ncp = s.add(cp)
assert len(s.light_nodes) == len(s.lights) == 3
assert len(s.point_light_nodes) == len(s.point_lights) == 1
assert npl in s.point_light_nodes
assert len(s.spot_light_nodes) == len(s.spot_lights) == 1
assert nsl in s.spot_light_nodes
assert len(s.directional_light_nodes) == len(s.directional_lights) == 1
assert ndl in s.directional_light_nodes
assert len(s.cameras) == len(s.camera_nodes) == 2
assert s.main_camera_node == nco
s.main_camera_node = ncp
s.remove_node(ncp)
assert len(s.cameras) == len(s.camera_nodes) == 1
assert s.main_camera_node == nco
s.remove_node(n2)
assert len(s.meshes) == 1
s.remove_node(n1)
assert len(s.meshes) == 0
s.remove_node(nsl)
assert len(s.lights) == 0
s.remove_node(nco)
assert s.main_camera_node is None
s.add_node(n1)
s.clear()
assert len(s.nodes) == 0
# Trigger final errors
with pytest.raises(ValueError):
s.main_camera_node = None
with pytest.raises(ValueError):
s.main_camera_node = ncp
with pytest.raises(ValueError):
s.add(m, parent_node=n1)
with pytest.raises(ValueError):
s.add(m, name='asdf')
s.add(m, name='asdf')
s.add(m, parent_name='asdf')
with pytest.raises(ValueError):
s.add(m, parent_name='asfd')
with pytest.raises(TypeError):
s.add(None)
s.clear()
# Test bounds
m1 = Mesh.from_trimesh(trimesh.creation.box())
m2 = Mesh.from_trimesh(trimesh.creation.box())
m3 = Mesh.from_trimesh(trimesh.creation.box())
n1 = Node(mesh=m1)
n2 = Node(mesh=m2, translation=[1.0, 0.0, 0.0])
n3 = Node(mesh=m3, translation=[0.5, 0.0, 1.0])
s.add_node(n1)
s.add_node(n2)
s.add_node(n3)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [1.5, 0.5, 1.5]])
s.clear()
s.add_node(n1)
s.add_node(n2, parent_node=n1)
s.add_node(n3, parent_node=n2)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [2.0, 0.5, 1.5]])
tf = np.eye(4)
tf[:3, 3] = np.ones(3)
s.set_pose(n3, tf)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [2.5, 1.5, 1.5]])
s.remove_node(n2)
assert np.allclose(s.bounds, [[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]])
s.clear()
assert np.allclose(s.bounds, 0.0)
# which can be added if we want more leaves in the background decor_file_name = 'VG06_4.obj' decor_type = file_to_type[decor_file_name] decor_path_name = './models/flower_models/obj/'+decor_type+'/' decor_trimesh = trimesh.load(decor_path_name+decor_file_name) decor_trimesh_list = decor_trimesh.dump() decor_mesh = Mesh.from_trimesh(decor_trimesh_list[0]) decor_pose = get_flower_pose(decor_type, decor_file_name) #============================================================================== # Light creation #============================================================================== # Lights for the walls side_intensity = 2.0 spot_l_sides = PointLight(color=np.ones(3), intensity=side_intensity) # Light for the flower pointing down at origin point_l = PointLight(color=np.ones(3), intensity=0.4) # Light for the flower min_intensity = 0.05 max_intensity = 0.2 number_of_intensities = 5 varying_light_intensities = np.linspace(min_intensity,max_intensity,number_of_intensities).tolist() point_l_cam = PointLight(color=np.ones(3), intensity=max_intensity) #============================================================================== # Pose creation #==============================================================================
RigidStatic.create_plane(material=Material(
static_friction=0.1, dynamic_friction=0.1, restitution=0.5)))
actor = RigidDynamic()
actor.attach_shape(Shape.create_box([0.2] * 3, Material(restitution=1.)))
actor.set_global_pose([0.5, 0.5, 1.0])
actor.set_mass(1.)
scene.add_actor(actor)
render = PyPhysxOffscreenRenderer()
# Offscreen render has no lighting by default - that is opposite to the pyrender viewer. Therefore, we create some
# lighting to get nice rendered pictures.
render.render_scene.ambient_light = [0.1] * 3
render.render_scene.add_node(
Node(light=PointLight(color=[0.2, 0.2, 1.0], intensity=3.0),
matrix=np.eye(4)))
render.add_physx_scene(scene) # add PyPhysX scene
frames = []
rate = Rate(60)
for _ in range(
20): # render 20 frames at 10 fps and store images into the list
for _ in range(6):
scene.simulate(rate.period()) # simulation runs at 60 Hz
render.update()
rgb, depth = render.get_rgb_and_depth()
depth = np.clip((depth * 20), 0, 255).astype(dtype=np.uint8)
frames.append(
np.concatenate(
# Different options for the walls
wall_types = ['hydroponic_farm', 'other']
wall_type = wall_types[0]
# Side "walls" mesh
wall_trimesh = trimesh.load('./models/scene/' + wall_type + '_wall.obj')
wall_mesh = Mesh.from_trimesh(wall_trimesh)
#==============================================================================
# Light creation
#==============================================================================
# Lights for the inf_base and walls
side_intensity = 3.0
inf_base_dir_l = DirectionalLight(color=np.ones(3), intensity=10.0)
spot_l_sides = PointLight(color=np.ones(3), intensity=side_intensity)
# Light for the flower
point_l = PointLight(color=np.ones(3), intensity=1.0)
#==============================================================================
# Poses
#==============================================================================
# Side wall poses
side_closeness = 1
side_height = 0
side_E_pose = spatial_transform_Matrix(roll=np.pi / 2,
yaw=np.pi / 2,
t_x=-1 / side_closeness,