def render(self, mode='rgb_array', resize=True):
params = {
'distance': 1.8,
'azimuth': 90,
'elevation': -60,
'crop_box': (16.75, 25.0, 105.0, 88.75),
'size': 120
}
camera = mujoco.Camera(physics=self.physics,
height=params['size'],
width=params['size'],
camera_id=-1)
camera._render_camera.distance = params['distance'] # pylint: disable=protected-access
camera._render_camera.azimuth = params['azimuth'] # pylint: disable=protected-access
camera._render_camera.elevation = params['elevation'] # pylint: disable=protected-access
camera._render_camera.lookat[:] = [0, 0.535, 1.1] # pylint: disable=protected-access
image = camera.render(depth=False, segmentation=False)
camera._scene.free() # pylint: disable=protected-access
if resize:
image = Image.fromarray(image).crop(box=params['crop_box'])
image = image.resize([self.image_size, self.image_size],
resample=Image.ANTIALIAS)
image = np.asarray(image)
return image
def render_offscreen(self,
width: int,
height: int,
mode: RenderMode = RenderMode.RGB,
camera_id: int = -1) -> np.ndarray:
"""Renders the camera view as a numpy array of pixels.
Args:
width: The viewport width (pixels).
height: The viewport height (pixels).
mode: The rendering mode.
camera_id: The ID of the camera to render from. By default, uses
the free camera.
Returns:
A numpy array of the pixels.
"""
assert width > 0 and height > 0
# TODO(michaelahn): Consider caching the camera.
camera = dm_mujoco.Camera(physics=self._sim,
height=height,
width=width,
camera_id=camera_id)
# Update the camera configuration for the free-camera.
if camera_id == -1:
self._update_camera_properties(camera._render_camera) # pylint: disable=protected-access
image = camera.render(depth=(mode == RenderMode.DEPTH),
segmentation=(mode == RenderMode.SEGMENTATION))
camera._scene.free() # pylint: disable=protected-access
return image
def __init__(self,
model_path='ant.xml',
height=120,
width=160,
camera_ids=None):
if camera_ids is None:
camera_ids = ['head_camera']
self._camera_ids = camera_ids
if not self._camera_ids:
raise ValueError('We need at least one camera ID')
self._height = height
self._width = width
ant.AntEnv.__init__(self, model_path=model_path)
self.all_cameras = {
camera_id: mujoco.Camera(self.physics,
height=self._height,
width=self._width,
camera_id=camera_id)
for camera_id in self._camera_ids
}
def camera_setup(self):
self.camera = mujoco.Camera(self.physics,
height=self._height,
width=self._width,
camera_id=self._camera_ids[0])
xyz_global = box_pos[:, None] + box_mat @ xyz_local
# # Get the world coordinates of the box corners
# box_pos = physics.named.data.geom_xpos['red_box']
# box_mat = physics.named.data.geom_xmat['red_box'].reshape(3, 3)
# box_size = physics.named.model.geom_size['red_box']
# offsets = np.array([-1, 1]) * box_size[:, None]
# xyz_local = np.stack(itertools.product(*offsets)).T
# xyz_global = box_pos[:, None] + box_mat @ xyz_local
# # Camera matrices multiply homogenous [x, y, z, 1] vectors.
# corners_homogeneous = np.ones((4, xyz_global.shape[1]), dtype=float)
# corners_homogeneous[:3, :] = xyz_global
# Get the camera matrix.
camera = mujoco.Camera(physics, 200, 150, "fixedcam")
depth_img = camera.render(depth=True)
depth_img[depth_img == np.max(depth_img)] = np.NaN
if (args.plots):
plt.imshow(depth_img)
plt.colorbar()
plt.show()
image, focal, rotation, translation = camera.matrices()
pc = cv2.rgbd.depthTo3d(depth_img, image @ focal[:, :3])
pc = pc.reshape(pc.shape[0] * pc.shape[1], 3)
pc = pc[np.isfinite(pc).any(axis=1)]
creatures = [make_creature(num_legs=num_legs) for num_legs in range(1, 7)]
# Place them on a grid in the arena.
height = .15
grid = 5 * BODY_RADIUS
xpos, ypos, zpos = np.meshgrid([-grid, 0, grid], [0, grid], [height])
for i, model in enumerate(creatures):
# Place spawn sites on a grid.
spawn_pos = (xpos.flat[i], ypos.flat[i], zpos.flat[i])
spawn_site = arena.worldbody.add('site', pos=spawn_pos, group=3)
# Attach to the arena at the spawn sites, with a free joint.
spawn_site.attach(model).add('freejoint')
# Instantiate the physics and render.
physics = mjcf.Physics.from_mjcf_model(arena)
camera = mujoco.Camera(physics, img_height, img_width)
plt.imshow(PIL.Image.fromarray(camera.render()))
plt.show()
duration = 10 # (Seconds)
framerate = 30 # (Hz)
video = []
pos_x = []
pos_y = []
torsos = [] # List of torso geom elements.
actuators = [] # List of actuator elements.
for creature in creatures:
torsos.append(creature.find('geom', 'torso'))
actuators.extend(creature.find_all('actuator'))
# Control signal frequency, phase, amplitude.
PIL.Image.fromarray(pixels.astype(np.uint8)) # Get the world coordinates of the box corners box_pos = physics.named.data.geom_xpos['red_box'] box_mat = physics.named.data.geom_xmat['red_box'].reshape(3, 3) box_size = physics.named.model.geom_size['red_box'] offsets = np.array([-1, 1]) * box_size[:, None] xyz_local = np.stack(itertools.product(*offsets)).T xyz_global = box_pos[:, None] + box_mat @ xyz_local # Camera matrices multiply homogenous [x, y, z, 1] vectors. corners_homogeneous = np.ones((4, xyz_global.shape[1]), dtype=float) corners_homogeneous[:3, :] = xyz_global # Get the camera matrix. camera = mujoco.Camera(physics) camera_matrix = camera.matrix # Project world coordinates into pixel space. See: # https://en.wikipedia.org/wiki/3D_projection#Mathematical_formula xs, ys, s = camera_matrix @ corners_homogeneous # x and y are in the pixel coordinate system. x = xs / s y = ys / s # Render the camera view and overlay the projected corner coordinates. pixels = camera.render() fig, ax = plt.subplots(1, 1) ax.imshow(pixels) ax.plot(x, y, '+', c='w') ax.set_axis_off()
zs = genz()
rots = genrot()
quats = make_quat(rots).T
with physics.reset_context():
physics.data.qpos[0::7] = xs
physics.data.qpos[1::7] = ys
physics.data.qpos[2::7] = zs
physics.data.qpos[3::7] = quats[0]
physics.data.qpos[4::7] = quats[1]
physics.data.qpos[5::7] = quats[2]
physics.data.qpos[6::7] = quats[3]
# physics.data.geom_xpos[1:,1][:] = geny()
# physics.data.geom_xpos[1:,2][:] = genz()
# physics.data.xquat[1:][:] = make_quad(genrot())
# print(physics.data.xpos)
# print(physics.data.xquat)
# help(physics.data)
camera = depth_camera.DepthCamera(
mujoco.Camera(physics, img_height, img_width, camera_id="carcam"))
camera.save_ply(f"{dataset_folder}/{idx:06d}.ply")
camera.save_kitti_pc(f"{dataset_folder}/velodyne/{idx:06d}.bin", True)
camera.save_img(f"{dataset_folder}/image_2/{idx:06d}.png")
camera.save_calibration(f"{dataset_folder}/calib/{idx:06d}.txt")
boxes = make_object_box(xs, ys, zs, rots, BODY_SIZE, camera)
save_object_boxes(boxes, f"{dataset_folder}/label_2/{idx:06d}.txt")
render_object_boxes(boxes, f"{dataset_folder}/{idx:06d}_labels.ply")
print("Saved instance {}".format(idx))