def get_pixel(x: int, y: int) -> Tuple[int, int, int]:
"""
Given an x, y, will return RGB color value of that point.
:param int x: x location
:param int y: y location
:returns: Color
"""
# noinspection PyCallingNonCallable,PyTypeChecker
# The window may be 'scaled' on hi-res displays. Particularly Macs. OpenGL
# won't account for this, so we need to.
window = get_window()
if not window:
raise ValueError("No window is available to get pixel data from.")
pixel_ratio = window.get_pixel_ratio()
x = int(pixel_ratio * x)
y = int(pixel_ratio * y)
a = (gl.GLubyte * 3)(0)
gl.glReadPixels(x, y, 1, 1, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, a)
red = a[0]
green = a[1]
blue = a[2]
return red, green, blue
def on_draw():
window.clear()
background.blit()
for pipe in pipes:
pipe.blit()
floor.blit()
bird.blit()
if not state.started:
tap_to_start.blit(0.5 * (window.width - tap_to_start.width * 0.37), 0.43 * window.height)
if bird.dying or bird.dead:
gameover.blit(0.5 * (window.width - gameover.width), 0.5 * window.height)
if callback is not None:
import numpy as np
buf = (gl.GLubyte * (3 * window.width * window.height))(0)
gl.glReadPixels(0, 0, window.width, window.height,
gl.GL_RGB,
gl.GL_UNSIGNED_BYTE, buf)
array = np.frombuffer(buf, dtype='<u1')
array = array.reshape(window.height, window.width, 3)
array = array[::settings.scale, ::settings.scale]
callback(array, click, alive=bird.alive)
# draw score
scoreLabel.draw()
def get_image(x: int = 0, y: int = 0, width: int = None, height: int = None):
"""
Get an image from the screen.
:param int x: Start (left) x location
:param int y: Start (top) y location
:param int width: Width of image. Leave blank for grabbing the 'rest' of the image
:param int height: Height of image. Leave blank for grabbing the 'rest' of the image
You can save the image like:
.. highlight:: python
.. code-block:: python
image = get_image()
image.save('screenshot.png', 'PNG')
"""
# Get the dimensions
window = get_window()
if width is None:
width = window.width - x
if height is None:
height = window.height - y
# Create an image buffer
image_buffer = (gl.GLubyte * (4 * width * height))(0)
gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE,
image_buffer)
image = PIL.Image.frombytes("RGBA", (width, height), image_buffer)
image = PIL.ImageOps.flip(image)
# image.save('glutout.png', 'PNG')
return image
def get_image(x=0, y=0, width=None, height=None):
"""
Get an image from the screen.
You can save the image like:
image = get_image()
image.save('screenshot.png', 'PNG')
"""
# Get the dimensions
window = get_window()
if width is None:
width = window.width - x
if height is None:
height = window.height - y
# Create an image buffer
image_buffer = (gl.GLubyte * (4 * width * height))(0)
gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, image_buffer)
image = PIL.Image.frombytes("RGBA", (width, height), image_buffer)
image = PIL.ImageOps.flip(image)
# image.save('glutout.png', 'PNG')
return image
def on_draw():
#w.clear()
global parent
global parentdiff
global olddrawing,newdrawing
global blitted
global image_pixels
global keeps
global i
if not blitted:
"""
At the start we've not seen the target before,
so draw it and store the pixel data.
"""
pic.blit(0,0)
blitted = 1
image_pixels = (gl.GLubyte * (4*size))(0)
gl.glReadPixels(0,
0,
newdrawing.width,
newdrawing.height,
gl.GL_RGBA,
gl.GL_UNSIGNED_BYTE,
image_pixels
)
image_pixels = np.frombuffer(image_pixels, dtype=np.uint8).astype(np.int32)
# Draw the new child
newdrawing.draw()
# Read the pixel data for the child and find out if its any good
gl.glBindFramebufferEXT(gl.GL_FRAMEBUFFER_EXT, newdrawing.fb)
gl.glReadPixels(0,
0,
newdrawing.width,
newdrawing.height,
gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, a)
gl.glBindFramebufferEXT(gl.GL_FRAMEBUFFER_EXT, 0)
diff = compute_diff(a)
if parent == None or diff < parentdiff:
# The new drawing is better.
# Redraw the parent as this child.
# Set this child's diff as the new one to beat.
parent = image.ImageData(newdrawing.width,newdrawing.height,"RGBA",a)
parentdiff = diff
draw_parent(parent, newdrawing.width)
else:
# The new drawing sucks. Replace it
# dump it's framebuffer first though!!!!
gl.glDeleteFramebuffersEXT(1, ctypes.byref(newdrawing.fb))
newdrawing = olddrawing
i += 1
if (i % 20 == 0):
# Use the window title to let the user know how we're doing
w.set_caption(str(fps.get_fps())+" "+str(parentdiff) + " " + str(log(parentdiff,10))+ " " + str(i))
fps.tick()
def get_pixel(x: int, y: int):
"""
Given an x, y, will return RGB color value of that point.
"""
a = (gl.GLubyte * 3)(0)
gl.glReadPixels(x, y, 1, 1, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, a)
red = a[0]
green = a[1]
blue = a[2]
return (red, green, blue)
def saveFromShader(self):
a = (gl.GLubyte * (4 * self.w * self.h))(0)
# Save without any GUI elements
self.drawGenerated()
gl.glReadPixels(0, 0, self.w, self.h, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, a)
image = pyglet.image.ImageData(self.w, self.h, 'RGBA', a)
scriptPath = os.path.dirname(os.path.realpath(__file__))
filePath = scriptPath + "/TESTSAVE_" + time.strftime("%Y%m%d_%H%M%S") + ".png"
print ("saved to {}".format(filePath))
image.save(filePath)
def on_draw():
#w.clear()
global parent
global parentdiff
global olddrawing, newdrawing
global blitted
global image_pixels
global keeps
global i
if not blitted:
"""
At the start we've not seen the target before,
so draw it and store the pixel data.
"""
pic.blit(0, 0)
blitted = 1
image_pixels = (gl.GLubyte * (4 * size))(0)
gl.glReadPixels(0, 0, newdrawing.width, newdrawing.height,
gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, image_pixels)
image_pixels = np.frombuffer(image_pixels,
dtype=np.uint8).astype(np.int32)
# Draw the new child
newdrawing.draw()
# Read the pixel data for the child and find out if its any good
gl.glBindFramebufferEXT(gl.GL_FRAMEBUFFER_EXT, newdrawing.fb)
gl.glReadPixels(0, 0, newdrawing.width, newdrawing.height, gl.GL_RGBA,
gl.GL_UNSIGNED_BYTE, a)
gl.glBindFramebufferEXT(gl.GL_FRAMEBUFFER_EXT, 0)
diff = compute_diff(a)
if parent == None or diff < parentdiff:
# The new drawing is better.
# Redraw the parent as this child.
# Set this child's diff as the new one to beat.
parent = image.ImageData(newdrawing.width, newdrawing.height,
"RGBA", a)
parentdiff = diff
draw_parent(parent, newdrawing.width)
else:
# The new drawing sucks. Replace it
# dump it's framebuffer first though!!!!
gl.glDeleteFramebuffersEXT(1, ctypes.byref(newdrawing.fb))
newdrawing = olddrawing
i += 1
if (i % 20 == 0):
# Use the window title to let the user know how we're doing
w.set_caption(
str(fps.get_fps()) + " " + str(parentdiff) + " " +
str(log(parentdiff, 10)) + " " + str(i))
fps.tick()
def get_image(x: int = 0, y: int = 0, width: int = None, height: int = None):
"""
Get an image from the screen.
:param int x: Start (left) x location
:param int y: Start (top) y location
:param int width: Width of image. Leave blank for grabbing the 'rest' of the image
:param int height: Height of image. Leave blank for grabbing the 'rest' of the image
You can save the image like:
.. highlight:: python
.. code-block:: python
image = get_image()
image.save('screenshot.png', 'PNG')
"""
# Get the dimensions
window = get_window()
pixel_ratio = window.get_pixel_ratio()
x = int(pixel_ratio * x)
y = int(pixel_ratio * y)
if width is None:
width = window.width - x
if height is None:
height = window.height - y
width = int(pixel_ratio * width)
height = int(pixel_ratio * height)
# Create an image buffer
# noinspection PyTypeChecker
image_buffer = (gl.GLubyte * (4 * width * height))(0)
gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, image_buffer)
image = PIL.Image.frombytes("RGBA", (width, height), image_buffer)
image = PIL.ImageOps.flip(image)
# image.save('glutout.png', 'PNG')
return image
# Methods associated with the frame buffer
def get_fbo(self):
return self.ctx.framebuffer(
color_attachments=self.ctx.texture(self.size, 4, samples=8),
depth_attachment=self.ctx.depth_texture(self.size, samples=8),
)
def get_pixel(x: int, y: int):
"""
Given an x, y, will return RGB color value of that point.
:param int x: x location
:param int y: y location
:returns: Color
"""
# noinspection PyCallingNonCallable,PyTypeChecker
a = (gl.GLubyte * 3)(0)
gl.glReadPixels(x, y, 1, 1, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, a)
red = a[0]
green = a[1]
blue = a[2]
return red, green, blue
def _execute_saving(self):
if self.flag < 3:
self.flag += 1
return
size_x, size_y = self._plot._window.get_size()
size = size_x*size_y*4*ctypes.sizeof(ctypes.c_ubyte)
image = ctypes.create_string_buffer(size)
pgl.glReadPixels(0, 0, size_x, size_y, pgl.GL_RGBA, pgl.GL_UNSIGNED_BYTE, image)
from PIL import Image
im = Image.frombuffer('RGBA', (size_x, size_y),
image.raw, 'raw', 'RGBA', 0, 1)
im.transpose(Image.FLIP_TOP_BOTTOM).save(self.outfile, self.format)
self.flag = 0
self.screenshot_requested = False
if self.invisibleMode:
self._plot._window.close()
def read_pixel(self,
name: str,
x: int,
y: int,
gl_type=gl.GL_FLOAT,
gl_format=gl.GL_RGBA):
"""
This is probably inefficient, but a useful way to find e.g. the scene position
under the mouse cursor.
"""
c_type = GLTYPE_TO_CTYPE[gl_type]
texture = self.textures[name]
position_value = (c_type * 4)()
with self:
gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + texture.unit)
gl.glReadPixels(x, y, 1, 1, gl_format, gl_type,
byref(position_value))
return list(position_value)
def read_buffer(self, format='rgb'):
"""Read back the recently rendered color data from the window's buffer.
Returns
-------
data : unit8 array, shape=(height, width, n_channels)
RGB or RGBA color data.
Details
-------
Call to `.read_buffer()` should be preceded by `.render()` in order
to read the latest color data. The dims of the returned color array
correspond to the width and height of the gl color buffer used for
rendering, and may not coincide with the dims of the original image.
Typically, they correspond to the window dims scaled by some factor.
"""
if not self.isopen:
raise RuntimeError('Cannot read color data from a closed viewer.')
assert format in ('rgb', 'rgba')
if format == 'rgb':
format, n_channels = gl.GL_RGB, 3
else:
format, n_channels = gl.GL_RGBA, 4
# ensure current gl context
self.switch_to()
# make sure the color data we read back is fresh
# self.on_draw()
# `.flip()` x2 may cause unnecessary vsync and flicker
# allocate unit8 buffer for RGB data
width, height = self.get_framebuffer_size()
buf = (gl.GLubyte * (height * width * n_channels))(0)
# read a block of pixels from the current display frame buffer
gl.glReadBuffer(gl.GL_FRONT) # FRONT -- display, BACK -- drawing
gl.glReadPixels(0, 0, width, height, format, gl.GL_UNSIGNED_BYTE, buf)
# properly reshape and reorder the data
flat = np.frombuffer(buf, dtype=np.uint8)
return flat.reshape(height, width, n_channels)[::-1].copy()
def cleanUP(self):
a = (gl.GLint * (self.dimx * self.dimy * 4))()
b = (gl.GLint * (self.dimx * self.dimy * 4))()
gl.glReadPixels(0, 0, self.dimx, self.dimy, gl.GL_RGBA,
gl.GL_FLOAT, b)
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, self.framebufferA1)
gl.glReadPixels(0, 0, self.dimx, self.dimy, gl.GL_RGBA,
gl.GL_FLOAT, a)
#self.flip() # This updates the screen, very much important.
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0)
bufA = np.frombuffer(b, dtype=np.float32)
bufB = np.frombuffer(b, dtype=np.float32)
bufA = bufA.reshape(args["A"].shape)
bufB = bufB.reshape(args["B"].shape)
#consider casting to float64
args["Bout"] = bufB
args["Aout"] = bufA
def on_mouse_motion(self, x, y, dx, dy):
depth = gl.GLfloat(0.0)
gl.glReadPixels(x, y, 1, 1, gl.GL_DEPTH_COMPONENT, gl.GL_FLOAT,
ctypes.pointer(depth))
xn = 2.0 * x / self.vp_width - 1.0
yn = 2.0 * y / self.vp_height - 1.0
zn = 2.0 * depth.value - 1.0
xs = 2.5 * xn + 5.0 * zn
ys = -2.5 * xn + 5.0 * zn
u = xs - int(xs)
v = ys - int(ys)
if u < 0.5 and v < 0.5:
c = 2
elif u < 0.5:
c = 1
elif v < 0.5:
c = 3
else:
c = 0
self.pointed = (int(xs), int(ys), c)
def read(self,
*,
viewport=None,
components=3,
attachment=0,
dtype="f1") -> bytearray:
"""Read framebuffer pixels"""
# TODO: use texture attachment info to determine read format?
try:
frmt = pixel_formats[dtype]
base_format = frmt[0][components]
pixel_type = frmt[2]
except Exception:
raise ValueError(f"Invalid dtype '{dtype}'")
with self:
# Configure attachment to read from
# gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + attachment)
x, y, width, height = 0, 0, self._width, self._height
data = (gl.GLubyte * (components * width * height))(0)
gl.glReadPixels(x, y, width, height, base_format, pixel_type, data)
return bytearray(data)
def _render_img(self, width, height, multi_fbo, final_fbo, img_array, top_down=True):
"""
Render an image of the environment into a frame buffer
Produce a numpy RGB array image as output
"""
if not self.graphics:
return
# Switch to the default context
# This is necessary on Linux nvidia drivers
# pyglet.gl._shadow_window.switch_to()
self.shadow_window.switch_to()
from pyglet import gl
# Bind the multisampled frame buffer
gl.glEnable(gl.GL_MULTISAMPLE)
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, multi_fbo)
gl.glViewport(0, 0, width, height)
# Clear the color and depth buffers
c0, c1, c2 = self.horizon_color
gl.glClearColor(c0, c1, c2, 1.0)
gl.glClearDepth(1.0)
gl.glClear(gl.GL_COLOR_BUFFER_BIT | gl.GL_DEPTH_BUFFER_BIT)
# Set the projection matrix
gl.glMatrixMode(gl.GL_PROJECTION)
gl.glLoadIdentity()
gl.gluPerspective(
self.cam_fov_y,
width / float(height),
0.04,
100.0
)
# Set modelview matrix
# Note: we add a bit of noise to the camera position for data augmentation
pos = self.cur_pos
angle = self.cur_angle
if self.domain_rand:
pos = pos + self.randomization_settings['camera_noise']
x, y, z = pos + self.cam_offset
dx, dy, dz = self.get_dir_vec(angle)
gl.glMatrixMode(gl.GL_MODELVIEW)
gl.glLoadIdentity()
if self.draw_bbox:
y += 0.8
gl.glRotatef(90, 1, 0, 0)
elif not top_down:
y += self.cam_height
gl.glRotatef(self.cam_angle[0], 1, 0, 0)
gl.glRotatef(self.cam_angle[1], 0, 1, 0)
gl.glRotatef(self.cam_angle[2], 0, 0, 1)
gl.glTranslatef(0, 0, self._perturb(gym_duckietown.simulator.CAMERA_FORWARD_DIST))
if top_down:
gl.gluLookAt(
# Eye position
(self.grid_width * self.road_tile_size) / 2,
self.top_cam_height,
(self.grid_height * self.road_tile_size) / 2,
# Target
(self.grid_width * self.road_tile_size) / 2,
0,
(self.grid_height * self.road_tile_size) / 2,
# Up vector
0, 0, -1.0
)
else:
gl.gluLookAt(
# Eye position
x,
y,
z,
# Target
x + dx,
y + dy,
z + dz,
# Up vector
0, 1.0, 0.0
)
# Draw the ground quad
gl.glDisable(gl.GL_TEXTURE_2D)
gl.glColor3f(*self.ground_color)
gl.glPushMatrix()
gl.glScalef(50, 1, 50)
self.ground_vlist.draw(gl.GL_QUADS)
gl.glPopMatrix()
# Draw the ground/noise triangles
self.tri_vlist.draw(gl.GL_TRIANGLES)
# Draw the road quads
gl.glEnable(gl.GL_TEXTURE_2D)
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
# For each grid tile
for j in range(self.grid_height):
for i in range(self.grid_width):
# Get the tile type and angle
tile = self._get_tile(i, j)
if tile is None:
continue
# kind = tile['kind']
angle = tile['angle']
color = tile['color']
texture = tile['texture']
gl.glColor3f(*color)
gl.glPushMatrix()
gl.glTranslatef((i + 0.5) * self.road_tile_size, 0, (j + 0.5) * self.road_tile_size)
gl.glRotatef(angle * 90, 0, 1, 0)
# Bind the appropriate texture
texture.bind()
self.road_vlist.draw(gl.GL_QUADS)
gl.glPopMatrix()
if self.draw_curve and tile['drivable']:
# Find curve with largest dotproduct with heading
curves = self._get_tile(i, j)['curves']
curve_headings = curves[:, -1, :] - curves[:, 0, :]
curve_headings = curve_headings / np.linalg.norm(curve_headings).reshape(1, -1)
dirVec = get_dir_vec(angle)
dot_prods = np.dot(curve_headings, dirVec)
# Current ("closest") curve drawn in Red
pts = curves[np.argmax(dot_prods)]
bezier_draw(pts, n=20, red=True)
pts = self._get_curve(i, j)
for idx, pt in enumerate(pts):
# Don't draw current curve in blue
if idx == np.argmax(dot_prods):
continue
bezier_draw(pt, n=20)
# For each object
for idx, obj in enumerate(self.objects):
obj.render(self.draw_bbox)
# Draw the agent's own bounding box
if self.draw_bbox:
corners = get_agent_corners(pos, angle)
gl.glColor3f(1, 0, 0)
gl.glBegin(gl.GL_LINE_LOOP)
gl.glVertex3f(corners[0, 0], 0.01, corners[0, 1])
gl.glVertex3f(corners[1, 0], 0.01, corners[1, 1])
gl.glVertex3f(corners[2, 0], 0.01, corners[2, 1])
gl.glVertex3f(corners[3, 0], 0.01, corners[3, 1])
gl.glEnd()
if top_down:
gl.glPushMatrix()
gl.glTranslatef(*self.cur_pos)
gl.glScalef(1, 1, 1)
gl.glRotatef(self.cur_angle * 180 / np.pi, 0, 1, 0)
# glColor3f(*self.color)
self.mesh.render()
gl.glPopMatrix()
# Resolve the multisampled frame buffer into the final frame buffer
gl.glBindFramebuffer(gl.GL_READ_FRAMEBUFFER, multi_fbo)
gl.glBindFramebuffer(gl.GL_DRAW_FRAMEBUFFER, final_fbo)
gl.glBlitFramebuffer(
0, 0,
width, height,
0, 0,
width, height,
gl.GL_COLOR_BUFFER_BIT,
gl.GL_LINEAR
)
# Copy the frame buffer contents into a numpy array
# Note: glReadPixels reads starting from the lower left corner
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, final_fbo)
gl.glReadPixels(
0,
0,
width,
height,
gl.GL_RGB,
gl.GL_UNSIGNED_BYTE,
img_array.ctypes.data_as(POINTER(gl.GLubyte))
)
# Unbind the frame buffer
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0)
# Flip the image because OpenGL maps (0,0) to the lower-left corner
# Note: this is necessary for gym.wrappers.Monitor to record videos
# properly, otherwise they are vertically inverted.
img_array = np.ascontiguousarray(np.flip(img_array, axis=0))
return img_array
def render_obs(self):
"""
Render an observation from the point of view of the agent
"""
observation = self._render_img(
self.camera_width,
self.camera_height,
self.multi_fbo,
self.final_fbo,
self.img_array,
top_down=True
)
# self.undistort - for UndistortWrapper
if self.distortion and not self.undistort:
observation = self.camera_model.distort(observation)
return observation
p['offset'] = [-1.0, 0.0]
p['width_ratio'] = float(WIDTH)/HEIGHT
p['zoom'] = 2.0
# Draw on the framebuffer
draw_canvas()
p.disable()
# Copy the data from the graphics card to system memory
import numpy as np
import ctypes
arr = np.empty((HEIGHT, WIDTH, 3), dtype=np.float32)
gl.glFinish()
gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0_EXT)
gl.glReadPixels(0, 0, WIDTH, HEIGHT, gl.GL_RGB, gl.GL_FLOAT,
arr.ctypes.data)
# Display using matplotlib (TODO: use opengl to display)
import matplotlib.pyplot as plt
plt.imshow(arr)
plt.show()
fbo.unbind()
gl.glPopAttrib()
def color_at_point(x, y):
a = (gl.GLubyte * 3)(0)
gl.glReadPixels(x, y, 1, 1, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, a)
return list(a)
def get_pixel_data(x, y, width, height):
p = (4 * width * height * gl.GLubyte)()
gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, p)
return [(r, g, b) for r, g, b, a in partition(p, 4)]
def get_pixel_data(x, y, width, height):
p = (4 * width * height * gl.GLubyte)()
gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, p)
return [(r, g, b) for r, g, b, a in partition(p, 4)]
