def get_pixel(x: int, y: int) -> Tuple[int, int, int]: """ Given an x, y, will return RGB color value of that point. :param int x: x location :param int y: y location :returns: Color """ # noinspection PyCallingNonCallable,PyTypeChecker # The window may be 'scaled' on hi-res displays. Particularly Macs. OpenGL # won't account for this, so we need to. window = get_window() if not window: raise ValueError("No window is available to get pixel data from.") pixel_ratio = window.get_pixel_ratio() x = int(pixel_ratio * x) y = int(pixel_ratio * y) a = (gl.GLubyte * 3)(0) gl.glReadPixels(x, y, 1, 1, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, a) red = a[0] green = a[1] blue = a[2] return red, green, blue
def on_draw(): window.clear() background.blit() for pipe in pipes: pipe.blit() floor.blit() bird.blit() if not state.started: tap_to_start.blit(0.5 * (window.width - tap_to_start.width * 0.37), 0.43 * window.height) if bird.dying or bird.dead: gameover.blit(0.5 * (window.width - gameover.width), 0.5 * window.height) if callback is not None: import numpy as np buf = (gl.GLubyte * (3 * window.width * window.height))(0) gl.glReadPixels(0, 0, window.width, window.height, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, buf) array = np.frombuffer(buf, dtype='<u1') array = array.reshape(window.height, window.width, 3) array = array[::settings.scale, ::settings.scale] callback(array, click, alive=bird.alive) # draw score scoreLabel.draw()
def get_image(x: int = 0, y: int = 0, width: int = None, height: int = None): """ Get an image from the screen. :param int x: Start (left) x location :param int y: Start (top) y location :param int width: Width of image. Leave blank for grabbing the 'rest' of the image :param int height: Height of image. Leave blank for grabbing the 'rest' of the image You can save the image like: .. highlight:: python .. code-block:: python image = get_image() image.save('screenshot.png', 'PNG') """ # Get the dimensions window = get_window() if width is None: width = window.width - x if height is None: height = window.height - y # Create an image buffer image_buffer = (gl.GLubyte * (4 * width * height))(0) gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, image_buffer) image = PIL.Image.frombytes("RGBA", (width, height), image_buffer) image = PIL.ImageOps.flip(image) # image.save('glutout.png', 'PNG') return image
def get_image(x=0, y=0, width=None, height=None): """ Get an image from the screen. You can save the image like: image = get_image() image.save('screenshot.png', 'PNG') """ # Get the dimensions window = get_window() if width is None: width = window.width - x if height is None: height = window.height - y # Create an image buffer image_buffer = (gl.GLubyte * (4 * width * height))(0) gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, image_buffer) image = PIL.Image.frombytes("RGBA", (width, height), image_buffer) image = PIL.ImageOps.flip(image) # image.save('glutout.png', 'PNG') return image
def on_draw(): #w.clear() global parent global parentdiff global olddrawing,newdrawing global blitted global image_pixels global keeps global i if not blitted: """ At the start we've not seen the target before, so draw it and store the pixel data. """ pic.blit(0,0) blitted = 1 image_pixels = (gl.GLubyte * (4*size))(0) gl.glReadPixels(0, 0, newdrawing.width, newdrawing.height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, image_pixels ) image_pixels = np.frombuffer(image_pixels, dtype=np.uint8).astype(np.int32) # Draw the new child newdrawing.draw() # Read the pixel data for the child and find out if its any good gl.glBindFramebufferEXT(gl.GL_FRAMEBUFFER_EXT, newdrawing.fb) gl.glReadPixels(0, 0, newdrawing.width, newdrawing.height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, a) gl.glBindFramebufferEXT(gl.GL_FRAMEBUFFER_EXT, 0) diff = compute_diff(a) if parent == None or diff < parentdiff: # The new drawing is better. # Redraw the parent as this child. # Set this child's diff as the new one to beat. parent = image.ImageData(newdrawing.width,newdrawing.height,"RGBA",a) parentdiff = diff draw_parent(parent, newdrawing.width) else: # The new drawing sucks. Replace it # dump it's framebuffer first though!!!! gl.glDeleteFramebuffersEXT(1, ctypes.byref(newdrawing.fb)) newdrawing = olddrawing i += 1 if (i % 20 == 0): # Use the window title to let the user know how we're doing w.set_caption(str(fps.get_fps())+" "+str(parentdiff) + " " + str(log(parentdiff,10))+ " " + str(i)) fps.tick()
def get_pixel(x: int, y: int): """ Given an x, y, will return RGB color value of that point. """ a = (gl.GLubyte * 3)(0) gl.glReadPixels(x, y, 1, 1, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, a) red = a[0] green = a[1] blue = a[2] return (red, green, blue)
def saveFromShader(self): a = (gl.GLubyte * (4 * self.w * self.h))(0) # Save without any GUI elements self.drawGenerated() gl.glReadPixels(0, 0, self.w, self.h, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, a) image = pyglet.image.ImageData(self.w, self.h, 'RGBA', a) scriptPath = os.path.dirname(os.path.realpath(__file__)) filePath = scriptPath + "/TESTSAVE_" + time.strftime("%Y%m%d_%H%M%S") + ".png" print ("saved to {}".format(filePath)) image.save(filePath)
def on_draw(): #w.clear() global parent global parentdiff global olddrawing, newdrawing global blitted global image_pixels global keeps global i if not blitted: """ At the start we've not seen the target before, so draw it and store the pixel data. """ pic.blit(0, 0) blitted = 1 image_pixels = (gl.GLubyte * (4 * size))(0) gl.glReadPixels(0, 0, newdrawing.width, newdrawing.height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, image_pixels) image_pixels = np.frombuffer(image_pixels, dtype=np.uint8).astype(np.int32) # Draw the new child newdrawing.draw() # Read the pixel data for the child and find out if its any good gl.glBindFramebufferEXT(gl.GL_FRAMEBUFFER_EXT, newdrawing.fb) gl.glReadPixels(0, 0, newdrawing.width, newdrawing.height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, a) gl.glBindFramebufferEXT(gl.GL_FRAMEBUFFER_EXT, 0) diff = compute_diff(a) if parent == None or diff < parentdiff: # The new drawing is better. # Redraw the parent as this child. # Set this child's diff as the new one to beat. parent = image.ImageData(newdrawing.width, newdrawing.height, "RGBA", a) parentdiff = diff draw_parent(parent, newdrawing.width) else: # The new drawing sucks. Replace it # dump it's framebuffer first though!!!! gl.glDeleteFramebuffersEXT(1, ctypes.byref(newdrawing.fb)) newdrawing = olddrawing i += 1 if (i % 20 == 0): # Use the window title to let the user know how we're doing w.set_caption( str(fps.get_fps()) + " " + str(parentdiff) + " " + str(log(parentdiff, 10)) + " " + str(i)) fps.tick()
def get_image(x: int = 0, y: int = 0, width: int = None, height: int = None): """ Get an image from the screen. :param int x: Start (left) x location :param int y: Start (top) y location :param int width: Width of image. Leave blank for grabbing the 'rest' of the image :param int height: Height of image. Leave blank for grabbing the 'rest' of the image You can save the image like: .. highlight:: python .. code-block:: python image = get_image() image.save('screenshot.png', 'PNG') """ # Get the dimensions window = get_window() pixel_ratio = window.get_pixel_ratio() x = int(pixel_ratio * x) y = int(pixel_ratio * y) if width is None: width = window.width - x if height is None: height = window.height - y width = int(pixel_ratio * width) height = int(pixel_ratio * height) # Create an image buffer # noinspection PyTypeChecker image_buffer = (gl.GLubyte * (4 * width * height))(0) gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, image_buffer) image = PIL.Image.frombytes("RGBA", (width, height), image_buffer) image = PIL.ImageOps.flip(image) # image.save('glutout.png', 'PNG') return image # Methods associated with the frame buffer def get_fbo(self): return self.ctx.framebuffer( color_attachments=self.ctx.texture(self.size, 4, samples=8), depth_attachment=self.ctx.depth_texture(self.size, samples=8), )
def get_pixel(x: int, y: int): """ Given an x, y, will return RGB color value of that point. :param int x: x location :param int y: y location :returns: Color """ # noinspection PyCallingNonCallable,PyTypeChecker a = (gl.GLubyte * 3)(0) gl.glReadPixels(x, y, 1, 1, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, a) red = a[0] green = a[1] blue = a[2] return red, green, blue
def _execute_saving(self): if self.flag < 3: self.flag += 1 return size_x, size_y = self._plot._window.get_size() size = size_x*size_y*4*ctypes.sizeof(ctypes.c_ubyte) image = ctypes.create_string_buffer(size) pgl.glReadPixels(0, 0, size_x, size_y, pgl.GL_RGBA, pgl.GL_UNSIGNED_BYTE, image) from PIL import Image im = Image.frombuffer('RGBA', (size_x, size_y), image.raw, 'raw', 'RGBA', 0, 1) im.transpose(Image.FLIP_TOP_BOTTOM).save(self.outfile, self.format) self.flag = 0 self.screenshot_requested = False if self.invisibleMode: self._plot._window.close()
def read_pixel(self, name: str, x: int, y: int, gl_type=gl.GL_FLOAT, gl_format=gl.GL_RGBA): """ This is probably inefficient, but a useful way to find e.g. the scene position under the mouse cursor. """ c_type = GLTYPE_TO_CTYPE[gl_type] texture = self.textures[name] position_value = (c_type * 4)() with self: gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + texture.unit) gl.glReadPixels(x, y, 1, 1, gl_format, gl_type, byref(position_value)) return list(position_value)
def read_buffer(self, format='rgb'): """Read back the recently rendered color data from the window's buffer. Returns ------- data : unit8 array, shape=(height, width, n_channels) RGB or RGBA color data. Details ------- Call to `.read_buffer()` should be preceded by `.render()` in order to read the latest color data. The dims of the returned color array correspond to the width and height of the gl color buffer used for rendering, and may not coincide with the dims of the original image. Typically, they correspond to the window dims scaled by some factor. """ if not self.isopen: raise RuntimeError('Cannot read color data from a closed viewer.') assert format in ('rgb', 'rgba') if format == 'rgb': format, n_channels = gl.GL_RGB, 3 else: format, n_channels = gl.GL_RGBA, 4 # ensure current gl context self.switch_to() # make sure the color data we read back is fresh # self.on_draw() # `.flip()` x2 may cause unnecessary vsync and flicker # allocate unit8 buffer for RGB data width, height = self.get_framebuffer_size() buf = (gl.GLubyte * (height * width * n_channels))(0) # read a block of pixels from the current display frame buffer gl.glReadBuffer(gl.GL_FRONT) # FRONT -- display, BACK -- drawing gl.glReadPixels(0, 0, width, height, format, gl.GL_UNSIGNED_BYTE, buf) # properly reshape and reorder the data flat = np.frombuffer(buf, dtype=np.uint8) return flat.reshape(height, width, n_channels)[::-1].copy()
def cleanUP(self): a = (gl.GLint * (self.dimx * self.dimy * 4))() b = (gl.GLint * (self.dimx * self.dimy * 4))() gl.glReadPixels(0, 0, self.dimx, self.dimy, gl.GL_RGBA, gl.GL_FLOAT, b) gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, self.framebufferA1) gl.glReadPixels(0, 0, self.dimx, self.dimy, gl.GL_RGBA, gl.GL_FLOAT, a) #self.flip() # This updates the screen, very much important. gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0) bufA = np.frombuffer(b, dtype=np.float32) bufB = np.frombuffer(b, dtype=np.float32) bufA = bufA.reshape(args["A"].shape) bufB = bufB.reshape(args["B"].shape) #consider casting to float64 args["Bout"] = bufB args["Aout"] = bufA
def on_mouse_motion(self, x, y, dx, dy): depth = gl.GLfloat(0.0) gl.glReadPixels(x, y, 1, 1, gl.GL_DEPTH_COMPONENT, gl.GL_FLOAT, ctypes.pointer(depth)) xn = 2.0 * x / self.vp_width - 1.0 yn = 2.0 * y / self.vp_height - 1.0 zn = 2.0 * depth.value - 1.0 xs = 2.5 * xn + 5.0 * zn ys = -2.5 * xn + 5.0 * zn u = xs - int(xs) v = ys - int(ys) if u < 0.5 and v < 0.5: c = 2 elif u < 0.5: c = 1 elif v < 0.5: c = 3 else: c = 0 self.pointed = (int(xs), int(ys), c)
def read(self, *, viewport=None, components=3, attachment=0, dtype="f1") -> bytearray: """Read framebuffer pixels""" # TODO: use texture attachment info to determine read format? try: frmt = pixel_formats[dtype] base_format = frmt[0][components] pixel_type = frmt[2] except Exception: raise ValueError(f"Invalid dtype '{dtype}'") with self: # Configure attachment to read from # gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + attachment) x, y, width, height = 0, 0, self._width, self._height data = (gl.GLubyte * (components * width * height))(0) gl.glReadPixels(x, y, width, height, base_format, pixel_type, data) return bytearray(data)
def _render_img(self, width, height, multi_fbo, final_fbo, img_array, top_down=True): """ Render an image of the environment into a frame buffer Produce a numpy RGB array image as output """ if not self.graphics: return # Switch to the default context # This is necessary on Linux nvidia drivers # pyglet.gl._shadow_window.switch_to() self.shadow_window.switch_to() from pyglet import gl # Bind the multisampled frame buffer gl.glEnable(gl.GL_MULTISAMPLE) gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, multi_fbo) gl.glViewport(0, 0, width, height) # Clear the color and depth buffers c0, c1, c2 = self.horizon_color gl.glClearColor(c0, c1, c2, 1.0) gl.glClearDepth(1.0) gl.glClear(gl.GL_COLOR_BUFFER_BIT | gl.GL_DEPTH_BUFFER_BIT) # Set the projection matrix gl.glMatrixMode(gl.GL_PROJECTION) gl.glLoadIdentity() gl.gluPerspective( self.cam_fov_y, width / float(height), 0.04, 100.0 ) # Set modelview matrix # Note: we add a bit of noise to the camera position for data augmentation pos = self.cur_pos angle = self.cur_angle if self.domain_rand: pos = pos + self.randomization_settings['camera_noise'] x, y, z = pos + self.cam_offset dx, dy, dz = self.get_dir_vec(angle) gl.glMatrixMode(gl.GL_MODELVIEW) gl.glLoadIdentity() if self.draw_bbox: y += 0.8 gl.glRotatef(90, 1, 0, 0) elif not top_down: y += self.cam_height gl.glRotatef(self.cam_angle[0], 1, 0, 0) gl.glRotatef(self.cam_angle[1], 0, 1, 0) gl.glRotatef(self.cam_angle[2], 0, 0, 1) gl.glTranslatef(0, 0, self._perturb(gym_duckietown.simulator.CAMERA_FORWARD_DIST)) if top_down: gl.gluLookAt( # Eye position (self.grid_width * self.road_tile_size) / 2, self.top_cam_height, (self.grid_height * self.road_tile_size) / 2, # Target (self.grid_width * self.road_tile_size) / 2, 0, (self.grid_height * self.road_tile_size) / 2, # Up vector 0, 0, -1.0 ) else: gl.gluLookAt( # Eye position x, y, z, # Target x + dx, y + dy, z + dz, # Up vector 0, 1.0, 0.0 ) # Draw the ground quad gl.glDisable(gl.GL_TEXTURE_2D) gl.glColor3f(*self.ground_color) gl.glPushMatrix() gl.glScalef(50, 1, 50) self.ground_vlist.draw(gl.GL_QUADS) gl.glPopMatrix() # Draw the ground/noise triangles self.tri_vlist.draw(gl.GL_TRIANGLES) # Draw the road quads gl.glEnable(gl.GL_TEXTURE_2D) gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR) gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR) # For each grid tile for j in range(self.grid_height): for i in range(self.grid_width): # Get the tile type and angle tile = self._get_tile(i, j) if tile is None: continue # kind = tile['kind'] angle = tile['angle'] color = tile['color'] texture = tile['texture'] gl.glColor3f(*color) gl.glPushMatrix() gl.glTranslatef((i + 0.5) * self.road_tile_size, 0, (j + 0.5) * self.road_tile_size) gl.glRotatef(angle * 90, 0, 1, 0) # Bind the appropriate texture texture.bind() self.road_vlist.draw(gl.GL_QUADS) gl.glPopMatrix() if self.draw_curve and tile['drivable']: # Find curve with largest dotproduct with heading curves = self._get_tile(i, j)['curves'] curve_headings = curves[:, -1, :] - curves[:, 0, :] curve_headings = curve_headings / np.linalg.norm(curve_headings).reshape(1, -1) dirVec = get_dir_vec(angle) dot_prods = np.dot(curve_headings, dirVec) # Current ("closest") curve drawn in Red pts = curves[np.argmax(dot_prods)] bezier_draw(pts, n=20, red=True) pts = self._get_curve(i, j) for idx, pt in enumerate(pts): # Don't draw current curve in blue if idx == np.argmax(dot_prods): continue bezier_draw(pt, n=20) # For each object for idx, obj in enumerate(self.objects): obj.render(self.draw_bbox) # Draw the agent's own bounding box if self.draw_bbox: corners = get_agent_corners(pos, angle) gl.glColor3f(1, 0, 0) gl.glBegin(gl.GL_LINE_LOOP) gl.glVertex3f(corners[0, 0], 0.01, corners[0, 1]) gl.glVertex3f(corners[1, 0], 0.01, corners[1, 1]) gl.glVertex3f(corners[2, 0], 0.01, corners[2, 1]) gl.glVertex3f(corners[3, 0], 0.01, corners[3, 1]) gl.glEnd() if top_down: gl.glPushMatrix() gl.glTranslatef(*self.cur_pos) gl.glScalef(1, 1, 1) gl.glRotatef(self.cur_angle * 180 / np.pi, 0, 1, 0) # glColor3f(*self.color) self.mesh.render() gl.glPopMatrix() # Resolve the multisampled frame buffer into the final frame buffer gl.glBindFramebuffer(gl.GL_READ_FRAMEBUFFER, multi_fbo) gl.glBindFramebuffer(gl.GL_DRAW_FRAMEBUFFER, final_fbo) gl.glBlitFramebuffer( 0, 0, width, height, 0, 0, width, height, gl.GL_COLOR_BUFFER_BIT, gl.GL_LINEAR ) # Copy the frame buffer contents into a numpy array # Note: glReadPixels reads starting from the lower left corner gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, final_fbo) gl.glReadPixels( 0, 0, width, height, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, img_array.ctypes.data_as(POINTER(gl.GLubyte)) ) # Unbind the frame buffer gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0) # Flip the image because OpenGL maps (0,0) to the lower-left corner # Note: this is necessary for gym.wrappers.Monitor to record videos # properly, otherwise they are vertically inverted. img_array = np.ascontiguousarray(np.flip(img_array, axis=0)) return img_array def render_obs(self): """ Render an observation from the point of view of the agent """ observation = self._render_img( self.camera_width, self.camera_height, self.multi_fbo, self.final_fbo, self.img_array, top_down=True ) # self.undistort - for UndistortWrapper if self.distortion and not self.undistort: observation = self.camera_model.distort(observation) return observation
p['offset'] = [-1.0, 0.0] p['width_ratio'] = float(WIDTH)/HEIGHT p['zoom'] = 2.0 # Draw on the framebuffer draw_canvas() p.disable() # Copy the data from the graphics card to system memory import numpy as np import ctypes arr = np.empty((HEIGHT, WIDTH, 3), dtype=np.float32) gl.glFinish() gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0_EXT) gl.glReadPixels(0, 0, WIDTH, HEIGHT, gl.GL_RGB, gl.GL_FLOAT, arr.ctypes.data) # Display using matplotlib (TODO: use opengl to display) import matplotlib.pyplot as plt plt.imshow(arr) plt.show() fbo.unbind() gl.glPopAttrib()
def color_at_point(x, y): a = (gl.GLubyte * 3)(0) gl.glReadPixels(x, y, 1, 1, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, a) return list(a)
def get_pixel_data(x, y, width, height): p = (4 * width * height * gl.GLubyte)() gl.glReadPixels(x, y, width, height, gl.GL_RGBA, gl.GL_UNSIGNED_BYTE, p) return [(r, g, b) for r, g, b, a in partition(p, 4)]