def test_sub_image(self): W = 640 H = 480 buf = bytearray(W*H*4) #the pixel value is derived from the sum of its coordinates (modulo 256) for x in range(W): for y in range(H): #4 bytes per pixel: for i in range(4): buf[y*(W*4) + x*4 + i] = (x+y) % 256 img = ImageWrapper(0, 0, W, H, buf, "RGBX", 24, W*4, planes=ImageWrapper.PACKED, thread_safe=True) #print("image pixels head=%s" % (binascii.hexlify(img.get_pixels()[:128]), )) for x in range(3): SW, SH = 6, 6 sub = img.get_sub_image(x, 0, SW, SH) #print("%s.get_sub_image%s=%s" % (img, (x, 0, SW, SH), sub)) assert sub.get_rowstride()==(SW*4) sub_buf = sub.get_pixels() #print("pixels for %ix%i: %i" % (SW, SH, len(sub_buf))) #print("pixels=%s" % (binascii.hexlify(sub_buf), )) #verify that the pixels are set to 1 / 0: for y in range(SH): v = (x+y)%256 for i in range(4): av = sub_buf[y*(SW*4)+i] try: #python2 (char) av = ord(av) except: #python3 (int already) av = int(av) assert av==v, "expected value %#x for pixel (0, %i) of sub-image %s at (%i, 0), but got %#x" % (v, y, sub, x, av) start = time.time() copy = img.get_sub_image(0, 0, W, H) end = time.time() if SHOW_PERF: print("image wrapper full %ix%i copy speed: %iMB/s" % (W, H, (W*4*H)/(end-start)/1024/1024)) assert copy.get_pixels()==img.get_pixels() total = 0 N = 10 for i in range(N): region = (W//4-N//2+i, H//4-N//2+i, W//2, H//2) start = time.time() copy = img.get_sub_image(*region) end = time.time() total += end-start if SHOW_PERF: print("image wrapper sub image %ix%i copy speed: %iMB/s" % (W//2, H//2, N*(W//2*4*H//2)/total/1024/1024))
def process_draw(self, packet): wid, x, y, width, height, encoding, pixels, _, rowstride, client_options = packet[1:11] #never modify mmap packets if encoding=="mmap": return True #we have a proxy video packet: rgb_format = client_options.get("rgb_format", "") log("proxy draw: client_options=%s", client_options) def send_updated(encoding, compressed_data, client_options): #update the packet with actual encoding data used: packet[6] = encoding packet[7] = compressed_data packet[10] = client_options log("returning %s bytes from %s", len(compressed_data), len(pixels)) return (wid not in self.lost_windows) def passthrough(strip_alpha=True): log("proxy draw: %s passthrough (rowstride: %s vs %s, strip alpha=%s)", rgb_format, rowstride, client_options.get("rowstride", 0), strip_alpha) if strip_alpha: #passthrough as plain RGB: Xindex = rgb_format.upper().find("X") if Xindex>=0 and len(rgb_format)==4: #force clear alpha (which may be garbage): newdata = bytearray(pixels) for i in range(len(pixels)/4): newdata[i*4+Xindex] = chr(255) packet[9] = client_options.get("rowstride", 0) cdata = bytes(newdata) else: cdata = pixels new_client_options = {"rgb_format" : rgb_format} else: #preserve cdata = pixels new_client_options = client_options wrapped = Compressed("%s pixels" % encoding, cdata) #FIXME: we should not assume that rgb32 is supported here... #(we may have to convert to rgb24..) return send_updated("rgb32", wrapped, new_client_options) proxy_video = client_options.get("proxy", False) if PASSTHROUGH and (encoding in ("rgb32", "rgb24") or proxy_video): #we are dealing with rgb data, so we can pass it through: return passthrough(proxy_video) elif not self.video_encoder_types or not client_options or not proxy_video: #ensure we don't try to re-compress the pixel data in the network layer: #(re-add the "compressed" marker that gets lost when we re-assemble packets) packet[7] = Compressed("%s pixels" % encoding, packet[7]) return True #video encoding: find existing encoder ve = self.video_encoders.get(wid) if ve: if ve in self.lost_windows: #we cannot clean the video encoder here, there may be more frames queue up #"lost-window" in encode_loop will take care of it safely return False #we must verify that the encoder is still valid #and scrap it if not (ie: when window is resized) if ve.get_width()!=width or ve.get_height()!=height: log("closing existing video encoder %s because dimensions have changed from %sx%s to %sx%s", ve, ve.get_width(), ve.get_height(), width, height) ve.clean() ve = None elif ve.get_encoding()!=encoding: log("closing existing video encoder %s because encoding has changed from %s to %s", ve.get_encoding(), encoding) ve.clean() ve = None #scaling and depth are proxy-encoder attributes: scaling = client_options.get("scaling", (1, 1)) depth = client_options.get("depth", 24) rowstride = client_options.get("rowstride", rowstride) quality = client_options.get("quality", -1) speed = client_options.get("speed", -1) timestamp = client_options.get("timestamp") image = ImageWrapper(x, y, width, height, pixels, rgb_format, depth, rowstride, planes=ImageWrapper.PACKED) if timestamp is not None: image.set_timestamp(timestamp) #the encoder options are passed through: encoder_options = client_options.get("options", {}) if not ve: #make a new video encoder: spec = self._find_video_encoder(encoding, rgb_format) if spec is None: #no video encoder! from xpra.server.picture_encode import PIL_encode, PIL, warn_encoding_once if PIL is None: warn_encoding_once("no-video-no-PIL", "no video encoder found for rgb format %s, sending as plain RGB!" % rgb_format) return passthrough() log("no video encoder available: sending as jpeg") coding, compressed_data, client_options, _, _, _, _ = PIL_encode("jpeg", image, quality, speed, False) return send_updated(coding, compressed_data, client_options) log("creating new video encoder %s for window %s", spec, wid) ve = spec.make_instance() #dst_formats is specified with first frame only: dst_formats = client_options.get("dst_formats") if dst_formats is not None: #save it in case we timeout the video encoder, #so we can instantiate it again, even from a frame no>1 self.video_encoders_dst_formats = dst_formats else: assert self.video_encoders_dst_formats, "BUG: dst_formats not specified for proxy and we don't have it either" dst_formats = self.video_encoders_dst_formats ve.init_context(width, height, rgb_format, dst_formats, encoding, quality, speed, scaling, {}) self.video_encoders[wid] = ve self.video_encoders_last_used_time[wid] = time.time() #just to make sure this is always set else: if quality>=0: ve.set_encoding_quality(quality) if speed>=0: ve.set_encoding_speed(speed) #actual video compression: log("proxy compression using %s with quality=%s, speed=%s", ve, quality, speed) data, client_options = ve.compress_image(image, encoder_options) self.video_encoders_last_used_time[wid] = time.time() return send_updated(ve.get_encoding(), Compressed(encoding, data), client_options)
def process_draw(self, packet): wid, x, y, width, height, encoding, pixels, _, rowstride, client_options = packet[ 1:11] encoding = bytestostr(encoding) #never modify mmap packets if encoding in ("mmap", "scroll"): return True client_options = typedict(client_options) #we have a proxy video packet: rgb_format = client_options.strget("rgb_format", "") enclog("proxy draw: encoding=%s, client_options=%s", encoding, client_options) def send_updated(encoding, compressed_data, updated_client_options): #update the packet with actual encoding data used: packet[6] = encoding packet[7] = compressed_data packet[10] = updated_client_options enclog("returning %s bytes from %s, options=%s", len(compressed_data), len(pixels), updated_client_options) return wid not in self.lost_windows def passthrough(strip_alpha=True): enclog( "proxy draw: %s passthrough (rowstride: %s vs %s, strip alpha=%s)", rgb_format, rowstride, client_options.intget("rowstride", 0), strip_alpha) if strip_alpha: #passthrough as plain RGB: Xindex = rgb_format.upper().find("X") if Xindex >= 0 and len(rgb_format) == 4: #force clear alpha (which may be garbage): newdata = bytearray(pixels) for i in range(len(pixels) / 4): newdata[i * 4 + Xindex] = chr(255) packet[9] = client_options.intget("rowstride", 0) cdata = bytes(newdata) else: cdata = pixels new_client_options = {"rgb_format": rgb_format} else: #preserve cdata = pixels new_client_options = client_options wrapped = Compressed("%s pixels" % encoding, cdata) #rgb32 is always supported by all clients: return send_updated("rgb32", wrapped, new_client_options) proxy_video = client_options.boolget("proxy", False) if PASSTHROUGH_RGB and (encoding in ("rgb32", "rgb24") or proxy_video): #we are dealing with rgb data, so we can pass it through: return passthrough(proxy_video) if not self.video_encoder_types or not client_options or not proxy_video: #ensure we don't try to re-compress the pixel data in the network layer: #(re-add the "compressed" marker that gets lost when we re-assemble packets) packet[7] = Compressed("%s pixels" % encoding, packet[7]) return True #video encoding: find existing encoder ve = self.video_encoders.get(wid) if ve: if ve in self.lost_windows: #we cannot clean the video encoder here, there may be more frames queue up #"lost-window" in encode_loop will take care of it safely return False #we must verify that the encoder is still valid #and scrap it if not (ie: when window is resized) if ve.get_width() != width or ve.get_height() != height: enclog( "closing existing video encoder %s because dimensions have changed from %sx%s to %sx%s", ve, ve.get_width(), ve.get_height(), width, height) ve.clean() ve = None elif ve.get_encoding() != encoding: enclog( "closing existing video encoder %s because encoding has changed from %s to %s", ve.get_encoding(), encoding) ve.clean() ve = None #scaling and depth are proxy-encoder attributes: scaling = client_options.inttupleget("scaling", (1, 1)) depth = client_options.intget("depth", 24) rowstride = client_options.intget("rowstride", rowstride) quality = client_options.intget("quality", -1) speed = client_options.intget("speed", -1) timestamp = client_options.intget("timestamp") image = ImageWrapper(x, y, width, height, pixels, rgb_format, depth, rowstride, planes=ImageWrapper.PACKED) if timestamp is not None: image.set_timestamp(timestamp) #the encoder options are passed through: encoder_options = client_options.dictget("options", {}) if not ve: #make a new video encoder: spec = self._find_video_encoder(encoding, rgb_format) if spec is None: #no video encoder! enc_pillow = get_codec("enc_pillow") if not enc_pillow: if first_time("no-video-no-PIL-%s" % rgb_format): enclog.warn( "Warning: no video encoder found for rgb format %s", rgb_format) enclog.warn(" sending as plain RGB") return passthrough(True) enclog("no video encoder available: sending as jpeg") coding, compressed_data, client_options = enc_pillow.encode( "jpeg", image, quality, speed, False)[:3] return send_updated(coding, compressed_data, client_options) enclog("creating new video encoder %s for window %s", spec, wid) ve = spec.make_instance() #dst_formats is specified with first frame only: dst_formats = client_options.strtupleget("dst_formats") if dst_formats is not None: #save it in case we timeout the video encoder, #so we can instantiate it again, even from a frame no>1 self.video_encoders_dst_formats = dst_formats else: if not self.video_encoders_dst_formats: raise Exception( "BUG: dst_formats not specified for proxy and we don't have it either" ) dst_formats = self.video_encoders_dst_formats ve.init_context(width, height, rgb_format, dst_formats, encoding, quality, speed, scaling, {}) self.video_encoders[wid] = ve self.video_encoders_last_used_time[wid] = monotonic( ) #just to make sure this is always set #actual video compression: enclog("proxy compression using %s with quality=%s, speed=%s", ve, quality, speed) data, out_options = ve.compress_image(image, quality, speed, encoder_options) #pass through some options if we don't have them from the encoder #(maybe we should also use the "pts" from the real server?) for k in ("timestamp", "rgb_format", "depth", "csc"): if k not in out_options and k in client_options: out_options[k] = client_options[k] self.video_encoders_last_used_time[wid] = monotonic() return send_updated(ve.get_encoding(), Compressed(encoding, data), out_options)
def test_restride(self): #restride of planar is not supported: img = ImageWrapper(0, 0, 1, 1, ["0"*10, "0"*10, "0"*10, "0"*10], "YUV420P", 24, 10, 3, planes=ImageWrapper.PLANAR_4) img.set_planes(ImageWrapper.PLANAR_3) img.clone_pixel_data() assert img.may_restride() is False img = ImageWrapper(0, 0, 1, 1, memoryview(b"0"*4), "BGRA", 24, 4, 4, planes=ImageWrapper.PACKED) img.clone_pixel_data() assert img.may_restride() is False img = ImageWrapper(0, 0, 10, 10, b"0"*40*10, "BGRA", 24, 40, 4, planes=ImageWrapper.PACKED) #restride bigger: img.restride(80) #change more attributes: img.set_timestamp(img.get_timestamp()+1) img.set_pixel_format("RGBA") img.set_palette(()) img.set_pixels("1"*10) assert img.allocate_buffer(0, 1)==0 assert img.get_palette()==() assert img.is_thread_safe() assert img.get_gpu_buffer() is None img.set_rowstride(20)
def get_image(self, x=0, y=0, width=0, height=0): start = time.time() metrics = get_virtualscreenmetrics() if self.metrics is None or self.metrics != metrics: #new metrics, start from scratch: self.metrics = metrics self.clean() dx, dy, dw, dh = metrics if width == 0: width = dw if height == 0: height = dh #clamp rectangle requested to the virtual desktop size: if x < dx: width -= x - dx x = dx if y < dy: height -= y - dy y = dy if width > dw: width = dw if height > dh: height = dh if not self.dc: self.wnd = GetDesktopWindow() self.dc = GetWindowDC(self.wnd) assert self.dc, "failed to get a drawing context from the desktop window %s" % self.wnd self.bit_depth = GetDeviceCaps(self.dc, win32con.BITSPIXEL) self.memdc = CreateCompatibleDC(self.dc) assert self.memdc, "failed to get a compatible drawing context from %s" % self.dc self.bitmap = CreateCompatibleBitmap(self.dc, width, height) assert self.bitmap, "failed to get a compatible bitmap from %s" % self.dc r = SelectObject(self.memdc, self.bitmap) if r == 0: log.error("Error: cannot select bitmap object") return None select_time = time.time() log("get_image up to SelectObject (%s) took %ims", REGION_CONSTS.get(r, r), (select_time - start) * 1000) try: if BitBlt(self.memdc, 0, 0, width, height, self.dc, x, y, win32con.SRCCOPY) == 0: e = ctypes.get_last_error() #rate limit the error message: now = time.time() if now - self.bitblt_err_time > 10: log.error("Error: failed to blit the screen, error %i", e) self.bitblt_err_time = now return None except Exception as e: log("BitBlt error", exc_info=True) log.error("Error: cannot capture screen") log.error(" %s", e) return None bitblt_time = time.time() log("get_image BitBlt took %ims", (bitblt_time - select_time) * 1000) rowstride = roundup(width * self.bit_depth // 8, 2) buf_size = rowstride * height pixels = ctypes.create_string_buffer(b"", buf_size) log("GetBitmapBits(%#x, %#x, %#x)", self.bitmap, buf_size, ctypes.addressof(pixels)) r = GetBitmapBits(self.bitmap, buf_size, ctypes.byref(pixels)) if r == 0: log.error("Error: failed to copy screen bitmap data") return None if r != buf_size: log.warn( "Warning: truncating pixel buffer, got %i bytes but expected %i", r, buf_size) pixels = pixels[:r] log("get_image GetBitmapBits took %ims", (time.time() - bitblt_time) * 1000) assert pixels, "no pixels returned from GetBitmapBits" if self.bit_depth == 32: rgb_format = "BGRX" elif self.bit_depth == 30: rgb_format = "r210" elif self.bit_depth == 24: rgb_format = "BGR" elif self.bit_depth == 16: rgb_format = "BGR565" elif self.bit_depth == 8: rgb_format = "RLE8" else: raise Exception("unsupported bit depth: %s" % self.bit_depth) bpp = self.bit_depth // 8 v = ImageWrapper(x, y, width, height, pixels, rgb_format, self.bit_depth, rowstride, bpp, planes=ImageWrapper.PACKED, thread_safe=True) if self.bit_depth == 8: count = GetSystemPaletteEntries(self.dc, 0, 0, None) log("palette size: %s", count) palette = [] if count > 0: buf = (PALETTEENTRY * count)() r = GetSystemPaletteEntries(self.dc, 0, count, ctypes.byref(buf)) #we expect 16-bit values, so bit-shift them: for p in buf: palette.append( (p.peRed << 8, p.peGreen << 8, p.peBlue << 8)) v.set_palette(palette) log("get_image%s=%s took %ims", (x, y, width, height), v, (time.time() - start) * 1000) return v
def get_image(self, x, y, width, height, logger=None): v = get_rgb_rawdata(self.window, x, y, width, height, logger=logger) if v is None: return None return ImageWrapper(*v)
def get_image(self, x, y, w, h): pixels = "0"*w*4*h return ImageWrapper(x, y, w, h, pixels, "BGRA", 32, w*4, 4, ImageWrapper.PACKED, True, None)
def convert_image_rgb(self, image): start = time.time() iplanes = image.get_planes() width = image.get_width() height = image.get_height() stride = image.get_rowstride() pixels = image.get_pixels() #log("convert_image(%s) planes=%s, pixels=%s, size=%s", image, iplanes, type(pixels), len(pixels)) assert iplanes==ImageWrapper.PACKED, "we only handle packed data as input!" assert image.get_pixel_format()==self.src_format, "invalid source format: %s (expected %s)" % (image.get_pixel_format(), self.src_format) assert width>=self.src_width and height>=self.src_height, "expected source image with dimensions of at least %sx%s but got %sx%s" % (self.src_width, self.src_height, width, height) #adjust work dimensions for subsampling: #(we process N pixels at a time in each dimension) divs = get_subsampling_divs(self.dst_format) wwidth = dimdiv(self.dst_width, max([x_div for x_div, _ in divs])) wheight = dimdiv(self.dst_height, max([y_div for _, y_div in divs])) globalWorkSize, localWorkSize = self.get_work_sizes(wwidth, wheight) #input image: iformat = pyopencl.ImageFormat(self.channel_order, pyopencl.channel_type.UNSIGNED_INT8) shape = (stride//4, self.src_height) log("convert_image() type=%s, input image format=%s, shape=%s, work size: local=%s, global=%s", type(pixels), iformat, shape, localWorkSize, globalWorkSize) idata = pixels if type(idata)==_memoryview: idata = memoryview_to_bytes(idata) if type(idata)==str: #str is not a buffer, so we have to copy the data #alternatively, we could copy it first ourselves using this: #pixels = numpy.fromstring(pixels, dtype=numpy.byte).data #but I think this would be even slower flags = mem_flags.READ_ONLY | mem_flags.COPY_HOST_PTR else: flags = mem_flags.READ_ONLY | mem_flags.USE_HOST_PTR iimage = pyopencl.Image(self.context, flags, iformat, shape=shape, hostbuf=idata) kernelargs = [self.queue, globalWorkSize, localWorkSize, iimage, numpy.int32(self.src_width), numpy.int32(self.src_height), numpy.int32(self.dst_width), numpy.int32(self.dst_height), self.sampler] #calculate plane strides and allocate output buffers: strides = [] out_buffers = [] out_sizes = [] for i in range(3): x_div, y_div = divs[i] p_stride = roundup(self.dst_width // x_div, max(2, localWorkSize[0])) p_height = roundup(self.dst_height // y_div, 2) p_size = p_stride * p_height #log("output buffer for channel %s: stride=%s, height=%s, size=%s", i, p_stride, p_height, p_size) out_buf = pyopencl.Buffer(self.context, mem_flags.WRITE_ONLY, p_size) out_buffers.append(out_buf) kernelargs += [out_buf, numpy.int32(p_stride)] strides.append(p_stride) out_sizes.append(p_size) kstart = time.time() log("convert_image(%s) calling %s%s after %.1fms", image, self.kernel_function_name, tuple(kernelargs), 1000.0*(kstart-start)) self.kernel_function(*kernelargs) self.queue.finish() #free input image: iimage.release() kend = time.time() log("%s took %.1fms", self.kernel_function_name, 1000.0*(kend-kstart)) #read back: pixels = [] for i in range(3): out_array = numpy.empty(out_sizes[i], dtype=numpy.byte) pixels.append(out_array.data) pyopencl.enqueue_read_buffer(self.queue, out_buffers[i], out_array, is_blocking=False) readstart = time.time() log("queue read events took %.1fms (3 planes of size %s, with strides=%s)", 1000.0*(readstart-kend), out_sizes, strides) self.queue.finish() readend = time.time() log("wait for read events took %.1fms", 1000.0*(readend-readstart)) #free output buffers: for out_buf in out_buffers: out_buf.release() return ImageWrapper(0, 0, self.dst_width, self.dst_height, pixels, self.dst_format, 24, strides, planes=ImageWrapper._3_PLANES)
def convert_image_yuv(self, image): start = time.time() iplanes = image.get_planes() width = image.get_width() height = image.get_height() strides = image.get_rowstride() pixels = image.get_pixels() assert iplanes==ImageWrapper._3_PLANES, "we only handle planar data as input!" assert image.get_pixel_format()==self.src_format, "invalid source format: %s (expected %s)" % (image.get_pixel_format(), self.src_format) assert len(strides)==len(pixels)==3, "invalid number of planes or strides (should be 3)" assert width>=self.src_width and height>=self.src_height, "expected source image with dimensions of at least %sx%s but got %sx%s" % (self.src_width, self.src_height, width, height) #adjust work dimensions for subsampling: #(we process N pixels at a time in each dimension) divs = get_subsampling_divs(self.src_format) wwidth = dimdiv(self.dst_width, max(x_div for x_div, _ in divs)) wheight = dimdiv(self.dst_height, max(y_div for _, y_div in divs)) globalWorkSize, localWorkSize = self.get_work_sizes(wwidth, wheight) kernelargs = [self.queue, globalWorkSize, localWorkSize] iformat = pyopencl.ImageFormat(pyopencl.channel_order.R, pyopencl.channel_type.UNSIGNED_INT8) input_images = [] for i in range(3): _, y_div = divs[i] plane = pixels[i] if type(plane)==_memoryview: plane = memoryview_to_bytes(plane) if type(plane)==str: flags = mem_flags.READ_ONLY | mem_flags.COPY_HOST_PTR else: flags = mem_flags.READ_ONLY | mem_flags.USE_HOST_PTR shape = strides[i], self.src_height//y_div iimage = pyopencl.Image(self.context, flags, iformat, shape=shape, hostbuf=plane) input_images.append(iimage) #output image: oformat = pyopencl.ImageFormat(self.channel_order, pyopencl.channel_type.UNORM_INT8) oimage = pyopencl.Image(self.context, mem_flags.WRITE_ONLY, oformat, shape=(self.dst_width, self.dst_height)) kernelargs += input_images + [numpy.int32(self.src_width), numpy.int32(self.src_height), numpy.int32(self.dst_width), numpy.int32(self.dst_height), self.sampler, oimage] kstart = time.time() log("convert_image(%s) calling %s%s after upload took %.1fms", image, self.kernel_function_name, tuple(kernelargs), 1000.0*(kstart-start)) self.kernel_function(*kernelargs) self.queue.finish() #free input images: for iimage in input_images: iimage.release() kend = time.time() log("%s took %.1fms", self.kernel_function, 1000.0*(kend-kstart)) out_array = numpy.empty(self.dst_width*self.dst_height*4, dtype=numpy.byte) pyopencl.enqueue_read_image(self.queue, oimage, (0, 0), (self.dst_width, self.dst_height), out_array) self.queue.finish() log("readback using %s took %.1fms", CHANNEL_ORDER_TO_STR.get(self.channel_order), 1000.0*(time.time()-kend)) self.time += time.time()-start self.frames += 1 return ImageWrapper(0, 0, self.dst_width, self.dst_height, out_array.data, self.dst_format, 24, self.dst_width*4, planes=ImageWrapper.PACKED)