def test_send_message_deflate_frame_bfinal(self): extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION) request = _create_request_from_rawdata( '', deflate_frame_request=extension) self.assertEquals(1, len(request.ws_extension_processors)) deflate_frame_processor = request.ws_extension_processors[0] deflate_frame_processor.set_bfinal(True) msgutil.send_message(request, 'Hello') msgutil.send_message(request, 'World') expected = '' compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) compressed_hello = compress.compress('Hello') compressed_hello += compress.flush(zlib.Z_FINISH) compressed_hello = compressed_hello + chr(0) expected += '\xc1%c' % len(compressed_hello) expected += compressed_hello compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) compressed_world = compress.compress('World') compressed_world += compress.flush(zlib.Z_FINISH) compressed_world = compressed_world + chr(0) expected += '\xc1%c' % len(compressed_world) expected += compressed_world self.assertEqual(expected, request.connection.written_data())
def test_receive_message_deflate_stream(self): compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) data = compress.compress('\x81\x85' + _mask_hybi('Hello')) data += compress.flush(zlib.Z_SYNC_FLUSH) data += compress.compress('\x81\x89' + _mask_hybi('WebSocket')) data += compress.flush(zlib.Z_FINISH) compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) data += compress.compress('\x81\x85' + _mask_hybi('World')) data += compress.flush(zlib.Z_SYNC_FLUSH) # Close frame data += compress.compress( '\x88\x8a' + _mask_hybi(struct.pack('!H', 1000) + 'Good bye')) data += compress.flush(zlib.Z_SYNC_FLUSH) request = _create_request_from_rawdata(data, deflate_stream=True) self.assertEqual('Hello', msgutil.receive_message(request)) self.assertEqual('WebSocket', msgutil.receive_message(request)) self.assertEqual('World', msgutil.receive_message(request)) self.assertFalse(request.drain_received_data_called) self.assertEqual(None, msgutil.receive_message(request)) self.assertTrue(request.drain_received_data_called)
def test_send_message_permessage_compress_deflate_fragmented_bfinal(self): extension = common.ExtensionParameter( common.PERMESSAGE_COMPRESSION_EXTENSION) extension.add_parameter('method', 'deflate') request = _create_request_from_rawdata( '', permessage_compression_request=extension) self.assertEquals(1, len(request.ws_extension_processors)) compression_processor = ( request.ws_extension_processors[0].get_compression_processor()) compression_processor.set_bfinal(True) msgutil.send_message(request, 'Hello', end=False) msgutil.send_message(request, 'World', end=True) expected = '' compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) compressed_hello = compress.compress('Hello') compressed_hello += compress.flush(zlib.Z_FINISH) compressed_hello = compressed_hello + chr(0) expected += '\x41%c' % len(compressed_hello) expected += compressed_hello compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) compressed_world = compress.compress('World') compressed_world += compress.flush(zlib.Z_FINISH) compressed_world = compressed_world + chr(0) expected += '\x80%c' % len(compressed_world) expected += compressed_world self.assertEqual(expected, request.connection.written_data())
def setUp(self): super(ZlibTest, self).setUp() self.text = b""" Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Maecenas porttitor congue massa. Fusce posuere, magna sed pulvinar ultricies, purus lectus malesuada libero, sit amet commodo magna eros quis urna. Nunc viverra imperdiet enim. Fusce est. Vivamus a tellus. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Proin pharetra nonummy pede. Mauris et orci. Aenean nec lorem. In porttitor. Donec laoreet nonummy augue. Suspendisse dui purus, scelerisque at, vulputate vitae, pretium mattis, nunc. Mauris eget neque at sem venenatis eleifend. Ut nonummy. Fusce aliquet pede non pede. Suspendisse dapibus lorem pellentesque magna. Integer nulla. Donec blandit feugiat ligula. Donec hendrerit, felis et imperdiet euismod, purus ipsum pretium metus, in lacinia nulla nisl eget sapien. Donec ut est in lectus consequat consequat. Etiam eget dui. Aliquam erat volutpat. Sed at lorem in nunc porta tristique. Proin nec augue. Quisque aliquam tempor magna. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Nunc ac magna. Maecenas odio dolor, vulputate vel, auctor ac, accumsan id, felis. Pellentesque cursus sagittis felis. Pellentesque porttitor, velit lacinia egestas auctor, diam eros tempus arcu, nec vulputate augue magna vel risus. Cras non magna vel ante adipiscing rhoncus. Vivamus a mi. Morbi neque. Aliquam erat volutpat. Integer ultrices lobortis eros. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Proin semper, ante vitae sollicitudin posuere, metus quam iaculis nibh, vitae scelerisque nunc massa eget pede. Sed velit urna, interdum vel, ultricies vel, faucibus at, quam. Donec elit est, consectetuer eget, consequat quis, tempus quis, wisi. In in nunc. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos hymenaeos. Donec ullamcorper fringilla eros. Fusce in sapien eu purus dapibus commodo. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Cras faucibus condimentum odio. Sed ac ligula. Aliquam at eros. Etiam at ligula et tellus ullamcorper ultrices. In fermentum, lorem non cursus porttitor, diam urna accumsan lacus, sed interdum wisi nibh nec nisl. Ut tincidunt volutpat urna. Mauris eleifend nulla eget mauris. Sed cursus quam id felis. Curabitur posuere quam vel nibh. Cras dapibus dapibus nisl. Vestibulum quis dolor a felis congue vehicula. Maecenas pede purus, tristique ac, tempus eget, egestas quis, mauris. Curabitur non eros. Nullam hendrerit bibendum justo. Fusce iaculis, est quis lacinia pretium, pede metus molestie lacus, at gravida wisi ante at libero. """ deflate_compress = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS) zlib_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS) self.deflate_data = deflate_compress.compress(self.text) + deflate_compress.flush() self.zlib_data = zlib_compress.compress(self.text) + zlib_compress.flush() self.gzip_data = create_gzip(self.text)
def test_receive_message_deflate_frame_comp_bit(self): compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) data = '' compressed_hello = compress.compress('Hello') compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH) compressed_hello = compressed_hello[:-4] data += '\xc1%c' % (len(compressed_hello) | 0x80) data += _mask_hybi(compressed_hello) data += '\x81\x85' + _mask_hybi('Hello') compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) compressed_2nd_hello = compress.compress('Hello') compressed_2nd_hello += compress.flush(zlib.Z_SYNC_FLUSH) compressed_2nd_hello = compressed_2nd_hello[:-4] data += '\xc1%c' % (len(compressed_2nd_hello) | 0x80) data += _mask_hybi(compressed_2nd_hello) extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION) request = _create_request_from_rawdata( data, deflate_frame_request=extension) for i in xrange(3): self.assertEqual('Hello', msgutil.receive_message(request))
async def _send_frame(self, message: bytes, opcode: int, compress: Optional[int]=None) -> None: """Send a frame over the websocket with message as its payload.""" if self._closing: ws_logger.warning('websocket connection is closing.') rsv = 0 # Only compress larger packets (disabled) # Does small packet needs to be compressed? # if self.compress and opcode < 8 and len(message) > 124: if (compress or self.compress) and opcode < 8: if compress: # Do not set self._compress if compressing is for this frame compressobj = zlib.compressobj(wbits=-compress) else: # self.compress if not self._compressobj: self._compressobj = zlib.compressobj(wbits=-self.compress) compressobj = self._compressobj message = compressobj.compress(message) message = message + compressobj.flush( zlib.Z_FULL_FLUSH if self.notakeover else zlib.Z_SYNC_FLUSH) if message.endswith(_WS_DEFLATE_TRAILING): message = message[:-4] rsv = rsv | 0x40 msg_length = len(message) use_mask = self.use_mask if use_mask: mask_bit = 0x80 else: mask_bit = 0 if msg_length < 126: header = PACK_LEN1(0x80 | rsv | opcode, msg_length | mask_bit) elif msg_length < (1 << 16): header = PACK_LEN2(0x80 | rsv | opcode, 126 | mask_bit, msg_length) else: header = PACK_LEN3(0x80 | rsv | opcode, 127 | mask_bit, msg_length) if use_mask: mask = self.randrange(0, 0xffffffff) mask = mask.to_bytes(4, 'big') message = bytearray(message) _websocket_mask(mask, message) self.transport.write(header + mask + message) self._output_size += len(header) + len(mask) + len(message) else: if len(message) > MSG_SIZE: self.transport.write(header) self.transport.write(message) else: self.transport.write(header + message) self._output_size += len(header) + len(message) if self._output_size > self._limit: self._output_size = 0 await self.protocol._drain_helper()
def __init__(self, level = 6): """ Constructor __init__(GzipCompressor) :since: v1.0.0 """ self.compressor = None """ Deflate compressor instance """ self.crc32 = None """ CRC32 from previous run """ self.header = None """ Gzip header """ self.size = None """ Total size of compressed data """ # Use the zlib magic +16 to generate the GZip header and trailer on flush() if supported try: self.compressor = compressobj(level, wbits = 16 + MAX_WBITS) except TypeError: self.compressor = compressobj(level) if (level == 9): deflate_flag = 2 elif (level == 1): deflate_flag = 4 else: deflate_flag = 0 self.header = pack("<8s2B", Binary.bytes("\x1f\x8b" + ("\x00" if (level == 0) else "\x08") + "\x00\x00\x00\x00\x00"), deflate_flag, 255) self.size = 0
def test_receive_message_permessage_deflate_compression(self): compress = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) data = "" compressed_hello = compress.compress("HelloWebSocket") compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH) compressed_hello = compressed_hello[:-4] split_position = len(compressed_hello) / 2 data += "\x41%c" % (split_position | 0x80) data += _mask_hybi(compressed_hello[:split_position]) data += "\x80%c" % ((len(compressed_hello) - split_position) | 0x80) data += _mask_hybi(compressed_hello[split_position:]) compress = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) compressed_world = compress.compress("World") compressed_world += compress.flush(zlib.Z_SYNC_FLUSH) compressed_world = compressed_world[:-4] data += "\xc1%c" % (len(compressed_world) | 0x80) data += _mask_hybi(compressed_world) # Close frame data += "\x88\x8a" + _mask_hybi(struct.pack("!H", 1000) + "Good bye") extension = common.ExtensionParameter(common.PERMESSAGE_COMPRESSION_EXTENSION) extension.add_parameter("method", "deflate") request = _create_request_from_rawdata(data, permessage_compression_request=extension) self.assertEqual("HelloWebSocket", msgutil.receive_message(request)) self.assertEqual("World", msgutil.receive_message(request)) self.assertEqual(None, msgutil.receive_message(request))
def test_receive_message_deflate_frame(self): compress = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) data = "" compressed_hello = compress.compress("Hello") compressed_hello += compress.flush(zlib.Z_SYNC_FLUSH) compressed_hello = compressed_hello[:-4] data += "\xc1%c" % (len(compressed_hello) | 0x80) data += _mask_hybi(compressed_hello) compressed_websocket = compress.compress("WebSocket") compressed_websocket += compress.flush(zlib.Z_FINISH) compressed_websocket += "\x00" data += "\xc1%c" % (len(compressed_websocket) | 0x80) data += _mask_hybi(compressed_websocket) compress = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS) compressed_world = compress.compress("World") compressed_world += compress.flush(zlib.Z_SYNC_FLUSH) compressed_world = compressed_world[:-4] data += "\xc1%c" % (len(compressed_world) | 0x80) data += _mask_hybi(compressed_world) # Close frame data += "\x88\x8a" + _mask_hybi(struct.pack("!H", 1000) + "Good bye") extension = common.ExtensionParameter(common.DEFLATE_FRAME_EXTENSION) request = _create_request_from_rawdata(data, deflate_frame_request=extension) self.assertEqual("Hello", msgutil.receive_message(request)) self.assertEqual("WebSocket", msgutil.receive_message(request)) self.assertEqual("World", msgutil.receive_message(request)) self.assertEqual(None, msgutil.receive_message(request))
def create_CompressedDataBody(alg, d): """Create a CompressedDataBody instance. :Parameters: - `alg`: integer compressed algorithm constant - `d`: string data to compress :Returns: `CompressedDataBody` instance """ if COMP_ZIP == alg: # ..from zipfile.py source cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) data = cmpr.compress(d) ex = cmpr.flush() if ex: data = data + ex elif COMP_ZLIB == alg: cmpr = zlib.compressobj() data = cmpr.compress(d) ex = cmpr.flush() if ex: data = data + ex elif COMP_UNCOMPRESSED == alg: data = d else: raise NotImplementedError("Unsupported compression algorithm->(%s)" % alg) return CompressedDataBody(''.join([chr(alg), data]))
def start_compress_message(self): # compressobj([level[, method[, wbits[, mem_level[, strategy]]]]]) # http://bugs.python.org/issue19278 # http://hg.python.org/cpython/rev/c54c8e71b79a if self._is_server: if self._compressor is None or self.server_no_context_takeover: self._compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -self.server_max_window_bits, self.mem_level) else: if self._compressor is None or self.client_no_context_takeover: self._compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -self.client_max_window_bits, self.mem_level)
def _initialize_compressor(self): if self._compression_type == CompressionTypes.BZIP2: self._compressor = bz2.BZ2Compressor() elif self._compression_type == CompressionTypes.DEFLATE: self._compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED) else: assert self._compression_type == CompressionTypes.GZIP self._compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, self._gzip_mask)
def startCompressMessage(self): if self._isServer: if self._compressor is None or self.s2c_no_context_takeover: self._compressor = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -self.s2c_max_window_bits ) else: if self._compressor is None or self.c2s_no_context_takeover: self._compressor = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -self.c2s_max_window_bits )
def test_multi_decoding_gzip_gzip(self): compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(b'foo') data += compress.flush() compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) data = compress.compress(data) data += compress.flush() fp = BytesIO(data) r = HTTPResponse(fp, headers={'content-encoding': 'gzip, gzip'}) assert r.data == b'foo'
def deflate_encoder(level=None): if level is None: obj = zlib.compressobj() else: obj = zlib.compressobj(level) def enc(data, final): ret = obj.compress(data) if final: ret += obj.flush() return ret return enc
def writeblock(mapdir, px,py,pz, block): sectordir = mapdir + get_sector_dir(px, pz); try: os.makedirs(sectordir) except OSError: pass path = sectordir+"/"+to4h(py) print("writing block file "+path) f = open(sectordir+"/"+to4h(py), "wb") if f == None: return # version version = 17 f.write(struct.pack('B', version)) # flags # 0x01=is_undg, 0x02=dn_diff, 0x04=lighting_expired flags = 0 + 0x02 + 0x04 f.write(struct.pack('B', flags)) # data c_obj = zlib.compressobj() c_obj.compress(block.serialize_data()) f.write(struct.pack('BB', 0x78, 0x9c)) # zlib magic number f.write(c_obj.flush()) # node metadata c_obj = zlib.compressobj() c_obj.compress(block.serialize_nodemeta()) f.write(struct.pack('BB', 0x78, 0x9c)) # zlib magic number f.write(c_obj.flush()) # mapblockobject count f.write(ser_u16(0)) # static objects f.write(block.serialize_staticobj()) # timestamp f.write(ser_u32(0xffffffff)) f.close()
def _add_packet_to_queue(self, packet, start_send_cb=None, end_send_cb=None): packets = self.encode(packet) if not self.raw_packets: assert len(packets)==1 try: self._write_lock.acquire() counter = 0 for index,compress,data in packets: if self.raw_packets: if compress and self._compression_level>0: level = self._compression_level if self._compressor is None: self._compressor = zlib.compressobj(level) data = self._compressor.compress(data)+self._compressor.flush(zlib.Z_SYNC_FLUSH) else: level = 0 l = len(data) #'p' + protocol-version + compression_level + packet_index + packet_size header = struct.pack('!cBBBL', "P", 0, level, index, l) else: assert index==0 l = len(data) header = ("PS%014d" % l).encode('latin1') scb, ecb = None, None #fire the start_send_callback just before the first packet is processed: if counter==0: scb = start_send_cb #fire the end_send callback when the last packet (index==0) makes it out: if index==0: ecb = end_send_cb if l<4096 and sys.version<'3': #send size and data together (low copy overhead): self._queue_write(header+data, scb, ecb, True) else: self._queue_write(header) self._queue_write(data, scb, ecb, True) counter += 1 finally: if packet[0]=="set_deflate": level = packet[1] log("set_deflate packet, changing compressor from %s to level=%s", self._compression_level, level) if self._compression_level!=level or self._compressor is None: if level>0: self._compressor = zlib.compressobj(level) else: self._compressor = None self.output_packetcount += 1 self._write_lock.release()
def png_write(width, height, image): def _chunk(name, data=""): length = struct.pack(">I", len(data)) crc = struct.pack(">i", crc32(name + data)) return length + name + data + crc _data = "" _data = struct.pack(">8B", 137, 80, 78, 71, 13, 10, 26, 10) _data += _chunk("IHDR", struct.pack(">2I5B", width, height, 8, 2, 0, 0, 0)) image_data = "" compressor = zlib.compressobj() for y in xrange(1, height+1): row = "" for x in xrange(1, width+1): p = x11.XGetPixel(image, x, y) p = struct.pack(">I", p)[1:] row += p image_data += compressor.compress(row) image_data += compressor.flush() _data += _chunk("IDAT", image_data) _data += _chunk("IEND") return _data
def testZLibFlushRecord(self): fn = self._WriteRecordsToFile([b"small record"], "small_record") with open(fn, "rb") as h: buff = h.read() # creating more blocks and trailing blocks shouldn't break reads compressor = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS) output = b"" for c in buff: if isinstance(c, int): c = six.int2byte(c) output += compressor.compress(c) output += compressor.flush(zlib.Z_FULL_FLUSH) output += compressor.flush(zlib.Z_FULL_FLUSH) output += compressor.flush(zlib.Z_FULL_FLUSH) output += compressor.flush(zlib.Z_FINISH) # overwrite the original file with the compressed data with open(fn, "wb") as h: h.write(output) with self.test_session() as sess: options = tf_record.TFRecordOptions( compression_type=TFRecordCompressionType.ZLIB) reader = io_ops.TFRecordReader(name="test_reader", options=options) queue = data_flow_ops.FIFOQueue(1, [dtypes.string], shapes=()) key, value = reader.read(queue) queue.enqueue(fn).run() queue.close().run() k, v = sess.run([key, value]) self.assertTrue(compat.as_text(k).startswith("%s:" % fn)) self.assertAllEqual(b"small record", v)
def __init__(self, conn): asyncore.dispatcher_with_send.__init__(self, conn) self.ssled = False self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True) self.consumed_ace = False self.data = "" self.binary_mode = False self.decompressor = zlib.decompressobj() self.compressor = zlib.compressobj() self.unzipped_input = "" self.unzipped_output_buffer = "" self.output_buffer = "" self.speech = dict() self.pong = 1 self.ping = 0 self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure) self.gotGoogleAnswer = False self.googleData = None self.lastRequestId = None self.dictation = None self.dbConnection = db.getConnection() self.assistant = None self.sendLock = threading.Lock() self.current_running_plugin = None self.current_location = None self.plugin_lastAceId = None self.logger = logging.getLogger("logger")
def __init__(self, filename = None, mode = None, compresslevel = 9, fileobj = None, mtime = None): if mode and 'b' not in mode: mode += 'b' if fileobj is None: fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb') if filename is None: if hasattr(fileobj, 'name'): filename = fileobj.name else: filename = '' if mode is None: if hasattr(fileobj, 'mode'): mode = fileobj.mode else: mode = 'rb' if mode[0:1] == 'r': self.mode = READ self._new_member = True self.extrabuf = '' self.extrasize = 0 self.extrastart = 0 self.name = filename self.min_readsize = 100 elif mode[0:1] == 'w' or mode[0:1] == 'a': self.mode = WRITE self._init_write(filename) self.compress = zlib.compressobj(compresslevel, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) else: raise IOError, 'Mode ' + mode + ' not supported' self.fileobj = fileobj self.offset = 0 self.mtime = mtime if self.mode == WRITE: self._write_gzip_header()
def start(self, listener): """ This method is used to make the actual connection to the server and prepare for audio streaming. listener is a HoundListener (or derived class) object """ self.audioFinished = False self.buffer = '' self.HoundRequestInfo['RequestID'] = str(uuid.uuid4()) self.conn = HTPConnection(self.hostname, self.port) htpMsg = self.conn.ReadMessage() challengeMsg = json.loads(htpMsg.data) if not challengeMsg.has_key('status') or challengeMsg['status'] <> 'ok': raise Exception("Error reading challenge message") nonce = challengeMsg['nonce'] signature = self._authenticate(nonce) ## Startup the listening thread (above) self.callbackTID = threading.Thread(target = self._callback, args = (listener,)) self.callbackTID.start() self.conn.SendMessage(HTPMessage(HTPMessage.HTP_TYPE_JSON, json.dumps({'access_id': self.clientID, 'signature': signature, 'version': '1.1'}))) HoundRequestInfo = json.dumps(self.HoundRequestInfo) gzip_compressor = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) msg = gzip_compressor.compress(HoundRequestInfo) + gzip_compressor.flush() self.conn.SendMessage(HTPMessage(HTPMessage.HTP_TYPE_BINARY, msg)) spxHeader = pySHSpeex.Init() self.conn.SendMessage(HTPMessage(HTPMessage.HTP_TYPE_BINARY, spxHeader))
def write_bytearray(self, filename, rows): stream = BinaryStream(big_endian=True) # http://www.w3.org/TR/PNG/#5PNG-file-signature stream.write(PyPngWriter._SIGNATURE) # http://www.w3.org/TR/PNG/#11IHDR PyPngWriter._write_chunk(stream, b'IHDR', struct.pack('!II B B BBB', self.width, self.height, 8, 6, 0, 0, 0)) # http://www.w3.org/TR/PNG/#11IDAT compressor = zlib.compressobj() data = bytearray() for row in rows: data.append(0) data.extend(row) if len(data) > self.chunk_limit: compressed = compressor.compress(bytes(data)) if len(compressed): PyPngWriter._write_chunk(stream, b'IDAT', compressed) data = bytearray() if len(data): compressed = compressor.compress(bytes(data)) else: compressed = bytes() flushed = compressor.flush() if len(compressed) or len(flushed): PyPngWriter._write_chunk(stream, b'IDAT', compressed + flushed) # http://www.w3.org/TR/PNG/#11IEND PyPngWriter._write_chunk(stream, b'IEND') stream.write_file(filename)
def compress(text): """ generate a possibly-compressed representation of text """ if not text: return ("", text) l = len(text) bin = None if l < 44: pass elif l > 1000000: # zlib makes an internal copy, thus doubling memory usage for # large files, so lets do this in pieces z = zlib.compressobj() p = [] pos = 0 while pos < l: pos2 = pos + 2**20 p.append(z.compress(text[pos:pos2])) pos = pos2 p.append(z.flush()) if sum(map(len, p)) < l: bin = "".join(p) else: bin = _compress(text) if bin is None or len(bin) > l: if text[0] == '\0': return ("", text) return ('u', text) return ("", bin)
def compress(body, compress_level): """Compress 'body' at the given compress_level.""" import zlib # See http://www.gzip.org/zlib/rfc-gzip.html yield ntob("\x1f\x8b") # ID1 and ID2: gzip marker yield ntob("\x08") # CM: compression method yield ntob("\x00") # FLG: none set # MTIME: 4 bytes yield struct.pack("<L", int(time.time()) & int("FFFFFFFF", 16)) yield ntob("\x02") # XFL: max compression, slowest algo yield ntob("\xff") # OS: unknown crc = zlib.crc32(ntob("")) size = 0 zobj = zlib.compressobj(compress_level, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) for line in body: size += len(line) crc = zlib.crc32(line, crc) yield zobj.compress(line) yield zobj.flush() # CRC32: 4 bytes yield struct.pack("<L", crc & int("FFFFFFFF", 16)) # ISIZE: 4 bytes yield struct.pack("<L", size & int("FFFFFFFF", 16))
def __init__(self, name, uncompressed_data_to_preload, level=None): """ Initializes the compressor. Usually, you'll want to use a helper method that reads a description blurb based on PLCD_SCHEME_JSON_SCHEMA. :param name: Name of the compressor, or None to disable version tags :type name: str or None :param uncompressed_data_to_preload: data you wish to seed the compressor with :type uncompressed_data_to_preload: str :param level: zlib compression level, or None to use the default :type level: int or None """ self.name = str(name) if name is not None else None self.name_tag = str(name) + ':' if name is not None else '' self.level = level _compressobj = zlib.compressobj(*([level] if level else [])) _decompressobj = zlib.decompressobj() assert isinstance(uncompressed_data_to_preload, str) self.preloaded_compressed = ( _compressobj.compress(uncompressed_data_to_preload) + _compressobj.flush(zlib.Z_SYNC_FLUSH) ) _decompressobj.decompress(self.preloaded_compressed) # To prevent bugs, we don't let anyone actually access # _new_compressor / _new_decompressor self._new_decompressor = _decompressobj.copy self._new_compressor = _compressobj.copy
def _write_block(self, block): #print("Saving %i bytes" % len(block)) start_offset = self._handle.tell() assert len(block) <= 65536 #Giving a negative window bits means no gzip/zlib headers, -15 used in samtools c = zlib.compressobj(self.compresslevel, zlib.DEFLATED, -15, zlib.DEF_MEM_LEVEL, 0) compressed = c.compress(block) + c.flush() del c assert len(compressed) < 65536, "TODO - Didn't compress enough, try less data in this block" crc = zlib.crc32(block) #Should cope with a mix of Python platforms... if crc < 0: crc = struct.pack("<i", crc) else: crc = struct.pack("<I", crc) bsize = struct.pack("<H", len(compressed)+25) # includes -1 crc = struct.pack("<I", zlib.crc32(block) & 0xffffffffL) uncompressed_length = struct.pack("<I", len(block)) #Fixed 16 bytes, # gzip magic bytes (4) mod time (4), # gzip flag (1), os (1), extra length which is six (2), # sub field which is BC (2), sub field length of two (2), #Variable data, #2 bytes: block length as BC sub field (2) #X bytes: the data #8 bytes: crc (4), uncompressed data length (4) data = _bgzf_header + bsize + compressed + crc + uncompressed_length self._handle.write(data)
def dump_inventory(self): self.info(bold('dumping object inventory... '), nonl=True) f = open(path.join(self.outdir, INVENTORY_FILENAME), 'wb') try: f.write((u'# Sphinx inventory version 2\n' u'# Project: %s\n' u'# Version: %s\n' u'# The remainder of this file is compressed using zlib.\n' % (self.config.project, self.config.version)).encode('utf-8')) compressor = zlib.compressobj(9) for domainname, domain in sorted(self.env.domains.items()): for name, dispname, type, docname, anchor, prio in \ sorted(domain.get_objects()): if anchor.endswith(name): # this can shorten the inventory by as much as 25% anchor = anchor[:-len(name)] + '$' uri = self.get_target_uri(docname) if anchor: uri += '#' + anchor if dispname == name: dispname = u'-' f.write(compressor.compress( (u'%s %s:%s %s %s %s\n' % (name, domainname, type, prio, uri, dispname)).encode('utf-8'))) f.write(compressor.flush()) finally: f.close() self.info('done')
def compress(contents, request): request.setHeader("content-encoding", "gzip") compress = zlib.compressobj(6, zlib.DEFLATED, zlib.MAX_WBITS + 16, zlib.DEF_MEM_LEVEL,0) contents = compress.compress(contents) contents += compress.flush() return contents
def writebuffer(self, buffer, zinfo_or_arcname, compress_type=None): if not isinstance(zinfo_or_arcname, zipfile.ZipInfo): zinfo = zipfile.ZipInfo( filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) zinfo.compress_type = self.compression zinfo.external_attr = 0600 << 16 else: zinfo = zinfo_or_arcname if compress_type is not None: zinfo.compress_type = compress_type zinfo.file_size = file_size = 0 zinfo.flag_bits = 0x00 zinfo.header_offset = self.fp.tell() self._writecheck(zinfo) self._didModify = True zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 self.fp.write(zinfo.FileHeader()) if zinfo.compress_type == zipfile.ZIP_DEFLATED: cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) else: cmpr = None while True: buf = buffer.read(1024 * 8) if not buf: break file_size = file_size + len(buf) CRC = binascii.crc32(buf, CRC) & 0xffffffff if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) self.fp.write(buf) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) self.fp.write(buf) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size # Seek backwards and write CRC and file sizes position = self.fp.tell() self.fp.seek(zinfo.header_offset + 14, 0) self.fp.write(struct.pack("<LLL", zinfo.CRC, zinfo.compress_size, zinfo.file_size)) self.fp.seek(position, 0) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo
local = [] central = [] local_size = 0 central_size = 0 for path, data in files: is_directory = path.endswith(b"/") version = 0x14 attributes = 0x10 if is_directory else 0x20 crc = zlib.crc32(data) if use_compression and not is_directory: compression = 8 obj = zlib.compressobj(wbits=-zlib.MAX_WBITS) compressed_data = obj.compress(data) compressed_data += obj.flush() else: compression = 0 compressed_data = data print("compressed_data:", len(compressed_data)) local_header = b"PK\3\4" local_header += struct.pack("<HHHHHIIII", version, 0, compression, 0, 0, crc, len(compressed_data), len(data), len(path)) #print(" ".join("%2x"%x for x in local_header + path + compressed_data))
def deflate_headerless(data): wbits = -15 # disable header/trailer by negating windowBits zobj = zlib.compressobj(wbits=wbits) compressed = zobj.compress(data) + zobj.flush() return compressed
def deflate_compress(bs): do = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS) return do.compress(bs) + do.flush()
def write(self, filename, arcname=None, compress_type=None): if not self.fp: raise RuntimeError('Attempt to write to ZIP archive that was already closed') st = os.stat(filename) isdir = stat.S_ISDIR(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] if arcname is None: arcname = filename arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) while arcname[0] in (os.sep, os.altsep): arcname = arcname[1:] if isdir: arcname += '/' zinfo = ZipInfo(arcname, date_time) zinfo.external_attr = (st[0] & 65535) << 16L if compress_type is None: zinfo.compress_type = self.compression else: zinfo.compress_type = compress_type zinfo.file_size = st.st_size zinfo.flag_bits = 0 zinfo.header_offset = self.fp.tell() self._writecheck(zinfo) self._didModify = True if isdir: zinfo.file_size = 0 zinfo.compress_size = 0 zinfo.CRC = 0 self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) return with open(filename, 'rb') as (fp): zinfo.CRC = CRC = 0 zinfo.compress_size = compress_size = 0 zip64 = self._allowZip64 and zinfo.file_size * 1.05 > ZIP64_LIMIT self.fp.write(zinfo.FileHeader(zip64)) if zinfo.compress_type == ZIP_DEFLATED: cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) else: cmpr = None file_size = 0 while 1: buf = fp.read(8192) if not buf: break file_size = file_size + len(buf) CRC = crc32(buf, CRC) & 4294967295L if cmpr: buf = cmpr.compress(buf) compress_size = compress_size + len(buf) self.fp.write(buf) if cmpr: buf = cmpr.flush() compress_size = compress_size + len(buf) self.fp.write(buf) zinfo.compress_size = compress_size else: zinfo.compress_size = file_size zinfo.CRC = CRC zinfo.file_size = file_size if not zip64 and self._allowZip64: if file_size > ZIP64_LIMIT: raise RuntimeError('File size has increased during compressing') if compress_size > ZIP64_LIMIT: raise RuntimeError('Compressed size larger than uncompressed size') position = self.fp.tell() self.fp.seek(zinfo.header_offset, 0) self.fp.write(zinfo.FileHeader(zip64)) self.fp.seek(position, 0) self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo return
def __init__(self, output, level=1): self._compressor = zlib.compressobj(level) self._output = output
def __init__(self, stream): self.stream = stream self.C = zlib.compressobj()
def gzip_encode(content): gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) data = gzip_compress.compress(content) + gzip_compress.flush() return data
def __init__(self, errors='strict'): raise errors == 'strict' or AssertionError self.errors = errors self.compressobj = zlib.compressobj()
def add(self, entry): """ Add an ENTRY to the CArchive. ENTRY must have: entry[0] is name (under which it will be saved). entry[1] is fullpathname of the file. entry[2] is a flag for it's storage format (0==uncompressed, 1==compressed) entry[3] is the entry's type code. Version 5: If the type code is 'o': entry[0] is the runtime option eg: v (meaning verbose imports) u (menaing unbuffered) W arg (warning option arg) s (meaning do site.py processing. """ (nm, pathnm, flag, typcd) = entry[:4] # FIXME Could we make the version 5 the default one? # Version 5 - allow type 'o' = runtime option. code_data = None fh = None try: if typcd in ('o', 'd'): ulen = 0 flag = 0 elif typcd == 's': # If it's a source code file, compile it to a code object and marshall # the object so it can be unmarshalled by the bootloader. code = get_code_object(nm, pathnm) code = strip_paths_in_code(code) code_data = marshal.dumps(code) ulen = len(code_data) else: fh = open(pathnm, 'rb') ulen = os.fstat(fh.fileno()).st_size except IOError: print("Cannot find ('%s', '%s', %s, '%s')" % (nm, pathnm, flag, typcd)) raise where = self.lib.tell() assert flag in range(3) if not fh and not code_data: # no need to write anything pass elif flag == 1: comprobj = zlib.compressobj(self.LEVEL) if code_data is not None: self.lib.write(comprobj.compress(code_data)) else: assert fh while 1: buf = fh.read(16 * 1024) if not buf: break self.lib.write(comprobj.compress(buf)) self.lib.write(comprobj.flush()) else: if code_data is not None: self.lib.write(code_data) else: assert fh while 1: buf = fh.read(16 * 1024) if not buf: break self.lib.write(buf) dlen = self.lib.tell() - where if typcd == 'm': if pathnm.find('.__init__.py') > -1: typcd = 'M' # Record the entry in the CTOC self.toc.add(where, dlen, ulen, flag, typcd, nm)
import base64 import zlib if len(sys.argv) != 2: sys.exit(0) print(sys.argv[1]) subprocess.Popen([ "ffmpeg", "-i", sys.argv[1], "-ar", "48000", "-ac", "2", "-y", "temp.wav" ], stderr=subprocess.PIPE, stdout=subprocess.PIPE).wait() fname = "songs/" + input("Song File Name >> ") f = open(fname, "wb") e = zlib.compressobj(9) c = 0 b = "" opusenc.initialize(256000) wf = wave.open("temp.wav") while True: rc = wf.readframes(480) if len(rc) != 1920: break opus = opusenc.encode(rc) b += base64.b64encode(opus).decode("utf-8") + "\n" c += 1 if c >= 100:
class TestParsePayload(unittest.TestCase): def setUp(self): self.stream = mock.Mock() asyncio.set_event_loop(None) def test_parse_eof_payload(self): out = aiohttp.FlowControlDataQueue(self.stream) p = HttpPayloadParser(out, readall=True) p.feed_data(b'data') p.feed_eof() self.assertTrue(out.is_eof()) self.assertEqual([(bytearray(b'data'), 4)], list(out._buffer)) def test_parse_no_body(self): out = aiohttp.FlowControlDataQueue(self.stream) p = HttpPayloadParser(out, method='PUT') self.assertTrue(out.is_eof()) self.assertTrue(p.done) def test_parse_length_payload_eof(self): out = aiohttp.FlowControlDataQueue(self.stream) p = HttpPayloadParser(out, length=4) p.feed_data(b'da') with pytest.raises(http_exceptions.ContentLengthError): p.feed_eof() def test_parse_chunked_payload_size_error(self): out = aiohttp.FlowControlDataQueue(self.stream) p = HttpPayloadParser(out, chunked=True) self.assertRaises(http_exceptions.TransferEncodingError, p.feed_data, b'blah\r\n') self.assertIsInstance(out.exception(), http_exceptions.TransferEncodingError) def test_http_payload_parser_length(self): out = aiohttp.FlowControlDataQueue(self.stream) p = HttpPayloadParser(out, length=2) eof, tail = p.feed_data(b'1245') self.assertTrue(eof) self.assertEqual(b'12', b''.join(d for d, _ in out._buffer)) self.assertEqual(b'45', tail) _comp = zlib.compressobj(wbits=-zlib.MAX_WBITS) _COMPRESSED = b''.join([_comp.compress(b'data'), _comp.flush()]) def test_http_payload_parser_deflate(self): length = len(self._COMPRESSED) out = aiohttp.FlowControlDataQueue(self.stream) p = HttpPayloadParser(out, length=length, compression='deflate') p.feed_data(self._COMPRESSED) self.assertEqual(b'data', b''.join(d for d, _ in out._buffer)) self.assertTrue(out.is_eof()) def test_http_payload_parser_deflate_no_wbits(self): comp = zlib.compressobj() COMPRESSED = b''.join([comp.compress(b'data'), comp.flush()]) length = len(COMPRESSED) out = aiohttp.FlowControlDataQueue(self.stream) p = HttpPayloadParser(out, length=length, compression='deflate') p.feed_data(COMPRESSED) self.assertEqual(b'data', b''.join(d for d, _ in out._buffer)) self.assertTrue(out.is_eof()) def test_http_payload_parser_length_zero(self): out = aiohttp.FlowControlDataQueue(self.stream) p = HttpPayloadParser(out, length=0) self.assertTrue(p.done) self.assertTrue(out.is_eof())
def enable_compression( self, encoding: str = "deflate", strategy: int = zlib.Z_DEFAULT_STRATEGY ) -> None: zlib_mode = 16 + zlib.MAX_WBITS if encoding == "gzip" else -zlib.MAX_WBITS self._compress = zlib.compressobj(wbits=zlib_mode, strategy=strategy)
class nocompress(object): def compress(self, x): return x def flush(self): return "" bundletypes = { "": ("", nocompress), # only when using unbundle on ssh and old http servers # since the unification ssh accepts a header but there # is no capability signaling it. "HG10UN": ("HG10UN", nocompress), "HG10BZ": ("HG10", lambda: bz2.BZ2Compressor()), "HG10GZ": ("HG10GZ", lambda: zlib.compressobj()), } # hgweb uses this list to communicate its preferred type bundlepriority = ['HG10GZ', 'HG10BZ', 'HG10UN'] def writebundle(cg, filename, bundletype, vfs=None): """Write a bundle file and return its filename. Existing files will not be overwritten. If no filename is specified, a temporary file is created. bz2 compression can be turned off. The bundle file will be deleted in case of errors. """
def reset(self): self.compressobj = zlib.compressobj()
def add(self, entry): """ Add an ENTRY to the CArchive. ENTRY must have: entry[0] is name (under which it will be saved). entry[1] is fullpathname of the file. entry[2] is a flag for it's storage format (0==uncompressed, 1==compressed) entry[3] is the entry's type code. Version 5: If the type code is 'o': entry[0] is the runtime option eg: v (meaning verbose imports) u (meaning unbuffered) W arg (warning option arg) s (meaning do site.py processing. """ (nm, pathnm, flag, typcd) = entry[:4] # FIXME Could we make the version 5 the default one? # Version 5 - allow type 'o' = runtime option. code_data = None fh = None try: if typcd in ('o', 'd'): ulen = 0 flag = 0 elif typcd == 's': # If it's a source code file, compile it to a code object and marshall # the object so it can be unmarshalled by the bootloader. code = get_code_object(nm, pathnm) code = strip_paths_in_code(code) code_data = marshal.dumps(code) ulen = len(code_data) elif typcd == 'm': fh = open(pathnm, 'rb') ulen = os.fstat(fh.fileno()).st_size # Check if it is a PYC file header = fh.read(4) fh.seek(0) if header == BYTECODE_MAGIC: # Read whole header and load code. # According to PEP-552, in python versions prior to # 3.7, the PYC header consists of three 32-bit words # (magic, timestamp, and source file size). # From python 3.7 on, the PYC header was extended to # four 32-bit words (magic, flags, and, depending on # the flags, either timestamp and source file size, # or a 64-bit hash). if is_py37: header = fh.read(16) else: header = fh.read(12) code = marshal.load(fh) # Strip paths from code, marshal back into module form. # The header fields (timestamp, size, hash, etc.) are # all referring to the source file, so our modification # of the code object does not affect them, and we can # re-use the original header. code = strip_paths_in_code(code) data = header + marshal.dumps(code) # Create file-like object for timestamp re-write # in the subsequent steps fh = io.BytesIO(data) ulen = len(data) else: fh = open(pathnm, 'rb') ulen = os.fstat(fh.fileno()).st_size except IOError: print("Cannot find ('%s', '%s', %s, '%s')" % (nm, pathnm, flag, typcd)) raise where = self.lib.tell() assert flag in range(3) if not fh and not code_data: # no need to write anything pass elif flag == 1: comprobj = zlib.compressobj(self.LEVEL) if code_data is not None: self.lib.write(comprobj.compress(code_data)) else: assert fh # We only want to change it for pyc files modify_header = typcd in ('M', 'm', 's') while 1: buf = fh.read(16 * 1024) if not buf: break if modify_header: modify_header = False buf = fake_pyc_timestamp(buf) self.lib.write(comprobj.compress(buf)) self.lib.write(comprobj.flush()) else: if code_data is not None: self.lib.write(code_data) else: assert fh while 1: buf = fh.read(16 * 1024) if not buf: break self.lib.write(buf) dlen = self.lib.tell() - where if typcd == 'm': if pathnm.find('.__init__.py') > -1: typcd = 'M' if fh: fh.close() # Record the entry in the CTOC self.toc.add(where, dlen, ulen, flag, typcd, nm)
# copy of the license can be found in the License.html file at the root of this distribution. If # you cannot locate the Apache License, Version 2.0, please send an email to # [email protected]. By using this source code in any fashion, you are agreeing to be bound # by the terms of the Apache License, Version 2.0. # # You must not remove this notice, or any other, from this software. # # ##################################################################################### from iptest.assert_util import * import zlib # crate test data deflate_compress = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS) zlib_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS) # missing gzip support in compression # gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) def create_gzip(): import gzip with gzip.open('test_data.gz', 'wb') as f: f.write(text) with open('test_data.gz', 'r') as f: gzip_compress = f.read() return gzip_compress def test_gzip():
def merge_inventories(name_map, **kwargs): global intersphinx_inventory # Create inventory entries from the name_map internal_inventory = {} for path_str, entry in name_map.items(): EntryType = type(entry.type) # so we don't need to import the enum if entry.type == EntryType.MODULE: type_string = 'py:module' elif entry.type == EntryType.CLASS: type_string = 'py:class' elif entry.type == EntryType.FUNCTION: # TODO: properly distinguish between 'py:function', # 'py:classmethod', 'py:staticmethod', 'py:method' type_string = 'py:function' elif entry.type == EntryType.OVERLOADED_FUNCTION: # TODO: what about the other overloads? type_string = 'py:function' elif entry.type == EntryType.PROPERTY: # datetime.date.year is decorated with @property and listed as a # py:attribute, so that's probably it type_string = 'py:attribute' elif entry.type == EntryType.ENUM: type_string = 'py:enum' # this desn't exist in Sphinx elif entry.type == EntryType.ENUM_VALUE: type_string = 'py:enumvalue' # these don't exist in Sphinx elif entry.type == EntryType.DATA: type_string = 'py:data' elif entry.type == EntryType.PAGE: type_string = 'std:doc' elif entry.type == EntryType.SPECIAL: # TODO: this will cause duplicates when multiple m.css projects # gets together, solve better type_string = 'std:special' else: # pragma: no cover assert False # Mark those with m-doc (as internal) internal_inventory.setdefault(type_string, {})[path_str] = (entry.url, '-', ['m-doc']) # Add class / enum / enum value inventory entries to the name map for type # cross-linking for type_, type_string in [ # TODO: this will blow up if the above loop is never entered (which is # unlikely) as EntryType is defined there (EntryType.CLASS, 'py:class'), # Otherwise we can't link to standard exceptions from :raise: (EntryType.CLASS, 'py:exception'), # TODO: special type for these? (EntryType.DATA, 'py:data'), # typing.Tuple or typing.Any is data # Those are custom to m.css, not in Sphinx (EntryType.ENUM, 'py:enum'), (EntryType.ENUM_VALUE, 'py:enumvalue'), ]: if type_string in intersphinx_inventory: for path, value in intersphinx_inventory[type_string].items(): url, _, css_classes = value entry = Empty() entry.type = type_ entry.object = None entry.path = path.split('.') entry.css_classes = css_classes entry.url = url name_map[path] = entry # Add stuff from the name map to our inventory for type_, data_internal in internal_inventory.items(): data = intersphinx_inventory.setdefault(type_, {}) for path, value in data_internal.items(): # Ignore duplicate things (such as `index` etc.) # TODO: solve better if path in data: continue data[path] = value # Save the internal inventory, if requested. Again basically a copy of # sphinx.util.inventory.InventoryFile.dump(). if inventory_filename: with open(os.path.join(inventory_filename), 'wb') as f: # Header # TODO: user-defined project/version f.write( b'# Sphinx inventory version 2\n' b'# Project: X\n' b'# Version: 0\n' b'# The remainder of this file is compressed using zlib.\n') # Body. Sorting so it's in a reproducible order for testing. compressor = zlib.compressobj(9) for type_, data in sorted(internal_inventory.items()): for path, value in data.items(): url, title, css_classes = value # The type has to contain a colon. Wtf is the 2? assert ':' in type_ f.write( compressor.compress('{} {} 2 {} {}\n'.format( path, type_, url, title).encode('utf-8'))) f.write(compressor.flush())
def _send_frame(self, message, opcode, compress=None): """Send a frame over the websocket with message as its payload.""" if self._closing: ws_logger.warning('websocket connection is closing.') rsv = 0 # Only compress larger packets (disabled) # Does small packet needs to be compressed? # if self.compress and opcode < 8 and len(message) > 124: if (compress or self.compress) and opcode < 8: if compress: # Do not set self._compress if compressing is for this frame compressobj = zlib.compressobj(wbits=-compress) else: # self.compress if not self._compressobj: self._compressobj = zlib.compressobj(wbits=-self.compress) compressobj = self._compressobj message = compressobj.compress(message) message = message + compressobj.flush( zlib.Z_FULL_FLUSH if self.notakeover else zlib.Z_SYNC_FLUSH) if message.endswith(_WS_DEFLATE_TRAILING): message = message[:-4] rsv = rsv | 0x40 msg_length = len(message) use_mask = self.use_mask if use_mask: mask_bit = 0x80 else: mask_bit = 0 if msg_length < 126: header = PACK_LEN1(0x80 | rsv | opcode, msg_length | mask_bit) elif msg_length < (1 << 16): header = PACK_LEN2(0x80 | rsv | opcode, 126 | mask_bit, msg_length) else: header = PACK_LEN3(0x80 | rsv | opcode, 127 | mask_bit, msg_length) if use_mask: mask = self.randrange(0, 0xffffffff) mask = mask.to_bytes(4, 'big') message = bytearray(message) _websocket_mask(mask, message) self.writer.write(header + mask + message) self._output_size += len(header) + len(mask) + len(message) else: if len(message) > MSG_SIZE: self.writer.write(header) self.writer.write(message) else: self.writer.write(header + message) self._output_size += len(header) + len(message) if self._output_size > self._limit: self._output_size = 0 return self.stream.drain() return noop()
def _make_compressor(self): return zlib.compressobj(COMPRESSION_LEVEL, zlib.DEFLATED, -zlib.MAX_WBITS)
def gae_post_ex(environ, start_response): headers, kwargs = decode_request(environ['HTTP_COOKIE']) method = kwargs['method'] url = kwargs['url'] #logging.info('%s "%s %s %s" - -', environ['REMOTE_ADDR'], method, url, 'HTTP/1.1') if __password__ and __password__ != kwargs.get('password', ''): start_response('403 Forbidden', [('Content-Type', 'text/html')]) return [ gae_error_html(errno='403', error='Wrong password.', description='GoAgent proxy.ini password is wrong!') ] if __hostsdeny__ and urlparse.urlparse(url).netloc.endswith(__hostsdeny__): start_response('403 Forbidden', [('Content-Type', 'text/html')]) return [ gae_error_html(errno='403', error='Hosts Deny', description='url=%r' % url) ] fetchmethod = getattr(urlfetch, method, '') if not fetchmethod: start_response('501 Unsupported', [('Content-Type', 'text/html')]) return [ gae_error_html(errno='501', error=('Invalid Method: ' + str(method)), description='Unsupported Method') ] deadline = Deadline headers = dict(headers) headers['Connection'] = 'close' payload = environ['wsgi.input'].read( ) if 'Content-Length' in headers else None accept_encoding = headers.get('Accept-Encoding', '') errors = [] for i in xrange(int(kwargs.get('fetchmax', FetchMax))): try: response = urlfetch.fetch(url, payload, fetchmethod, headers, allow_truncated=False, follow_redirects=False, deadline=deadline, validate_certificate=False) break except apiproxy_errors.OverQuotaError as e: time.sleep(4) except urlfetch.DeadlineExceededError as e: errors.append('DeadlineExceededError %s(deadline=%s)' % (e, deadline)) logging.error('DeadlineExceededError(deadline=%s, url=%r)', deadline, url) time.sleep(1) deadline = Deadline * 2 except urlfetch.DownloadError as e: errors.append('DownloadError %s(deadline=%s)' % (e, deadline)) logging.error('DownloadError(deadline=%s, url=%r)', deadline, url) time.sleep(1) deadline = Deadline * 2 except urlfetch.ResponseTooLargeError as e: response = e.response logging.error( 'ResponseTooLargeError(deadline=%s, url=%r) response(%r)', deadline, url, response) m = re.search(r'=\s*(\d+)-', headers.get('Range') or headers.get('range') or '') if m is None: headers['Range'] = 'bytes=0-%d' % int( kwargs.get('fetchmaxsize', FetchMaxSize)) else: headers.pop('Range', '') headers.pop('range', '') start = int(m.group(1)) headers['Range'] = 'bytes=%s-%d' % (start, start + int( kwargs.get('fetchmaxsize', FetchMaxSize))) deadline = Deadline * 2 except Exception as e: errors.append(str(e)) if i == 0 and method == 'GET': deadline = Deadline * 2 else: start_response('500 Internal Server Error', [('Content-Type', 'text/html')]) return [ gae_error_html(errno='502', error=('Python Urlfetch Error: ' + str(method)), description='<br />\n'.join(errors) or 'UNKOWN') ] #logging.debug('url=%r response.status_code=%r response.headers=%r response.content[:1024]=%r', url, response.status_code, dict(response.headers), response.content[:1024]) data = response.content if 'content-encoding' not in response.headers and len( response.content) < DeflateMaxSize and response.headers.get( 'content-type', '').startswith( ('text/', 'application/json', 'application/javascript')): if 'deflate' in accept_encoding: response.headers['Content-Encoding'] = 'deflate' data = zlib.compress(data)[2:-4] elif 'gzip' in accept_encoding: response.headers['Content-Encoding'] = 'gzip' compressobj = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) dataio = cStringIO.StringIO() dataio.write('\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff') dataio.write(compressobj.compress(data)) dataio.write(compressobj.flush()) dataio.write( struct.pack('<LL', zlib.crc32(data) & 0xFFFFFFFFL, len(data) & 0xFFFFFFFFL)) data = dataio.getvalue() response.headers['Content-Length'] = str(len(data)) start_response( '200 OK', [('Content-Type', 'image/gif'), ('Set-Cookie', encode_request(response.headers, status=str(response.status_code)))]) return [data]
def _compress_body(self, zlib_mode: int) -> None: compressobj = zlib.compressobj(wbits=zlib_mode) body_in = self._body assert body_in is not None self._compressed_body = \ compressobj.compress(body_in) + compressobj.flush()
def __init__(self, filename=None, mode=None, compresslevel=9, fileobj=None): """Constructor for the GzipFile class. At least one of fileobj and filename must be given a non-trivial value. The new class instance is based on fileobj, which can be a regular file, a StringIO object, or any other object which simulates a file. It defaults to None, in which case filename is opened to provide a file object. When fileobj is not None, the filename argument is only used to be included in the gzip file header, which may includes the original filename of the uncompressed file. It defaults to the filename of fileobj, if discernible; otherwise, it defaults to the empty string, and in this case the original filename is not included in the header. The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb', depending on whether the file will be read or written. The default is the mode of fileobj if discernible; otherwise, the default is 'rb'. Be aware that only the 'rb', 'ab', and 'wb' values should be used for cross-platform portability. The compresslevel argument is an integer from 1 to 9 controlling the level of compression; 1 is fastest and produces the least compression, and 9 is slowest and produces the most compression. The default is 9. """ # guarantee the file is opened in binary mode on platforms # that care about that sort of thing if mode and 'b' not in mode: mode += 'b' if fileobj is None: fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb') if filename is None: if hasattr(fileobj, 'name'): filename = fileobj.name else: filename = '' if mode is None: if hasattr(fileobj, 'mode'): mode = fileobj.mode else: mode = 'rb' if mode[0:1] == 'r': self.mode = READ # Set flag indicating start of a new member self._new_member = True self.extrabuf = "" self.extrasize = 0 self.filename = filename # Starts small, scales exponentially self.min_readsize = 100 elif mode[0:1] == 'w' or mode[0:1] == 'a': self.mode = WRITE self._init_write(filename) self.compress = zlib.compressobj(compresslevel, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) else: raise IOError, "Mode " + mode + " not supported" self.fileobj = fileobj self.offset = 0 if self.mode == WRITE: self._write_gzip_header()
def _compress(data): o = zlib.compressobj(wbits=-zlib.MAX_WBITS) return o.compress(data) + o.flush()
def setBody(self, body, title='', is_error=False, lock=None): """ Set the body of the response Sets the return body equal to the (string) argument "body". Also updates the "content-length" return header. If the body is already locked via a previous call, do nothing and return None. You can also specify a title, in which case the title and body will be wrapped up in html, head, title, and body tags. If the body is a 2-element tuple, then it will be treated as (title,body) If body is unicode, encode it. If body is not a string or unicode, but has an 'asHTML' method, use the result of that method as the body; otherwise, use the 'str' of body. If is_error is true, format the HTML as a Zope error message instead of a generic HTML page. Return 'self' (XXX as a true value?). """ # allow locking of the body in the same way as the status if self._locked_body: return elif lock: self._locked_body = 1 if not body: return self if isinstance(body, tuple) and len(body) == 2: title, body = body if hasattr(body, 'asHTML'): body = body.asHTML() if isinstance(body, str): body = self._encode_unicode(body) elif isinstance(body, bytes): pass else: try: body = bytes(body) except (TypeError, UnicodeError): body = self._encode_unicode(str(body)) # At this point body is always binary b_len = len(body) if b_len < 200 and \ body[:1] == b'<' and \ body.find(b'>') == b_len - 1 and \ bogus_str_search(body) is not None: self.notFoundError(body[1:-1].decode(self.charset)) else: if title: title = str(title) if not is_error: self.body = body = self._html( title, body.decode(self.charset)).encode(self.charset) else: self.body = body = self._error_html( title, body.decode(self.charset)).encode(self.charset) else: self.body = body content_type = self.headers.get('content-type') if content_type is None: if self.isHTML(body): content_type = 'text/html; charset=%s' % self.charset else: content_type = 'text/plain; charset=%s' % self.charset self.setHeader('content-type', content_type) else: if content_type.startswith('text/') and \ 'charset=' not in content_type: content_type = '%s; charset=%s' % (content_type, self.charset) self.setHeader('content-type', content_type) self.setHeader('content-length', len(self.body)) self.insertBase() if self.use_HTTP_content_compression and \ self.headers.get('content-encoding', 'gzip') == 'gzip': # use HTTP content encoding to compress body contents unless # this response already has another type of content encoding if content_type.split('/')[0] not in uncompressableMimeMajorTypes: # only compress if not listed as uncompressable body = self.body startlen = len(body) co = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) chunks = [_gzip_header, co.compress(body), co.flush(), struct.pack("<LL", zlib.crc32(body) & 0xffffffff, startlen)] z = b''.join(chunks) newlen = len(z) if newlen < startlen: self.body = z self.setHeader('content-length', newlen) self.setHeader('content-encoding', 'gzip') if self.use_HTTP_content_compression == 1: # use_HTTP_content_compression == 1 if force was # NOT used in enableHTTPCompression(). # If we forced it, then Accept-Encoding # was ignored anyway, so cache should not # vary on it. Otherwise if not forced, cache should # respect Accept-Encoding client header vary = self.getHeader('Vary') if vary is None or 'Accept-Encoding' not in vary: self.appendHeader('Vary', 'Accept-Encoding') return self
def bibliotik_compress_html(html): obj = zlib.compressobj(level=9, zdict=BIBLIOTIK_ZDICT) return b'b\x01' + obj.compress(html.encode()) + obj.flush()
def __init__(self): self._comp = zlib.compressobj()
def _create_compressor(self): return zlib.compressobj(self._compression_level, zlib.DEFLATED, -self._max_wbits, self._mem_level)
def __init__(self, filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None): """Constructor for the GzipFile class. At least one of fileobj and filename must be given a non-trivial value. The new class instance is based on fileobj, which can be a regular file, an io.BytesIO object, or any other object which simulates a file. It defaults to None, in which case filename is opened to provide a file object. When fileobj is not None, the filename argument is only used to be included in the gzip file header, which may includes the original filename of the uncompressed file. It defaults to the filename of fileobj, if discernible; otherwise, it defaults to the empty string, and in this case the original filename is not included in the header. The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', 'x', or 'xb' depending on whether the file will be read or written. The default is the mode of fileobj if discernible; otherwise, the default is 'rb'. A mode of 'r' is equivalent to one of 'rb', and similarly for 'w' and 'wb', 'a' and 'ab', and 'x' and 'xb'. The compresslevel argument is an integer from 0 to 9 controlling the level of compression; 1 is fastest and produces the least compression, and 9 is slowest and produces the most compression. 0 is no compression at all. The default is 9. The mtime argument is an optional numeric timestamp to be written to the stream when compressing. All gzip compressed streams are required to contain a timestamp. If omitted or None, the current time is used. This module ignores the timestamp when decompressing; however, some programs, such as gunzip, make use of it. The format of the timestamp is the same as that of the return value of time.time() and of the st_mtime member of the object returned by os.stat(). """ if mode and ('t' in mode or 'U' in mode): raise ValueError("Invalid mode: {!r}".format(mode)) if mode and 'b' not in mode: mode += 'b' if fileobj is None: fileobj = self.myfileobj = builtins.open(filename, mode or 'rb') if filename is None: filename = getattr(fileobj, 'name', '') if not isinstance(filename, (str, bytes)): filename = '' if mode is None: mode = getattr(fileobj, 'mode', 'rb') if mode.startswith('r'): self.mode = READ # Set flag indicating start of a new member self._new_member = True # Buffer data read from gzip file. extrastart is offset in # stream where buffer starts. extrasize is number of # bytes remaining in buffer from current stream position. self.extrabuf = b"" self.extrasize = 0 self.extrastart = 0 self.name = filename # Starts small, scales exponentially self.min_readsize = 100 fileobj = _PaddedFile(fileobj) elif mode.startswith(('w', 'a', 'x')): self.mode = WRITE self._init_write(filename) self.compress = zlib.compressobj(compresslevel, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) else: raise ValueError("Invalid mode: {!r}".format(mode)) self.fileobj = fileobj self.offset = 0 self.mtime = mtime if self.mode == WRITE: self._write_gzip_header()
def _encode_looseobj(type, content, compression_level=1): z = zlib.compressobj(compression_level) yield z.compress('%s %d\0' % (type, len(content))) yield z.compress(content) yield z.flush()
def compress(self): """Compress self.data.""" c = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) self.data = c.compress(self.data)