def test_compression_is_stable(self): data = json.dumps({"foo": "bar", "blee": "bloo", "dog": "cat"}) data = data.encode("utf-8") data = gzip.compress(data) data = gzip.decompress(data) data = gzip.compress(data) data = gzip.decompress(data) self.assertEqual( json.loads(data.decode("utf-8")), {"foo": "bar", "blee": "bloo", "dog": "cat"} )
def test_compression_is_stable(self): data = json.dumps({"foo": "bar", "blee": "bloo", "dog": "cat"}) data = data.encode("utf-8") data = gzip.compress(data) data = gzip.decompress(data) data = gzip.compress(data) data = gzip.decompress(data) self.assertEqual(json.loads(data.decode("utf-8")), { "foo": "bar", "blee": "bloo", "dog": "cat" })
def compressed(cls, compression, msgs): """ Returns a `MessageSet` with the given messages optionally compressed. If compression is used, the message set is rendered, compressed, and then a *new* message set is created with the raw output as a value in a single message. If no compression is set, the returned instance will have a (<offset>, <message>) tuple for each given message, where the offset is -1. """ if not compression: return cls([(-1, msg) for msg in msgs]) set_format, set_data = cls([(-1, msg) for msg in msgs]).render() # compressed message sets are nested and don't include the size set_format = set_format[1:] set_data.pop(0) raw_set = struct.pack("!" + set_format, *set_data) if compression == GZIP: compressed_set = gzip.compress(raw_set) elif compression == SNAPPY: compressed_set = snappy.compress(raw_set) container_msg = Message(magic=0, attributes=compression, key=None, value=compressed_set) return cls([(-1, container_msg)])
def compressed(cls, compression, msgs): """ Returns a `MessageSet` with the given messages optionally compressed. If compression is used, the message set is rendered, compressed, and then a *new* message set is created with the raw output as a value in a single message. If no compression is set, the returned instance will have a (<offset>, <message>) tuple for each given message, where the offset is -1. """ if not compression: return cls([(-1, msg) for msg in msgs]) set_format, set_data = cls([(-1, msg) for msg in msgs]).render() # compressed message sets are nested and don't include the size set_format = set_format[1:] set_data.pop(0) raw_set = struct.pack("!" + set_format, *set_data) if compression == GZIP: compressed_set = gzip.compress(raw_set) elif compression == SNAPPY: compressed_set = snappy.compress(raw_set) container_msg = Message( magic=0, attributes=compression, key=None, value=compressed_set ) return cls([(-1, container_msg)])