def test_compression_is_stable(self):
data = json.dumps({"foo": "bar", "dog": "cat"}).encode("utf-8")
data = snappy.compress(data)
data = snappy.decompress(data)
data = snappy.compress(data)
data = snappy.decompress(data)
self.assertEqual(
json.loads(data.decode("utf-8")),
{"foo": "bar", "dog": "cat"}
)
def test_compression_is_stable(self):
data = json.dumps({"foo": "bar", "dog": "cat"}).encode("utf-8")
data = snappy.compress(data)
data = snappy.decompress(data)
data = snappy.compress(data)
data = snappy.decompress(data)
self.assertEqual(json.loads(data.decode("utf-8")), {
"foo": "bar",
"dog": "cat"
})
def compressed(cls, compression, msgs):
"""
Returns a `MessageSet` with the given messages optionally compressed.
If compression is used, the message set is rendered, compressed, and
then a *new* message set is created with the raw output as a value
in a single message.
If no compression is set, the returned instance will have a
(<offset>, <message>) tuple for each given message, where the offset
is -1.
"""
if not compression:
return cls([(-1, msg) for msg in msgs])
set_format, set_data = cls([(-1, msg) for msg in msgs]).render()
# compressed message sets are nested and don't include the size
set_format = set_format[1:]
set_data.pop(0)
raw_set = struct.pack("!" + set_format, *set_data)
if compression == GZIP:
compressed_set = gzip.compress(raw_set)
elif compression == SNAPPY:
compressed_set = snappy.compress(raw_set)
container_msg = Message(magic=0,
attributes=compression,
key=None,
value=compressed_set)
return cls([(-1, container_msg)])
def test_compression_includes_magic_header(self):
data = json.dumps(
{"foo": "bar", "blee": "bloo", "dog": "cat"}).encode("utf-8")
data = snappy.compress(data)
header = struct.unpack_from("!bccccccbii", data)
self.assertEqual(
header,
(-126, b'S', b'N', b'A', b'P', b'P', b'Y', 0, 1, 1)
)
def test_compression_includes_magic_header(self):
data = json.dumps({
"foo": "bar",
"blee": "bloo",
"dog": "cat"
}).encode("utf-8")
data = snappy.compress(data)
header = struct.unpack_from("!bccccccbii", data)
self.assertEqual(header,
(-126, b'S', b'N', b'A', b'P', b'P', b'Y', 0, 1, 1))
def compressed(cls, compression, msgs):
"""
Returns a `MessageSet` with the given messages optionally compressed.
If compression is used, the message set is rendered, compressed, and
then a *new* message set is created with the raw output as a value
in a single message.
If no compression is set, the returned instance will have a
(<offset>, <message>) tuple for each given message, where the offset
is -1.
"""
if not compression:
return cls([(-1, msg) for msg in msgs])
set_format, set_data = cls([(-1, msg) for msg in msgs]).render()
# compressed message sets are nested and don't include the size
set_format = set_format[1:]
set_data.pop(0)
raw_set = struct.pack("!" + set_format, *set_data)
if compression == GZIP:
compressed_set = gzip.compress(raw_set)
elif compression == SNAPPY:
compressed_set = snappy.compress(raw_set)
container_msg = Message(
magic=0,
attributes=compression,
key=None,
value=compressed_set
)
return cls([(-1, container_msg)])