Пример #1
0
    def test_compression_is_stable(self):
        data = json.dumps({"foo": "bar", "dog": "cat"}).encode("utf-8")

        data = snappy.compress(data)
        data = snappy.decompress(data)
        data = snappy.compress(data)
        data = snappy.decompress(data)

        self.assertEqual(
            json.loads(data.decode("utf-8")),
            {"foo": "bar", "dog": "cat"}
        )
Пример #2
0
    def test_compression_is_stable(self):
        data = json.dumps({"foo": "bar", "dog": "cat"}).encode("utf-8")

        data = snappy.compress(data)
        data = snappy.decompress(data)
        data = snappy.compress(data)
        data = snappy.decompress(data)

        self.assertEqual(json.loads(data.decode("utf-8")), {
            "foo": "bar",
            "dog": "cat"
        })
Пример #3
0
    def compressed(cls, compression, msgs):
        """
        Returns a `MessageSet` with the given messages optionally compressed.

        If compression is used, the message set is rendered, compressed, and
        then a *new* message set is created with the raw output as a value
        in a single message.

        If no compression is set, the returned instance will have a
        (<offset>, <message>) tuple for each given message, where the offset
        is -1.
        """
        if not compression:
            return cls([(-1, msg) for msg in msgs])

        set_format, set_data = cls([(-1, msg) for msg in msgs]).render()

        # compressed message sets are nested and don't include the size
        set_format = set_format[1:]
        set_data.pop(0)

        raw_set = struct.pack("!" + set_format, *set_data)

        if compression == GZIP:
            compressed_set = gzip.compress(raw_set)
        elif compression == SNAPPY:
            compressed_set = snappy.compress(raw_set)

        container_msg = Message(magic=0,
                                attributes=compression,
                                key=None,
                                value=compressed_set)

        return cls([(-1, container_msg)])
Пример #4
0
    def test_compression_includes_magic_header(self):
        data = json.dumps(
            {"foo": "bar", "blee": "bloo", "dog": "cat"}).encode("utf-8")
        data = snappy.compress(data)

        header = struct.unpack_from("!bccccccbii", data)

        self.assertEqual(
            header,
            (-126, b'S', b'N', b'A', b'P', b'P', b'Y', 0, 1, 1)
        )
Пример #5
0
    def test_compression_includes_magic_header(self):
        data = json.dumps({
            "foo": "bar",
            "blee": "bloo",
            "dog": "cat"
        }).encode("utf-8")
        data = snappy.compress(data)

        header = struct.unpack_from("!bccccccbii", data)

        self.assertEqual(header,
                         (-126, b'S', b'N', b'A', b'P', b'P', b'Y', 0, 1, 1))
Пример #6
0
    def compressed(cls, compression, msgs):
        """
        Returns a `MessageSet` with the given messages optionally compressed.

        If compression is used, the message set is rendered, compressed, and
        then a *new* message set is created with the raw output as a value
        in a single message.

        If no compression is set, the returned instance will have a
        (<offset>, <message>) tuple for each given message, where the offset
        is -1.
        """
        if not compression:
            return cls([(-1, msg) for msg in msgs])

        set_format, set_data = cls([(-1, msg) for msg in msgs]).render()

        # compressed message sets are nested and don't include the size
        set_format = set_format[1:]
        set_data.pop(0)

        raw_set = struct.pack("!" + set_format, *set_data)

        if compression == GZIP:
            compressed_set = gzip.compress(raw_set)
        elif compression == SNAPPY:
            compressed_set = snappy.compress(raw_set)

        container_msg = Message(
            magic=0,
            attributes=compression,
            key=None,
            value=compressed_set
        )

        return cls([(-1, container_msg)])