Esempio n. 1
0
def gzip_decode(payload):
    buffer = StringIO(payload)
    handle = gzip.GzipFile(fileobj=buffer, mode="r")
    result = handle.read()
    handle.close()
    buffer.close()
    return result
Esempio n. 2
0
def gzip_decode(payload):
    buffer = StringIO(payload)
    handle = gzip.GzipFile(fileobj=buffer, mode='r')
    result = handle.read()
    handle.close()
    buffer.close()
    return result
Esempio n. 3
0
def snappy_encode(payload, xerial_compatible=False, xerial_blocksize=32 * 1024):
    """Encodes the given data with snappy if xerial_compatible is set then the
       stream is encoded in a fashion compatible with the xerial snappy library

       The block size (xerial_blocksize) controls how frequent the blocking occurs
       32k is the default in the xerial library.

       The format winds up being
        +-------------+------------+--------------+------------+--------------+
        |   Header    | Block1 len | Block1 data  | Blockn len | Blockn data  |
        |-------------+------------+--------------+------------+--------------|
        |  16 bytes   |  BE int32  | snappy bytes |  BE int32  | snappy bytes |
        +-------------+------------+--------------+------------+--------------+

        It is important to not that the blocksize is the amount of uncompressed
        data presented to snappy at each block, whereas the blocklen is the
        number of bytes that will be present in the stream, that is the
        length will always be <= blocksize.
    """

    if not _has_snappy:
        raise NotImplementedError("Snappy codec is not available")

    if xerial_compatible:

        def _chunker():
            for i in compat.xrange(0, len(payload), xerial_blocksize):
                yield payload[i : i + xerial_blocksize]

        out = StringIO()

        header = b"".join([struct.pack("!" + fmt, dat) for fmt, dat in zip(_XERIAL_V1_FORMAT, _XERIAL_V1_HEADER)])

        out.write(header)
        for chunk in _chunker():
            block = snappy.compress(chunk)
            block_size = len(block)
            out.write(struct.pack("!i", block_size))
            out.write(block)

        out.seek(0)
        return out.read()

    else:
        return snappy.compress(payload)
Esempio n. 4
0
def gzip_encode(payload):
    buffer = StringIO()
    handle = gzip.GzipFile(fileobj=buffer, mode="w")
    handle.write(payload)
    handle.close()
    buffer.seek(0)
    result = buffer.read()
    buffer.close()
    return result
Esempio n. 5
0
def gzip_encode(payload):
    buffer = StringIO()
    handle = gzip.GzipFile(fileobj=buffer, mode="w")
    handle.write(payload)
    handle.close()
    buffer.seek(0)
    result = buffer.read()
    buffer.close()
    return result
Esempio n. 6
0
def snappy_decode(payload):
    if not _has_snappy:
        raise NotImplementedError("Snappy codec is not available")

    if _detect_xerial_stream(payload):
        # TODO ? Should become a fileobj ?
        out = StringIO()
        byt = compat.buffer(payload[16:])
        length = len(byt)
        cursor = 0

        while cursor < length:
            block_size = struct.unpack_from('!i', byt[cursor:])[0]
            # Skip the block size
            cursor += 4
            end = cursor + block_size
            out.write(snappy.decompress(bytes(byt[cursor:end])))
            cursor = end

        out.seek(0)
        return out.read()
    else:
        return snappy.decompress(payload)
Esempio n. 7
0
def snappy_decode(payload):
    if not _has_snappy:
        raise NotImplementedError("Snappy codec is not available")

    if _detect_xerial_stream(payload):
        # TODO ? Should become a fileobj ?
        out = StringIO()
        byt = compat.buffer(payload[16:])
        length = len(byt)
        cursor = 0

        while cursor < length:
            block_size = struct.unpack_from("!i", byt[cursor:])[0]
            # Skip the block size
            cursor += 4
            end = cursor + block_size
            out.write(snappy.decompress(bytes(byt[cursor:end])))
            cursor = end

        out.seek(0)
        return out.read()
    else:
        return snappy.decompress(payload)
Esempio n. 8
0
def snappy_encode(payload,
                  xerial_compatible=False,
                  xerial_blocksize=32 * 1024):
    """Encodes the given data with snappy if xerial_compatible is set then the
       stream is encoded in a fashion compatible with the xerial snappy library

       The block size (xerial_blocksize) controls how frequent the blocking occurs
       32k is the default in the xerial library.

       The format winds up being
        +-------------+------------+--------------+------------+--------------+
        |   Header    | Block1 len | Block1 data  | Blockn len | Blockn data  |
        |-------------+------------+--------------+------------+--------------|
        |  16 bytes   |  BE int32  | snappy bytes |  BE int32  | snappy bytes |
        +-------------+------------+--------------+------------+--------------+

        It is important to not that the blocksize is the amount of uncompressed
        data presented to snappy at each block, whereas the blocklen is the
        number of bytes that will be present in the stream, that is the
        length will always be <= blocksize.
    """

    if not _has_snappy:
        raise NotImplementedError("Snappy codec is not available")

    if xerial_compatible:

        def _chunker():
            for i in compat.xrange(0, len(payload), xerial_blocksize):
                yield payload[i:i + xerial_blocksize]

        out = StringIO()

        header = b''.join([
            struct.pack('!' + fmt, dat)
            for fmt, dat in zip(_XERIAL_V1_FORMAT, _XERIAL_V1_HEADER)
        ])

        out.write(header)
        for chunk in _chunker():
            block = snappy.compress(chunk)
            block_size = len(block)
            out.write(struct.pack('!i', block_size))
            out.write(block)

        out.seek(0)
        return out.read()

    else:
        return snappy.compress(payload)