Beispiel #1
0
def check_zz_encode(int_type):
    src_array = (int_type * ARRAY_SIZE)()
    src_array_addr = addressof(src_array)
    dst_len = 9 * ARRAY_SIZE
    dst_array = (c_uint8 * dst_len)()
    dst_array_addr = addressof(dst_array)

    res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr,
                 dst_len)
    # should be 1 byte set to 0x13 (10 zeros => value = -10, or 0x13 in zigzag
    # encoding
    assert res == 1
    assert dst_array[0] == 0x13

    # last counter set to 1
    # the encoded result should be 2 bytes long
    # 0x11   (9 zeros => -9 coded as 17)
    # 0x02   (1 is coded as 2)
    src_array[ARRAY_SIZE - 1] = 1
    res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr,
                 dst_len)
    assert res == 2
    assert dst_array[0] == 0x11
    assert dst_array[1] == 0x02

    # all counters set to 1, we should get a zigzag encoded of
    # 10 bytes all set to 0x02 (in zigzag encoding 1 is coded as 2)
    for index in range(ARRAY_SIZE):
        src_array[index] = 1
    res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr,
                 dst_len)
    assert res == ARRAY_SIZE
    for index in range(ARRAY_SIZE):
        assert dst_array[index] == 2
def check_zz_encode(int_type):
    src_array = (int_type * ARRAY_SIZE)()
    src_array_addr = addressof(src_array)
    dst_len = 9 * ARRAY_SIZE
    dst_array = (c_uint8 * dst_len)()
    dst_array_addr = addressof(dst_array)

    res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len)
    # should be 1 byte set to 0x13 (10 zeros => value = -10, or 0x13 in zigzag
    # encoding
    assert(res == 1)
    assert(dst_array[0] == 0x13)

    # last counter set to 1
    # the encoded result should be 2 bytes long
    # 0x11   (9 zeros => -9 coded as 17)
    # 0x02   (1 is coded as 2)
    src_array[ARRAY_SIZE - 1] = 1
    res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len)
    assert(res == 2)
    assert(dst_array[0] == 0x11)
    assert(dst_array[1] == 0x02)

    # all counters set to 1, we should get a zigzag encoded of
    # 10 bytes all set to 0x02 (in zigzag encoding 1 is coded as 2)
    for index in range(ARRAY_SIZE):
        src_array[index] = 1
    res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len)
    assert(res == ARRAY_SIZE)
    for index in range(ARRAY_SIZE):
        assert(dst_array[index] == 2)
Beispiel #3
0
    def compress(self, counts_limit):
        '''Compress this payload instance
        Args:
            counts_limit how many counters should be encoded
                           starting from index 0 (can be 0),
        Return:
            the compressed payload (python string)
        '''
        if self.payload:
            # worst case varint encoded length is when each counter is at the maximum value
            # in this case 1 more byte per counter is needed due to the more bits
            varint_len = counts_limit * (self.word_size + 1)
            # allocate enough space to fit the header and the varint string
            encode_buf = (c_byte * (payload_header_size + varint_len))()

            # encode past the payload header
            varint_len = encode(addressof(self.counts), counts_limit,
                                self.word_size,
                                addressof(encode_buf) + payload_header_size,
                                varint_len)

            # copy the header after updating the varint stream length
            self.payload.payload_len = varint_len
            ctypes.memmove(addressof(encode_buf), addressof(self.payload),
                           payload_header_size)

            cdata = zlib.compress(
                ctypes.string_at(encode_buf, payload_header_size + varint_len))
            return cdata
        # can't compress if no payload
        raise RuntimeError('No payload to compress')
Beispiel #4
0
    def compress(self, counts_limit):
        '''Compress this payload instance
        Args:
            counts_limit how many counters should be encoded
                           starting from index 0 (can be 0),
        Return:
            the compressed payload (python string)
        '''
        if self.payload:
            # worst case varint encoded length is when each counter is at the maximum value
            # in this case 1 more byte per counter is needed due to the more bits
            varint_len = counts_limit * (self.word_size + 1)
            # allocate enough space to fit the header and the varint string
            encode_buf = (c_byte * (payload_header_size + varint_len))()

            # encode past the payload header
            varint_len = encode(addressof(self.counts), counts_limit,
                                self.word_size,
                                addressof(encode_buf) + payload_header_size,
                                varint_len)

            # copy the header after updating the varint stream length
            self.payload.payload_len = varint_len
            ctypes.memmove(addressof(encode_buf), addressof(self.payload), payload_header_size)

            cdata = zlib.compress(ctypes.string_at(encode_buf, payload_header_size + varint_len))
            return cdata
        # can't compress if no payload
        raise RuntimeError('No payload to compress')
def check_zz_identity(src_array, int_type, min_nz_index, max_nz_index, total_count, offset):
    dst_len = (sizeof(int_type) + 1) * ARRAY_SIZE
    dst = (c_uint8 * (offset + dst_len))()

    varint_len = encode(addressof(src_array), ARRAY_SIZE, sizeof(int_type),
                        addressof(dst) + offset, dst_len)
    varint_string = string_at(dst, varint_len + offset)

    dst_array = (int_type * ARRAY_SIZE)()
    res = decode(varint_string, offset, addressof(dst_array), ARRAY_SIZE, sizeof(int_type))
    assert(res['total'] == total_count)
    if total_count:
        assert(res['min_nonzero_index'] == min_nz_index)
        assert(res['max_nonzero_index'] == max_nz_index)
    for index in range(ARRAY_SIZE):
        assert(dst_array[index] == src_array[index])
Beispiel #6
0
def check_zz_identity(src_array, int_type, min_nz_index, max_nz_index,
                      total_count, offset):
    dst_len = (sizeof(int_type) + 1) * ARRAY_SIZE
    dst = (c_uint8 * (offset + dst_len))()

    varint_len = encode(addressof(src_array), ARRAY_SIZE, sizeof(int_type),
                        addressof(dst) + offset, dst_len)
    varint_string = string_at(dst, varint_len + offset)

    dst_array = (int_type * ARRAY_SIZE)()
    res = decode(varint_string, offset, addressof(dst_array), ARRAY_SIZE,
                 sizeof(int_type))
    assert res['total'] == total_count
    if total_count:
        assert res['min_nonzero_index'] == min_nz_index
        assert res['max_nonzero_index'] == max_nz_index
    for index in range(ARRAY_SIZE):
        assert dst_array[index] == src_array[index]
Beispiel #7
0
def test_zz_encode_errors():
    with pytest.raises(TypeError):
        encode()
    with pytest.raises(TypeError):
        encode(None, None, 0, 0)
    src_array = (c_uint16 * ARRAY_SIZE)()
    src_array_addr = addressof(src_array)
    dst_len = 9 * ARRAY_SIZE

    # negative length
    with pytest.raises(ValueError):
        encode(src_array_addr, -1, sizeof(c_uint16), 0, dst_len)
    # dest length too small
    with pytest.raises(ValueError):
        encode(src_array_addr, ARRAY_SIZE, 4, 0, 4)
    # invalid word size
    with pytest.raises(ValueError):
        encode(src_array_addr, ARRAY_SIZE, 3, 0, 0)
    # Null dest ptr
    with pytest.raises(ValueError):
        encode(src_array_addr, ARRAY_SIZE, 4, 0, dst_len)
def test_zz_encode_errors():
    with pytest.raises(TypeError):
        encode()
    with pytest.raises(TypeError):
        encode(None, None, 0, 0)
    src_array = (c_uint16 * ARRAY_SIZE)()
    src_array_addr = addressof(src_array)
    dst_len = 9 * ARRAY_SIZE

    # negative length
    with pytest.raises(ValueError):
        encode(src_array_addr, -1, sizeof(c_uint16), 0, dst_len)
    # dest length too small
    with pytest.raises(ValueError):
        encode(src_array_addr, ARRAY_SIZE, 4, 0, 4)
    # invalid word size
    with pytest.raises(ValueError):
        encode(src_array_addr, ARRAY_SIZE, 3, 0, 0)
    # Null dest ptr
    with pytest.raises(ValueError):
        encode(src_array_addr, ARRAY_SIZE, 4, 0, dst_len)