def check_zz_encode(int_type):
src_array = (int_type * ARRAY_SIZE)()
src_array_addr = addressof(src_array)
dst_len = 9 * ARRAY_SIZE
dst_array = (c_uint8 * dst_len)()
dst_array_addr = addressof(dst_array)
res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr,
dst_len)
# should be 1 byte set to 0x13 (10 zeros => value = -10, or 0x13 in zigzag
# encoding
assert res == 1
assert dst_array[0] == 0x13
# last counter set to 1
# the encoded result should be 2 bytes long
# 0x11 (9 zeros => -9 coded as 17)
# 0x02 (1 is coded as 2)
src_array[ARRAY_SIZE - 1] = 1
res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr,
dst_len)
assert res == 2
assert dst_array[0] == 0x11
assert dst_array[1] == 0x02
# all counters set to 1, we should get a zigzag encoded of
# 10 bytes all set to 0x02 (in zigzag encoding 1 is coded as 2)
for index in range(ARRAY_SIZE):
src_array[index] = 1
res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr,
dst_len)
assert res == ARRAY_SIZE
for index in range(ARRAY_SIZE):
assert dst_array[index] == 2
def check_zz_encode(int_type):
src_array = (int_type * ARRAY_SIZE)()
src_array_addr = addressof(src_array)
dst_len = 9 * ARRAY_SIZE
dst_array = (c_uint8 * dst_len)()
dst_array_addr = addressof(dst_array)
res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len)
# should be 1 byte set to 0x13 (10 zeros => value = -10, or 0x13 in zigzag
# encoding
assert(res == 1)
assert(dst_array[0] == 0x13)
# last counter set to 1
# the encoded result should be 2 bytes long
# 0x11 (9 zeros => -9 coded as 17)
# 0x02 (1 is coded as 2)
src_array[ARRAY_SIZE - 1] = 1
res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len)
assert(res == 2)
assert(dst_array[0] == 0x11)
assert(dst_array[1] == 0x02)
# all counters set to 1, we should get a zigzag encoded of
# 10 bytes all set to 0x02 (in zigzag encoding 1 is coded as 2)
for index in range(ARRAY_SIZE):
src_array[index] = 1
res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len)
assert(res == ARRAY_SIZE)
for index in range(ARRAY_SIZE):
assert(dst_array[index] == 2)
def compress(self, counts_limit):
'''Compress this payload instance
Args:
counts_limit how many counters should be encoded
starting from index 0 (can be 0),
Return:
the compressed payload (python string)
'''
if self.payload:
# worst case varint encoded length is when each counter is at the maximum value
# in this case 1 more byte per counter is needed due to the more bits
varint_len = counts_limit * (self.word_size + 1)
# allocate enough space to fit the header and the varint string
encode_buf = (c_byte * (payload_header_size + varint_len))()
# encode past the payload header
varint_len = encode(addressof(self.counts), counts_limit,
self.word_size,
addressof(encode_buf) + payload_header_size,
varint_len)
# copy the header after updating the varint stream length
self.payload.payload_len = varint_len
ctypes.memmove(addressof(encode_buf), addressof(self.payload),
payload_header_size)
cdata = zlib.compress(
ctypes.string_at(encode_buf, payload_header_size + varint_len))
return cdata
# can't compress if no payload
raise RuntimeError('No payload to compress')
def compress(self, counts_limit):
'''Compress this payload instance
Args:
counts_limit how many counters should be encoded
starting from index 0 (can be 0),
Return:
the compressed payload (python string)
'''
if self.payload:
# worst case varint encoded length is when each counter is at the maximum value
# in this case 1 more byte per counter is needed due to the more bits
varint_len = counts_limit * (self.word_size + 1)
# allocate enough space to fit the header and the varint string
encode_buf = (c_byte * (payload_header_size + varint_len))()
# encode past the payload header
varint_len = encode(addressof(self.counts), counts_limit,
self.word_size,
addressof(encode_buf) + payload_header_size,
varint_len)
# copy the header after updating the varint stream length
self.payload.payload_len = varint_len
ctypes.memmove(addressof(encode_buf), addressof(self.payload), payload_header_size)
cdata = zlib.compress(ctypes.string_at(encode_buf, payload_header_size + varint_len))
return cdata
# can't compress if no payload
raise RuntimeError('No payload to compress')
def check_zz_identity(src_array, int_type, min_nz_index, max_nz_index, total_count, offset):
dst_len = (sizeof(int_type) + 1) * ARRAY_SIZE
dst = (c_uint8 * (offset + dst_len))()
varint_len = encode(addressof(src_array), ARRAY_SIZE, sizeof(int_type),
addressof(dst) + offset, dst_len)
varint_string = string_at(dst, varint_len + offset)
dst_array = (int_type * ARRAY_SIZE)()
res = decode(varint_string, offset, addressof(dst_array), ARRAY_SIZE, sizeof(int_type))
assert(res['total'] == total_count)
if total_count:
assert(res['min_nonzero_index'] == min_nz_index)
assert(res['max_nonzero_index'] == max_nz_index)
for index in range(ARRAY_SIZE):
assert(dst_array[index] == src_array[index])
def check_zz_identity(src_array, int_type, min_nz_index, max_nz_index,
total_count, offset):
dst_len = (sizeof(int_type) + 1) * ARRAY_SIZE
dst = (c_uint8 * (offset + dst_len))()
varint_len = encode(addressof(src_array), ARRAY_SIZE, sizeof(int_type),
addressof(dst) + offset, dst_len)
varint_string = string_at(dst, varint_len + offset)
dst_array = (int_type * ARRAY_SIZE)()
res = decode(varint_string, offset, addressof(dst_array), ARRAY_SIZE,
sizeof(int_type))
assert res['total'] == total_count
if total_count:
assert res['min_nonzero_index'] == min_nz_index
assert res['max_nonzero_index'] == max_nz_index
for index in range(ARRAY_SIZE):
assert dst_array[index] == src_array[index]
def test_zz_encode_errors():
with pytest.raises(TypeError):
encode()
with pytest.raises(TypeError):
encode(None, None, 0, 0)
src_array = (c_uint16 * ARRAY_SIZE)()
src_array_addr = addressof(src_array)
dst_len = 9 * ARRAY_SIZE
# negative length
with pytest.raises(ValueError):
encode(src_array_addr, -1, sizeof(c_uint16), 0, dst_len)
# dest length too small
with pytest.raises(ValueError):
encode(src_array_addr, ARRAY_SIZE, 4, 0, 4)
# invalid word size
with pytest.raises(ValueError):
encode(src_array_addr, ARRAY_SIZE, 3, 0, 0)
# Null dest ptr
with pytest.raises(ValueError):
encode(src_array_addr, ARRAY_SIZE, 4, 0, dst_len)
def test_zz_encode_errors():
with pytest.raises(TypeError):
encode()
with pytest.raises(TypeError):
encode(None, None, 0, 0)
src_array = (c_uint16 * ARRAY_SIZE)()
src_array_addr = addressof(src_array)
dst_len = 9 * ARRAY_SIZE
# negative length
with pytest.raises(ValueError):
encode(src_array_addr, -1, sizeof(c_uint16), 0, dst_len)
# dest length too small
with pytest.raises(ValueError):
encode(src_array_addr, ARRAY_SIZE, 4, 0, 4)
# invalid word size
with pytest.raises(ValueError):
encode(src_array_addr, ARRAY_SIZE, 3, 0, 0)
# Null dest ptr
with pytest.raises(ValueError):
encode(src_array_addr, ARRAY_SIZE, 4, 0, dst_len)
