def check_zz_encode(int_type): src_array = (int_type * ARRAY_SIZE)() src_array_addr = addressof(src_array) dst_len = 9 * ARRAY_SIZE dst_array = (c_uint8 * dst_len)() dst_array_addr = addressof(dst_array) res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len) # should be 1 byte set to 0x13 (10 zeros => value = -10, or 0x13 in zigzag # encoding assert res == 1 assert dst_array[0] == 0x13 # last counter set to 1 # the encoded result should be 2 bytes long # 0x11 (9 zeros => -9 coded as 17) # 0x02 (1 is coded as 2) src_array[ARRAY_SIZE - 1] = 1 res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len) assert res == 2 assert dst_array[0] == 0x11 assert dst_array[1] == 0x02 # all counters set to 1, we should get a zigzag encoded of # 10 bytes all set to 0x02 (in zigzag encoding 1 is coded as 2) for index in range(ARRAY_SIZE): src_array[index] = 1 res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len) assert res == ARRAY_SIZE for index in range(ARRAY_SIZE): assert dst_array[index] == 2
def check_zz_encode(int_type): src_array = (int_type * ARRAY_SIZE)() src_array_addr = addressof(src_array) dst_len = 9 * ARRAY_SIZE dst_array = (c_uint8 * dst_len)() dst_array_addr = addressof(dst_array) res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len) # should be 1 byte set to 0x13 (10 zeros => value = -10, or 0x13 in zigzag # encoding assert(res == 1) assert(dst_array[0] == 0x13) # last counter set to 1 # the encoded result should be 2 bytes long # 0x11 (9 zeros => -9 coded as 17) # 0x02 (1 is coded as 2) src_array[ARRAY_SIZE - 1] = 1 res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len) assert(res == 2) assert(dst_array[0] == 0x11) assert(dst_array[1] == 0x02) # all counters set to 1, we should get a zigzag encoded of # 10 bytes all set to 0x02 (in zigzag encoding 1 is coded as 2) for index in range(ARRAY_SIZE): src_array[index] = 1 res = encode(src_array_addr, ARRAY_SIZE, sizeof(int_type), dst_array_addr, dst_len) assert(res == ARRAY_SIZE) for index in range(ARRAY_SIZE): assert(dst_array[index] == 2)
def compress(self, counts_limit): '''Compress this payload instance Args: counts_limit how many counters should be encoded starting from index 0 (can be 0), Return: the compressed payload (python string) ''' if self.payload: # worst case varint encoded length is when each counter is at the maximum value # in this case 1 more byte per counter is needed due to the more bits varint_len = counts_limit * (self.word_size + 1) # allocate enough space to fit the header and the varint string encode_buf = (c_byte * (payload_header_size + varint_len))() # encode past the payload header varint_len = encode(addressof(self.counts), counts_limit, self.word_size, addressof(encode_buf) + payload_header_size, varint_len) # copy the header after updating the varint stream length self.payload.payload_len = varint_len ctypes.memmove(addressof(encode_buf), addressof(self.payload), payload_header_size) cdata = zlib.compress( ctypes.string_at(encode_buf, payload_header_size + varint_len)) return cdata # can't compress if no payload raise RuntimeError('No payload to compress')
def compress(self, counts_limit): '''Compress this payload instance Args: counts_limit how many counters should be encoded starting from index 0 (can be 0), Return: the compressed payload (python string) ''' if self.payload: # worst case varint encoded length is when each counter is at the maximum value # in this case 1 more byte per counter is needed due to the more bits varint_len = counts_limit * (self.word_size + 1) # allocate enough space to fit the header and the varint string encode_buf = (c_byte * (payload_header_size + varint_len))() # encode past the payload header varint_len = encode(addressof(self.counts), counts_limit, self.word_size, addressof(encode_buf) + payload_header_size, varint_len) # copy the header after updating the varint stream length self.payload.payload_len = varint_len ctypes.memmove(addressof(encode_buf), addressof(self.payload), payload_header_size) cdata = zlib.compress(ctypes.string_at(encode_buf, payload_header_size + varint_len)) return cdata # can't compress if no payload raise RuntimeError('No payload to compress')
def check_zz_identity(src_array, int_type, min_nz_index, max_nz_index, total_count, offset): dst_len = (sizeof(int_type) + 1) * ARRAY_SIZE dst = (c_uint8 * (offset + dst_len))() varint_len = encode(addressof(src_array), ARRAY_SIZE, sizeof(int_type), addressof(dst) + offset, dst_len) varint_string = string_at(dst, varint_len + offset) dst_array = (int_type * ARRAY_SIZE)() res = decode(varint_string, offset, addressof(dst_array), ARRAY_SIZE, sizeof(int_type)) assert(res['total'] == total_count) if total_count: assert(res['min_nonzero_index'] == min_nz_index) assert(res['max_nonzero_index'] == max_nz_index) for index in range(ARRAY_SIZE): assert(dst_array[index] == src_array[index])
def check_zz_identity(src_array, int_type, min_nz_index, max_nz_index, total_count, offset): dst_len = (sizeof(int_type) + 1) * ARRAY_SIZE dst = (c_uint8 * (offset + dst_len))() varint_len = encode(addressof(src_array), ARRAY_SIZE, sizeof(int_type), addressof(dst) + offset, dst_len) varint_string = string_at(dst, varint_len + offset) dst_array = (int_type * ARRAY_SIZE)() res = decode(varint_string, offset, addressof(dst_array), ARRAY_SIZE, sizeof(int_type)) assert res['total'] == total_count if total_count: assert res['min_nonzero_index'] == min_nz_index assert res['max_nonzero_index'] == max_nz_index for index in range(ARRAY_SIZE): assert dst_array[index] == src_array[index]
def test_zz_encode_errors(): with pytest.raises(TypeError): encode() with pytest.raises(TypeError): encode(None, None, 0, 0) src_array = (c_uint16 * ARRAY_SIZE)() src_array_addr = addressof(src_array) dst_len = 9 * ARRAY_SIZE # negative length with pytest.raises(ValueError): encode(src_array_addr, -1, sizeof(c_uint16), 0, dst_len) # dest length too small with pytest.raises(ValueError): encode(src_array_addr, ARRAY_SIZE, 4, 0, 4) # invalid word size with pytest.raises(ValueError): encode(src_array_addr, ARRAY_SIZE, 3, 0, 0) # Null dest ptr with pytest.raises(ValueError): encode(src_array_addr, ARRAY_SIZE, 4, 0, dst_len)