def test_binary_pack_filters(self):
encoding_size = 128
filters = [(random.randint(0, 2 ** 32 - 1), generate_bytes(encoding_size)) for _ in range(10)]
packed_filters = binary_pack_filters(filters, encoding_size)
bin_format = binary_format(encoding_size)
for filter, packed_filter in zip(filters, packed_filters):
assert len(packed_filter) == encoding_size + 4
unpacked = binary_unpack_one(packed_filter, bin_format)
assert filter == unpacked
def test_insert_many_clks(self):
num_entities = 10_000
encoding_size = 2048 # non default encoding size
binary_formatter = binary_format(encoding_size)
raw_data = [generate_bytes(encoding_size) for i in range(100)]
encodings = [
binary_formatter.pack(i, raw_data[i % 100])
for i in range(num_entities)
]
blocks = [['1'] for _ in range(num_entities)]
project_id, project_auth_token, dp_id, dp_auth_token = self._create_project_and_dp(
)
conn, cur = _get_conn_and_cursor()
insert_encodings_into_blocks(conn,
dp_id,
block_ids=blocks,
encoding_ids=list(range(num_entities)),
encodings=encodings)
conn.commit()
stored_encoding_ids = list(get_encodingblock_ids(conn, dp_id, '1'))
assert len(stored_encoding_ids) == num_entities
for stored_encoding_id, original_id in zip(stored_encoding_ids,
range(num_entities)):
assert stored_encoding_id == original_id
stored_encodings = list(
get_chunk_of_encodings(conn,
dp_id,
stored_encoding_ids,
stored_binary_size=(encoding_size + 4)))
assert len(stored_encodings) == num_entities
for stored_encoding, original_encoding in zip(stored_encodings,
encodings):
assert bytes(stored_encoding) == original_encoding
block_names, block_sizes = zip(*list(get_block_metadata(conn, dp_id)))
assert len(block_names) == 1
assert len(block_sizes) == 1
assert block_names[0] == '1'
assert block_sizes[0] == 10_000
def test_fetch_chunk(self):
data = [generate_bytes(128) for _ in range(100)]
project_id, project_auth_token, dp_id, dp_auth_token = self._create_project_and_dp(
)
conn, cur = _get_conn_and_cursor()
num_entities = 10_000
blocks = [['1'] for _ in range(num_entities)]
encodings = [data[i % 100] for i in range(num_entities)]
insert_encodings_into_blocks(conn,
dp_id,
block_ids=blocks,
encoding_ids=list(range(num_entities)),
encodings=encodings)
conn.commit()
stored_encoding_ids = list(
get_encodingblock_ids(conn, dp_id, '1', offset=10, limit=20))
assert len(stored_encoding_ids) == 20
for i, stored_encoding_id in enumerate(stored_encoding_ids):
assert stored_encoding_id == i + 10
def test_binary_unpack_filters(self):
encoding_size = 128
filters = [(random.randint(0, 2 ** 32 - 1), generate_bytes(encoding_size)) for _ in range(10)]
laundered_filters = binary_unpack_filters(binary_pack_filters(filters, encoding_size),
encoding_size=encoding_size)
assert filters == laundered_filters
def test_random_bytes(self):
rb = generate_bytes(2048//8)
srb = serialize_bytes(rb)
dsrb = deserialize_bytes(srb)
self.assertEqual(dsrb, rb)
def test_random_bytes(self):
with temp_file_containing(generate_bytes(128)) as fp:
filename = fp.name
with pytest.raises(InvalidConfiguration):
load_yaml_config(filename)