def test_binary_pack_filters(self): encoding_size = 128 filters = [(random.randint(0, 2 ** 32 - 1), generate_bytes(encoding_size)) for _ in range(10)] packed_filters = binary_pack_filters(filters, encoding_size) bin_format = binary_format(encoding_size) for filter, packed_filter in zip(filters, packed_filters): assert len(packed_filter) == encoding_size + 4 unpacked = binary_unpack_one(packed_filter, bin_format) assert filter == unpacked
def test_insert_many_clks(self): num_entities = 10_000 encoding_size = 2048 # non default encoding size binary_formatter = binary_format(encoding_size) raw_data = [generate_bytes(encoding_size) for i in range(100)] encodings = [ binary_formatter.pack(i, raw_data[i % 100]) for i in range(num_entities) ] blocks = [['1'] for _ in range(num_entities)] project_id, project_auth_token, dp_id, dp_auth_token = self._create_project_and_dp( ) conn, cur = _get_conn_and_cursor() insert_encodings_into_blocks(conn, dp_id, block_ids=blocks, encoding_ids=list(range(num_entities)), encodings=encodings) conn.commit() stored_encoding_ids = list(get_encodingblock_ids(conn, dp_id, '1')) assert len(stored_encoding_ids) == num_entities for stored_encoding_id, original_id in zip(stored_encoding_ids, range(num_entities)): assert stored_encoding_id == original_id stored_encodings = list( get_chunk_of_encodings(conn, dp_id, stored_encoding_ids, stored_binary_size=(encoding_size + 4))) assert len(stored_encodings) == num_entities for stored_encoding, original_encoding in zip(stored_encodings, encodings): assert bytes(stored_encoding) == original_encoding block_names, block_sizes = zip(*list(get_block_metadata(conn, dp_id))) assert len(block_names) == 1 assert len(block_sizes) == 1 assert block_names[0] == '1' assert block_sizes[0] == 10_000
def test_fetch_chunk(self): data = [generate_bytes(128) for _ in range(100)] project_id, project_auth_token, dp_id, dp_auth_token = self._create_project_and_dp( ) conn, cur = _get_conn_and_cursor() num_entities = 10_000 blocks = [['1'] for _ in range(num_entities)] encodings = [data[i % 100] for i in range(num_entities)] insert_encodings_into_blocks(conn, dp_id, block_ids=blocks, encoding_ids=list(range(num_entities)), encodings=encodings) conn.commit() stored_encoding_ids = list( get_encodingblock_ids(conn, dp_id, '1', offset=10, limit=20)) assert len(stored_encoding_ids) == 20 for i, stored_encoding_id in enumerate(stored_encoding_ids): assert stored_encoding_id == i + 10
def test_binary_unpack_filters(self): encoding_size = 128 filters = [(random.randint(0, 2 ** 32 - 1), generate_bytes(encoding_size)) for _ in range(10)] laundered_filters = binary_unpack_filters(binary_pack_filters(filters, encoding_size), encoding_size=encoding_size) assert filters == laundered_filters
def test_random_bytes(self): rb = generate_bytes(2048//8) srb = serialize_bytes(rb) dsrb = deserialize_bytes(srb) self.assertEqual(dsrb, rb)
def test_random_bytes(self): with temp_file_containing(generate_bytes(128)) as fp: filename = fp.name with pytest.raises(InvalidConfiguration): load_yaml_config(filename)