def test_content_storage_closest_and_furthest_iteration(base_storage):
content_keys = tuple(b"key-" + bytes([i]) for i in range(32))
for idx, content_key in enumerate(content_keys):
base_storage.set_content(content_key, b"dummy-" + bytes([idx]))
target = NodeIDFactory()
expected_closest = tuple(
sorted(
content_keys,
key=lambda content_key: compute_content_distance(
target, content_key_to_content_id(content_key)
),
)
)
expected_furthest = tuple(
sorted(
content_keys,
key=lambda content_key: compute_content_distance(
target, content_key_to_content_id(content_key)
),
reverse=True,
)
)
actual_closest = tuple(base_storage.iter_closest(target))
actual_furthest = tuple(base_storage.iter_furthest(target))
assert actual_closest == expected_closest
assert actual_furthest == expected_furthest
def iter_closest(self, target: NodeID) -> Iterable[ContentKey]:
yield from sorted(
self._db.keys(),
key=lambda content_key: compute_content_distance(
target, content_key_to_content_id(content_key)
),
)
def create(
cls,
content_key: ContentKey,
hash_tree_root: Hash32,
private_key: keys.PrivateKey,
expires_at: Optional[datetime.datetime] = None,
) -> "Advertisement":
if expires_at is None:
expires_at = datetime.datetime.utcnow().replace(
microsecond=0) + ONE_DAY
content_id = content_key_to_content_id(content_key)
signature = create_advertisement_signature(
content_id,
hash_tree_root,
expires_at,
private_key,
)
return cls(
content_key=content_key,
hash_tree_root=hash_tree_root,
expires_at=expires_at,
signature_v=signature.v,
signature_r=signature.r,
signature_s=signature.s,
)
class Params:
private_key = factory.SubFactory(PrivateKeyFactory)
signature = factory.LazyAttribute(
lambda o: create_advertisement_signature(
content_id=content_key_to_content_id(o.content_key),
hash_tree_root=o.hash_tree_root,
expires_at=o.expires_at,
private_key=o.private_key,
))
def content_radius(self) -> int:
if self.is_full:
furthest_key = first(
self.content_storage.iter_furthest(
self._network.local_node_id))
content_id = content_key_to_content_id(furthest_key)
return compute_content_distance(self._network.local_node_id,
content_id)
else:
return 2**256 - 1
def insert_content(
conn: sqlite3.Connection,
content_key: ContentKey,
content: bytes,
) -> None:
content_id = content_key_to_content_id(content_key)
# The high 64 bits of the content id for doing proximate queries
short_content_id = int.from_bytes(content_id, "big") >> 193
with conn:
conn.execute(
STORAGE_INSERT_QUERY,
(content_key, short_content_id, content),
)
async def process_content(self, content_key: ContentKey,
content: bytes) -> None:
if self.content_storage.has_content(content_key):
local_content = self.content_storage.get_content(content_key)
if local_content == content:
self.logger.debug(
"Ignoring content we already have: content_key=%s",
content_key.hex(),
)
return
self.logger.debug(
"Processing content: content_key=%s content=%s",
content_key.hex(),
content.hex(),
)
content_id = content_key_to_content_id(content_key)
# TODO: computationally expensive
hash_tree_root = ssz.get_hash_tree_root(content, sedes=content_sedes)
known_hash_tree_roots = set(
self._local_advertisement_db.get_hash_tree_roots_for_content_id(
content_id, ))
# We should avoid "polution" of our content database with mismatching
# roots. This is a stop gap right now because we will need a mechanism
# for inserting our own "correct" content into the system even in the
# case where the existing "network" content doesn't agree on the hash
# tree root.
if known_hash_tree_roots and hash_tree_root not in known_hash_tree_roots:
known_roots_display = "|".join(
(root.hex() for root in known_hash_tree_roots))
raise NotImplementedError(
f"Content hash tree root mismatch: "
f"content_key={content_key.hex()} root={hash_tree_root.hex()} "
f"known={known_roots_display}")
self.content_storage.set_content(content_key, content, exists_ok=True)
advertisement = self._get_or_create_advertisement(
content_key, hash_tree_root)
await self._network.broadcast(advertisement)
self.logger.debug(
"Processed content: content_key=%s content=%s",
content_key.hex(),
content.hex(),
)
def __init__(
self,
network: AlexandriaNetworkAPI,
content_key: ContentKey,
hash_tree_root: Hash32,
max_node_ids: int = 32,
concurrency: int = 3,
) -> None:
self._network = network
self._concurrency = concurrency
self.content_key = content_key
self.hash_tree_root = hash_tree_root
self.content_id = content_key_to_content_id(content_key)
self.node_queue = ResourceQueue(())
self._content_ready = trio.Event()
def __init__(
self,
network: AlexandriaNetworkAPI,
content_key: ContentKey,
concurrency: int = 3,
) -> None:
self.logger = get_extended_debug_logger("ddht.Seeker")
self.concurrency = concurrency
self.content_key = content_key
self.content_id = content_key_to_content_id(content_key)
self._network = network
self._content_send, self.content_receive = trio.open_memory_channel[
bytes](0)
async def test_alexandria_network_retrieve_content_api(
alice,
tester,
):
async with AsyncExitStack() as stack:
networks = await stack.enter_async_context(
tester.alexandria.network_group(8))
all_node_ids = {alice.node_id
} | {network.local_node_id
for network in networks}
# we need a content_key for which "alice" is the furthest from the content
for i in range(4096):
content_key = b"key-%d" % i
content_id = content_key_to_content_id(content_key)
furthest_node_id = max(
all_node_ids,
key=lambda node_id: compute_content_distance(
node_id, content_id),
)
if furthest_node_id == alice.node_id:
break
# get the closest node and set the content key in their storage
closest_network = min(
networks,
key=lambda network: compute_content_distance(
network.local_node_id, content_id),
)
closest_network.storage.set_content(content_key, b"content-value")
bootnodes = tuple(network.enr_manager.enr for network in networks)
alice_network = await stack.enter_async_context(
alice.alexandria.network(bootnodes=bootnodes))
# give alice a little time to connect to the network as well
with trio.fail_after(20):
for _ in range(1000):
await trio.lowlevel.checkpoint()
with trio.fail_after(10):
content = await alice_network.retrieve_content(content_key)
assert content == b"content-value"
def test_advertisement_creation():
private_key = PrivateKeyFactory()
content_key = b"\x01testkey"
hash_tree_root = b"\x12" * 32
ad = Advertisement.create(content_key, hash_tree_root, private_key)
assert ad.content_id == content_key_to_content_id(content_key)
signature = ad.signature
assert signature.v == ad.signature_v
assert signature.r == ad.signature_r
assert signature.s == ad.signature_s
assert ad.public_key == private_key.public_key
assert ad.is_valid
ad.verify()
expected_node_id = keccak(private_key.public_key.to_bytes())
assert ad.node_id == expected_node_id
def set_content(
self, content_key: ContentKey, content: bytes, exists_ok: bool = False
) -> None:
"""
/content_id.hex()[:2]/content_id.hex()[2:4]/content_id.hex()
"""
if self.has_content(content_key):
if exists_ok:
self.delete_content(content_key)
else:
raise ContentAlreadyExists(
f"Content already exists for key: content_key={content_key.hex()}"
)
content_id = content_key_to_content_id(content_key)
content_id_hex = content_id.hex()
# For some content_id: 0xdeadbeef12345...
# The directory is: <base-dir>/de/ad/deadbeef1234...
content_path = (
self.base_dir / content_id_hex[:2] / content_id_hex[2:4] / content_id_hex
)
content_path_rel = content_path.relative_to(self.base_dir)
# Lazily create the directory structure
content_path.parent.mkdir(parents=True, exist_ok=True)
# We have already checked that the file doesn't exist and that the
# `content_key` is not present in the database, however, there is the
# possibility for a race condition at the filesystem level where the
# file appears between the check and writing to it. In the case of any
# error we want to avoid our filesystem and database being out-of-sync.
try:
with content_path.open("wb") as content_file:
content_file.write(content)
insert_content(self._conn, content_key, len(content), content_path_rel)
except Exception:
content_path.unlink(missing_ok=True)
delete_content(self._conn, content_key)
raise
async def _serve_find_content(self) -> None:
async with self.client.subscribe(FindContentMessage) as subscription:
self._find_content_handler_ready.set()
async for request in subscription:
# if content in storage, serve it....
# else serve ENR records that we know of which are *closest*
content_key = request.message.payload.content_key
if self.storage.has_content(content_key):
content = self.storage.get_content(content_key)
await self.client.send_found_content(
request.sender_node_id,
request.sender_endpoint,
enrs=None,
content=content,
request_id=request.request_id,
)
else:
content_id = content_key_to_content_id(content_key)
distance = compute_content_distance(self.local_node_id, content_id)
try:
response_enrs = self._source_nodes((distance,))
except ValidationError as err:
self.logger.debug(
"Ignoring invalid FindNodesMessage from %s@%s: %s",
request.sender_node_id.hex(),
request.sender_endpoint,
err,
)
else:
await self.client.send_found_content(
request.sender_node_id,
request.sender_endpoint,
enrs=response_enrs,
content=None,
request_id=request.request_id,
)
def content_id(self) -> ContentID:
return content_key_to_content_id(self.content_key)
