async def create(cls,
node_id: DHTID,
bucket_size: int,
depth_modulo: int,
num_replicas: int,
wait_timeout: float,
parallel_rpc: Optional[int] = None,
cache_size: Optional[int] = None,
listen=True,
listen_on='0.0.0.0:*',
endpoint: Optional[Endpoint] = None,
channel_options: Sequence[Tuple[str, Any]] = (),
**kwargs) -> DHTProtocol:
"""
A protocol that allows DHT nodes to request keys/neighbors from other DHT nodes.
As a side-effect, DHTProtocol also maintains a routing table as described in
https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf
See DHTNode (node.py) for a more detailed description.
:note: the rpc_* methods defined in this class will be automatically exposed to other DHT nodes,
for instance, def rpc_ping can be called as protocol.call_ping(endpoint, dht_id) from a remote machine
Only the call_* methods are meant to be called publicly, e.g. from DHTNode
Read more: https://github.com/bmuller/rpcudp/tree/master/rpcudp
"""
self = cls(_initialized_with_create=True)
self.node_id, self.bucket_size, self.num_replicas = node_id, bucket_size, num_replicas
self.wait_timeout, self.channel_options = wait_timeout, tuple(
channel_options)
self.storage, self.cache = DHTLocalStorage(), DHTLocalStorage(
maxsize=cache_size)
self.routing_table = RoutingTable(node_id, bucket_size, depth_modulo)
self.rpc_semaphore = asyncio.Semaphore(
parallel_rpc if parallel_rpc is not None else float('inf'))
if listen: # set up server to process incoming rpc requests
grpc.aio.init_grpc_aio()
self.server = grpc.aio.server(**kwargs,
options=GRPC_KEEPALIVE_OPTIONS)
dht_grpc.add_DHTServicer_to_server(self, self.server)
self.port = self.server.add_insecure_port(listen_on)
assert self.port != 0, f"Failed to listen to {listen_on}"
if endpoint is not None and endpoint.endswith('*'):
endpoint = replace_port(endpoint, self.port)
self.node_info = dht_pb2.NodeInfo(
node_id=node_id.to_bytes(),
rpc_port=self.port,
endpoint=endpoint
or dht_pb2.NodeInfo.endpoint.DESCRIPTOR.default_value)
await self.server.start()
else: # not listening to incoming requests, client-only mode
# note: use empty node_info so peers won't add you to their routing tables
self.node_info, self.server, self.port = dht_pb2.NodeInfo(
), None, None
if listen_on != '0.0.0.0:*' or len(kwargs) != 0:
logger.warning(
f"DHTProtocol has no server (due to listen=False), listen_on"
f"and kwargs have no effect (unused kwargs: {kwargs})")
return self
def test_get_expired():
d = DHTLocalStorage()
d.store(DHTID.generate("key"), b"val", get_dht_time() + 0.1)
time.sleep(0.5)
assert d.get(
DHTID.generate("key")) is None, "Expired value must be deleted"
print("Test get expired passed")
def test_maxsize_cache():
d = DHTLocalStorage(maxsize=1)
d.store(DHTID.generate("key1"), b"val1", get_dht_time() + 1)
d.store(DHTID.generate("key2"), b"val2", get_dht_time() + 200)
assert d.get(DHTID.generate(
"key2"))[0] == b"val2", "Value with bigger exp. time must be kept"
assert d.get(DHTID.generate(
"key1")) is None, "Value with less exp time, must be deleted"
def test_change_expiration_time():
d = DHTLocalStorage()
d.store(DHTID.generate("key"), b"val1", get_dht_time() + 1)
assert d.get(DHTID.generate("key"))[0] == b"val1", "Wrong value"
d.store(DHTID.generate("key"), b"val2", get_dht_time() + 200)
time.sleep(1)
assert d.get(DHTID.generate(
"key"))[0] == b"val2", "Value must be changed, but still kept in table"
print("Test change expiration time passed")
def test_localstorage_freeze():
d = DHTLocalStorage(maxsize=2)
with d.freeze():
d.store(DHTID.generate("key1"), b"val1", get_dht_time() + 0.01)
assert DHTID.generate("key1") in d
time.sleep(0.03)
assert DHTID.generate("key1") in d
assert DHTID.generate("key1") not in d
with d.freeze():
d.store(DHTID.generate("key1"), b"val1", get_dht_time() + 1)
d.store(DHTID.generate("key2"), b"val2", get_dht_time() + 2)
d.store(DHTID.generate("key3"), b"val3",
get_dht_time() + 3) # key3 will push key1 out due to maxsize
assert DHTID.generate("key1") in d
assert DHTID.generate("key1") not in d
def test_localstorage_top():
d = DHTLocalStorage(maxsize=3)
d.store(DHTID.generate("key1"), b"val1", get_dht_time() + 1)
d.store(DHTID.generate("key2"), b"val2", get_dht_time() + 2)
d.store(DHTID.generate("key3"), b"val3", get_dht_time() + 4)
assert d.top()[0] == DHTID.generate("key1") and d.top()[1].value == b"val1"
d.store(DHTID.generate("key1"), b"val1_new", get_dht_time() + 3)
assert d.top()[0] == DHTID.generate("key2") and d.top()[1].value == b"val2"
del d[DHTID.generate('key2')]
assert d.top()[0] == DHTID.generate(
"key1") and d.top()[1].value == b"val1_new"
d.store(DHTID.generate("key2"), b"val2_new", get_dht_time() + 5)
d.store(DHTID.generate("key4"), b"val4",
get_dht_time() + 6) # key4 will push out key1 due to maxsize
assert d.top()[0] == DHTID.generate("key3") and d.top()[1].value == b"val3"
def test_localstorage_nested():
time = get_dht_time()
d1 = DHTLocalStorage()
d2 = DictionaryDHTValue()
d2.store('subkey1', b'value1', time + 2)
d2.store('subkey2', b'value2', time + 3)
d2.store('subkey3', b'value3', time + 1)
assert d2.latest_expiration_time == time + 3
for subkey, (subvalue, subexpiration) in d2.items():
assert d1.store_subkey(DHTID.generate('foo'), subkey, subvalue,
subexpiration)
assert d1.store(DHTID.generate('bar'), b'456', time + 2)
assert d1.get(DHTID.generate('foo'))[0].data == d2.data
assert d1.get(DHTID.generate('foo'))[1] == d2.latest_expiration_time
assert d1.get(DHTID.generate('foo'))[0].get('subkey1') == (b'value1',
time + 2)
assert len(d1.get(DHTID.generate('foo'))[0]) == 3
assert d1.store_subkey(DHTID.generate('foo'), 'subkey4', b'value4',
time + 4)
assert len(d1.get(DHTID.generate('foo'))[0]) == 4
assert d1.store_subkey(DHTID.generate('bar'), 'subkeyA', b'valueA',
time + 1) is False # prev has better expiration
assert d1.store_subkey(DHTID.generate('bar'), 'subkeyA', b'valueA',
time + 3) # new value has better expiration
assert d1.store_subkey(DHTID.generate('bar'), 'subkeyB', b'valueB',
time + 4) # new value has better expiration
assert d1.store_subkey(DHTID.generate('bar'), 'subkeyA', b'valueA+',
time + 5) # overwrite subkeyA under key bar
assert all(subkey in d1.get(DHTID.generate('bar'))[0]
for subkey in ('subkeyA', 'subkeyB'))
assert len(d1.get(DHTID.generate('bar'))[0]) == 2 and d1.get(
DHTID.generate('bar'))[1] == time + 5
assert d1.store(DHTID.generate('foo'), b'nothing', time +
3.5) is False # previous value has better expiration
assert d1.get(DHTID.generate('foo'))[0].get('subkey2') == (b'value2',
time + 3)
assert d1.store(DHTID.generate('foo'), b'nothing',
time + 5) is True # new value has better expiraiton
assert d1.get(DHTID.generate('foo')) == (b'nothing', time + 5
) # value should be replaced
def test_store():
d = DHTLocalStorage()
d.store(DHTID.generate("key"), b"val", get_dht_time() + 0.5)
assert d.get(DHTID.generate("key"))[0] == b"val", "Wrong value"
print("Test store passed")
def test_get_empty():
d = DHTLocalStorage()
assert d.get(DHTID.generate(
source="key")) is None, "DHTLocalStorage returned non-existent value"
print("Test get expired passed")