def test_get_expired():
d = LocalStorage()
d.store(DHTID.generate("key"), b"val", get_dht_time() + 0.1)
time.sleep(0.5)
assert d.get(
DHTID.generate("key")) == (None, None), "Expired value must be deleted"
print("Test get expired passed")
def test_change_expiration_time():
d = LocalStorage()
d.store(DHTID.generate("key"), b"val1", get_dht_time() + 1)
assert d.get(DHTID.generate("key"))[0] == b"val1", "Wrong value"
d.store(DHTID.generate("key"), b"val2", get_dht_time() + 200)
time.sleep(1)
assert d.get(DHTID.generate("key"))[0] == b"val2", "Value must be changed, but still kept in table"
print("Test change expiration time passed")
def test_maxsize_cache():
d = LocalStorage(maxsize=1)
d.store(DHTID.generate("key1"), b"val1", get_dht_time() + 1)
d.store(DHTID.generate("key2"), b"val2", get_dht_time() + 200)
assert d.get(DHTID.generate(
"key2"))[0] == b"val2", "Value with bigger exp. time must be kept"
assert d.get(DHTID.generate(
"key1"))[0] is None, "Value with less exp time, must be deleted"
def test_localstorage_freeze():
d = LocalStorage(maxsize=2)
with d.freeze():
d.store(DHTID.generate("key1"), b"val1", get_dht_time() + 0.01)
assert DHTID.generate("key1") in d
time.sleep(0.03)
assert DHTID.generate("key1") in d
assert DHTID.generate("key1") not in d
with d.freeze():
d.store(DHTID.generate("key1"), b"val1", get_dht_time() + 1)
d.store(DHTID.generate("key2"), b"val2", get_dht_time() + 2)
d.store(DHTID.generate("key3"), b"val3", get_dht_time() + 3) # key3 will push key1 out due to maxsize
assert DHTID.generate("key1") in d
assert DHTID.generate("key1") not in d
def test_localstorage_top():
d = LocalStorage(maxsize=3)
d.store(DHTID.generate("key1"), b"val1", get_dht_time() + 1)
d.store(DHTID.generate("key2"), b"val2", get_dht_time() + 2)
d.store(DHTID.generate("key3"), b"val3", get_dht_time() + 4)
assert d.top()[:2] == (DHTID.generate("key1"), b"val1")
d.store(DHTID.generate("key1"), b"val1_new", get_dht_time() + 3)
assert d.top()[:2] == (DHTID.generate("key2"), b"val2")
del d[DHTID.generate('key2')]
assert d.top()[:2] == (DHTID.generate("key1"), b"val1_new")
d.store(DHTID.generate("key2"), b"val2_new", get_dht_time() + 5)
d.store(DHTID.generate("key4"), b"val4", get_dht_time() + 6) # key4 will push out key1 due to maxsize
assert d.top()[:2] == (DHTID.generate("key3"), b"val3")
def test_store():
d = LocalStorage()
d.store(DHTID.generate("key"), b"val", get_dht_time() + 0.5)
assert d.get(DHTID.generate("key"))[0] == b"val", "Wrong value"
print("Test store passed")
def test_get_empty():
d = LocalStorage()
assert d.get(DHTID.generate(source="key")) == (None, None), "LocalStorage returned non-existent value"
print("Test get expired passed")
async def create(cls,
node_id: Optional[DHTID] = None,
initial_peers: List[Endpoint] = (),
bucket_size: int = 20,
num_replicas: int = 5,
depth_modulo: int = 5,
parallel_rpc: int = None,
wait_timeout: float = 5,
refresh_timeout: Optional[float] = None,
bootstrap_timeout: Optional[float] = None,
cache_locally: bool = True,
cache_nearest: int = 1,
cache_size=None,
cache_refresh_before_expiry: float = 5,
reuse_get_requests: bool = True,
num_workers: int = 1,
listen: bool = True,
listen_on: Endpoint = "0.0.0.0:*",
**kwargs) -> DHTNode:
"""
:param node_id: current node's identifier, determines which keys it will store locally, defaults to random id
:param initial_peers: connects to these peers to populate routing table, defaults to no peers
:param bucket_size: max number of nodes in one k-bucket (k). Trying to add {k+1}st node will cause a bucket to
either split in two buckets along the midpoint or reject the new node (but still save it as a replacement)
Recommended value: k is chosen s.t. any given k nodes are very unlikely to all fail after staleness_timeout
:param num_replicas: number of nearest nodes that will be asked to store a given key, default = bucket_size (≈k)
:param depth_modulo: split full k-bucket if it contains root OR up to the nearest multiple of this value (≈b)
:param parallel_rpc: maximum number of concurrent outgoing RPC requests emitted by DHTProtocol
Reduce this value if your RPC requests register no response despite the peer sending the response.
:param wait_timeout: a kademlia rpc request is deemed lost if we did not receive a reply in this many seconds
:param refresh_timeout: refresh buckets if no node from that bucket was updated in this many seconds
if staleness_timeout is None, DHTNode will not refresh stale buckets (which is usually okay)
:param bootstrap_timeout: after one of peers responds, await other peers for at most this many seconds
:param cache_locally: if True, caches all values (stored or found) in a node-local cache
:param cache_nearest: whenever DHTNode finds a value, it will also store (cache) this value on this many
nodes nearest nodes visited by search algorithm. Prefers nodes that are nearest to :key: but have no value yet
:param cache_size: if specified, local cache will store up to this many records (as in LRU cache)
:param cache_refresh_before_expiry: if nonzero, refreshes locally cached values
if they are accessed this many seconds before expiration time.
:param reuse_get_requests: if True, DHTNode allows only one traverse_dht procedure for every key
all concurrent get requests for the same key will reuse the procedure that is currently in progress
:param num_workers: concurrent workers in traverse_dht (see traverse_dht num_workers param)
:param listen: if True (default), this node will accept incoming request and otherwise be a DHT "citzen"
if False, this node will refuse any incoming request, effectively being only a "client"
:param listen_on: network interface, e.g. "0.0.0.0:1337" or "localhost:*" (* means pick any port) or "[::]:7654"
:param channel_options: options for grpc.aio.insecure_channel, e.g. [('grpc.enable_retries', 0)]
see https://grpc.github.io/grpc/core/group__grpc__arg__keys.html for a list of all options
:param kwargs: extra parameters used in grpc.aio.server
"""
if cache_refresh_before_expiry > 0 and not cache_locally:
logger.warning(
"If cache_locally is False, cache_refresh_before_expiry has no effect. To silence this"
" warning, please specify cache_refresh_before_expiry=0")
self = cls(_initialized_with_create=True)
self.node_id = node_id = node_id if node_id is not None else DHTID.generate(
)
self.num_replicas, self.num_workers = num_replicas, num_workers
self.is_alive = True # if set to False, cancels all background jobs such as routing table refresh
self.reuse_get_requests = reuse_get_requests
self.pending_get_requests = defaultdict(
partial(SortedList,
key=lambda _res: -_res.sufficient_expiration_time))
# caching policy
self.refresh_timeout = refresh_timeout
self.cache_locally, self.cache_nearest = cache_locally, cache_nearest
self.cache_refresh_before_expiry = cache_refresh_before_expiry
self.cache_refresh_queue = LocalStorage()
self.cache_refresh_available = asyncio.Event()
if cache_refresh_before_expiry:
asyncio.create_task(self._refresh_stale_cache_entries())
self.protocol = await DHTProtocol.create(self.node_id, bucket_size,
depth_modulo, num_replicas,
wait_timeout, parallel_rpc,
cache_size, listen, listen_on,
**kwargs)
self.port = self.protocol.port
if initial_peers:
# stage 1: ping initial_peers, add each other to the routing table
bootstrap_timeout = bootstrap_timeout if bootstrap_timeout is not None else wait_timeout
start_time = get_dht_time()
ping_tasks = map(self.protocol.call_ping, initial_peers)
finished_pings, unfinished_pings = await asyncio.wait(
ping_tasks, return_when=asyncio.FIRST_COMPLETED)
# stage 2: gather remaining peers (those who respond within bootstrap_timeout)
if unfinished_pings:
finished_in_time, stragglers = await asyncio.wait(
unfinished_pings,
timeout=bootstrap_timeout - get_dht_time() + start_time)
for straggler in stragglers:
straggler.cancel()
finished_pings |= finished_in_time
if not finished_pings:
warn(
"DHTNode bootstrap failed: none of the initial_peers responded to a ping."
)
# stage 3: traverse dht to find my own nearest neighbors and populate the routing table
# ... maybe receive some values that we are meant to store (see protocol.update_routing_table)
# note: using asyncio.wait instead of wait_for because wait_for cancels task on timeout
await asyncio.wait([
asyncio.create_task(self.find_nearest_nodes([self.node_id])),
asyncio.sleep(bootstrap_timeout - get_dht_time() + start_time)
],
return_when=asyncio.FIRST_COMPLETED)
if self.refresh_timeout is not None:
asyncio.create_task(
self._refresh_routing_table(period=self.refresh_timeout))
return self