def setUp(self):
self.catcher = []
observer = self.catcher.append
log.addObserver(observer)
self.addCleanup(log.removeObserver, observer)
self.node = Node(digest("test"), "127.0.0.1", 1234)
self.router = RoutingTable(self, 20, self.node.id)
def prepare():
"""
Prepare the application state.
"""
thisNodeIP = config.NODE_IP
thisNodePort = config.NODE_PORT
thisNodeHash = Hash(sha1(config.NODE_ID_NAME).digest())
AppState.thisNode = Node(thisNodeHash, (thisNodeIP, thisNodePort))
AppState.heartbeat = config.HEARTBEAT
AppState.tokenSecret = utils.randomBits(160)
AppState.maxPeersPerTorrent = config.MAX_PEERS_PER_TORRENT
AppState.k = config.K
AppState.maxNodesPerPucket = config.MAX_NODES_PER_BUCKET
AppState.routingTable = RoutingTable()
if config.PEER_STORAGE == 'file':
AppState.peerStorage = dht.peerstorage.FilePeerStorage(
config.PEER_STORAGE_DIR)
elif config.PEER_STORAGE == 'mysql':
AppState.peerStorage = dht.peerstorage.MySQLPeerStorage()
# {transactionID: {(RPCQuery, Node, timestamp)}}
AppState.outstandingQueries = {}
def __init__(self, sourceNode, storage, ksize):
self.ksize = ksize
self.router = RoutingTable(self, ksize, sourceNode)
self.storage = storage
self.sourceNode = sourceNode
self.multiplexer = None
self.log = Logger(system=self)
self.handled_commands = [
PING, STUN, STORE, DELETE, FIND_NODE, FIND_VALUE, HOLE_PUNCH
]
RPCProtocol.__init__(self, sourceNode.getProto(), self.router)
async def create(cls,
node_id: DHTID,
bucket_size: int,
depth_modulo: int,
num_replicas: int,
wait_timeout: float,
parallel_rpc: Optional[int] = None,
cache_size: Optional[int] = None,
listen=True,
listen_on='0.0.0.0:*',
channel_options: Optional[Sequence[Tuple[str,
Any]]] = None,
**kwargs) -> DHTProtocol:
"""
A protocol that allows DHT nodes to request keys/neighbors from other DHT nodes.
As a side-effect, DHTProtocol also maintains a routing table as described in
https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf
See DHTNode (node.py) for a more detailed description.
:note: the rpc_* methods defined in this class will be automatically exposed to other DHT nodes,
for instance, def rpc_ping can be called as protocol.call_ping(endpoint, dht_id) from a remote machine
Only the call_* methods are meant to be called publicly, e.g. from DHTNode
Read more: https://github.com/bmuller/rpcudp/tree/master/rpcudp
"""
self = cls(_initialized_with_create=True)
self.node_id, self.bucket_size, self.num_replicas = node_id, bucket_size, num_replicas
self.wait_timeout, self.channel_options = wait_timeout, channel_options
self.storage, self.cache = LocalStorage(), LocalStorage(
maxsize=cache_size)
self.routing_table = RoutingTable(node_id, bucket_size, depth_modulo)
self.rpc_semaphore = asyncio.Semaphore(
parallel_rpc if parallel_rpc is not None else float('inf'))
if listen: # set up server to process incoming rpc requests
grpc.experimental.aio.init_grpc_aio()
self.server = grpc.experimental.aio.server(**kwargs)
dht_grpc.add_DHTServicer_to_server(self, self.server)
found_port = self.server.add_insecure_port(listen_on)
assert found_port != 0, f"Failed to listen to {listen_on}"
self.node_info = dht_pb2.NodeInfo(node_id=node_id.to_bytes(),
rpc_port=found_port)
self.port = found_port
await self.server.start()
else: # not listening to incoming requests, client-only mode
# note: use empty node_info so peers wont add you to their routing tables
self.node_info, self.server, self.port = dht_pb2.NodeInfo(
), None, None
if listen_on != '0.0.0.0:*' or len(kwargs) != 0:
warn(
f"DHTProtocol has no server (due to listen=False), listen_on"
f"and kwargs have no effect (unused kwargs: {kwargs})")
return self
def __init__(self, sourceNode, storage, ksize, database, signing_key):
self.ksize = ksize
self.router = RoutingTable(self, ksize, sourceNode)
self.storage = storage
self.sourceNode = sourceNode
self.multiplexer = None
self.db = database
self.signing_key = signing_key
self.log = Logger(system=self)
self.handled_commands = [PING, STUN, STORE, DELETE, FIND_NODE, FIND_VALUE, HOLE_PUNCH, INV, VALUES]
self.recent_transfers = set()
RPCProtocol.__init__(self, sourceNode, self.router)
def test_routing_table_search():
for table_size, lower_active, upper_active in [(10, 10, 10),
(10_000, 800, 1100)]:
node_id = DHTID.generate()
routing_table = RoutingTable(node_id, bucket_size=20, depth_modulo=5)
num_added = 0
total_nodes = 0
for phony_neighbor_port in random.sample(range(1_000_000), table_size):
routing_table.add_or_update_node(
DHTID.generate(), f'{LOCALHOST}:{phony_neighbor_port}')
new_total = sum(
len(bucket.nodes_to_endpoint)
for bucket in routing_table.buckets)
num_added += new_total > total_nodes
total_nodes = new_total
def test_routing_table_parameters():
for (bucket_size, modulo, min_nbuckets, max_nbuckets) in [
(20, 5, 45, 65),
(50, 5, 35, 45),
(20, 10, 650, 800),
(20, 1, 7, 15),
]:
node_id = DHTID.generate()
routing_table = RoutingTable(node_id,
bucket_size=bucket_size,
depth_modulo=modulo)
for phony_neighbor_port in random.sample(range(1_000_000), 10_000):
routing_table.add_or_update_node(
DHTID.generate(), f'{LOCALHOST}:{phony_neighbor_port}')
for bucket in routing_table.buckets:
assert len(bucket.replacement_nodes) == 0 or len(
bucket.nodes_to_endpoint) <= bucket.size
assert min_nbuckets <= len(routing_table.buckets) <= max_nbuckets, (
f"Unexpected number of buckets: {min_nbuckets} <= {len(routing_table.buckets)} <= {max_nbuckets}"
)
def test_routing_table_basic():
node_id = DHTID.generate()
routing_table = RoutingTable(node_id, bucket_size=20, depth_modulo=5)
added_nodes = []
for phony_neighbor_port in random.sample(range(10000), 100):
phony_id = DHTID.generate()
routing_table.add_or_update_node(phony_id,
f'{LOCALHOST}:{phony_neighbor_port}')
assert phony_id in routing_table
assert f'{LOCALHOST}:{phony_neighbor_port}' in routing_table
assert routing_table[phony_id] == f'{LOCALHOST}:{phony_neighbor_port}'
assert routing_table[f'{LOCALHOST}:{phony_neighbor_port}'] == phony_id
added_nodes.append(phony_id)
assert routing_table.buckets[
0].lower == DHTID.MIN and routing_table.buckets[-1].upper == DHTID.MAX
for bucket in routing_table.buckets:
assert len(
bucket.replacement_nodes
) == 0, "There should be no replacement nodes in a table with 100 entries"
assert 3 <= len(routing_table.buckets) <= 10, len(routing_table.buckets)
random_node = random.choice(added_nodes)
assert routing_table.get(node_id=random_node) == routing_table[random_node]
dummy_node = DHTID.generate()
assert (dummy_node
not in routing_table) == (routing_table.get(node_id=dummy_node) is
None)
for node in added_nodes:
found_bucket_index = routing_table.get_bucket_index(node)
for bucket_index, bucket in enumerate(routing_table.buckets):
if bucket.lower <= node < bucket.upper:
break
else:
raise ValueError(
"Naive search could not find bucket. Universe has gone crazy.")
assert bucket_index == found_bucket_index
def __init__(self, sourceID, ksize=20):
self.router = RoutingTable(self, ksize, Node(sourceID))
self.storage = {}
self.sourceID = sourceID
def setUp(self):
self.node = Node(digest("test"), "127.0.0.1", 1234)
self.router = RoutingTable(self, 20, self.node.id)