class MessageRelayer(object): def __init__(self, server, dest, hop_limit, message): self.server = server self.node = self.server.node self.dest = KademliaNode(dest) self.hop_limit = hop_limit self.message = message self.nearest = None @run_in_reactor def start(self): self.nearest = self.server.protocol.router.findNeighbors( self.dest, exclude=self.server.node ) txt = "{1}: Relaying to nearest peers: {0}" _log.debug(txt.format(repr(self.nearest), self.server.get_address())) self.nearest.reverse() # reverse so you can pop the next self.attempt_relay([True, None]) def __call__(self, result): self.attempt_relay(result) def attempt_relay(self, result): success = bool(result[0] and result[1]) dest_address = storjnode.util.node_id_to_address(self.dest.id) if success: txt = "{1}: Successfully relayed message for {0}" _log.debug(txt.format(dest_address, self.server.get_address())) return # relay only to nearest peer, avoid amplification attacks! elif not self.nearest: txt = "{1}: Failed to relay message for {0}" _log.debug(txt.format(dest_address, self.server.get_address())) return relay_node = self.nearest.pop() address = storjnode.util.node_id_to_address(relay_node.id) # do not relay away from node if self.dest.distanceTo(self.node) <= self.dest.distanceTo(relay_node): txt = "{1}: Aborting relay attempt, {0} farther then self." _log.debug(txt.format(address, self.get_address())) return # attempt to relay message txt = "{1}: Attempting to relay message for {0}" _log.debug(txt.format(address, self.server.get_address())) self.server.protocol.callRelayMessage( relay_node, self.dest.id, self.hop_limit, self.message ).addCallback(self)
def test_distanceCalculation(self): ridone = hashlib.sha1(os.urandom(32)) ridtwo = hashlib.sha1(os.urandom(32)) shouldbe = long(ridone.hexdigest(), 16) ^ long(ridtwo.hexdigest(), 16) none = Node(ridone.digest()) ntwo = Node(ridtwo.digest()) self.assertEqual(none.distanceTo(ntwo), shouldbe)
def test_distanceCalculation(self): ridone = hashlib.sha1(str(random.getrandbits(255))) ridtwo = hashlib.sha1(str(random.getrandbits(255))) shouldbe = long(ridone.hexdigest(), 16) ^ long(ridtwo.hexdigest(), 16) none = Node(ridone.digest()) ntwo = Node(ridtwo.digest()) self.assertEqual(none.distanceTo(ntwo), shouldbe)
def test_distanceCalculation(self): ridone = hashlib.sha1(str(random.getrandbits(255)).encode()) ridtwo = hashlib.sha1(str(random.getrandbits(255)).encode()) shouldbe = int(ridone.hexdigest(), 16) ^ int(ridtwo.hexdigest(), 16) none = Node(ridone.digest()) ntwo = Node(ridtwo.digest()) self.assertEqual(none.distanceTo(ntwo), shouldbe)
def _relay_message(self, entry): """Returns entry if failed to relay to a closer node or None""" dest = KademliaNode(entry["dest"]) nearest = self.protocol.router.findNeighbors(dest, exclude=self.node) _log.debug("Relaying to nearest: %s" % repr(nearest)) for relay_node in nearest: # do not relay away from node if dest.distanceTo(self.node) <= dest.distanceTo(relay_node): msg = "Skipping %s, farther then self." _log.debug(msg % repr(relay_node)) continue # relay message address = storjnode.util.node_id_to_address(relay_node.id) _log.debug("Attempting to relay message for %s" % address) defered = self.protocol.callRelayMessage( relay_node, entry["dest"], entry["hop_limit"], entry["message"] ) defered = storjnode.util.default_defered(defered, None) # wait for relay result try: result = storjnode.util.wait_for_defered(defered, timeout=QUERY_TIMEOUT) except TimeoutError: # pragma: no cover msg = "Timeout while relayed message to %s" # pragma: no cover _log.debug(msg % address) # pragma: no cover result = None # pragma: no cover # successfull relay if result is not None: _log.debug("Successfully relayed message to %s" % address) return # relay to nearest peer, avoid amplification attacks # failed to relay message dest_address = storjnode.util.node_id_to_address(entry["dest"]) _log.debug("Failed to relay message for %s" % dest_address)
def _relay_message(self, entry): """Returns entry if failed to relay to a closer node or None""" dest = Node(entry["dest"]) nearest = self.protocol.router.findNeighbors(dest, exclude=self.node) self.log.debug("Relaying to nearest: %s" % repr(nearest)) for relay_node in nearest: # do not relay away from node if dest.distanceTo(self.node) <= dest.distanceTo(relay_node): msg = "Skipping %s, farther then self." self.log.debug(msg % repr(relay_node)) continue # relay message hexid = binascii.hexlify(relay_node.id) self.log.debug("Attempting to relay message for %s" % hexid) defered = self.protocol.callRelayMessage(relay_node, entry["dest"], entry["hop_limit"], entry["message"]) defered = util.default_defered(defered, None) # wait for relay result try: result = util.wait_for_defered(defered, timeout=QUERY_TIMEOUT) except TimeoutError: # pragma: no cover msg = "Timeout while relayed message to %s" # pragma: no cover self.log.debug(msg % hexid) # pragma: no cover result = None # pragma: no cover # successfull relay if result is not None: self.log.debug("Successfully relayed message to %s" % hexid) return # relay to nearest peer, avoid amplification attacks # failed to relay message dest_hexid = binascii.hexlify(entry["dest"]) self.log.debug("Failed to relay message for %s" % dest_hexid)
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ protocol_class = KademliaProtocol def __init__(self, ksize=20, alpha=3, node_id=None, storage=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha # self.storage = storage or ForgetfulStorage() self.storage = ForgetfulStorage() # PermanentStorage() self.node = Node(node_id or digest(random.getrandbits(255))) self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None # print('from network:Server node_id =', self.node, type(self.node)) log.debug('node_id=%s type=%s', self.node, type(self.node)) log.debug('storage type: %s', type(self.storage)) def stop(self): if self.transport is not None: self.transport.close() if self.refresh_loop: self.refresh_loop.cancel() if self.save_state_loop: self.save_state_loop.cancel() def _create_protocol(self): return self.protocol_class(self.node, self.storage, self.ksize) def listen(self, port, interface='0.0.0.0'): """ Start listening on the given port. Provide interface="::" to accept ipv6 address """ loop = asyncio.get_event_loop() listen = loop.create_datagram_endpoint(self._create_protocol, local_addr=(interface, port)) log.info("Node %i listening on %s:%i", self.node.long_id, interface, port) self.transport, self.protocol = loop.run_until_complete(listen) # finally, schedule refreshing table self.refresh_table() def refresh_table(self): log.debug("Refreshing routing table") asyncio.ensure_future(self._refresh_table()) loop = asyncio.get_event_loop() self.refresh_loop = loop.call_later(3600, self.refresh_table) async def _refresh_table(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for node_id in self.protocol.getRefreshIDs(): log.debug('node_id=%s type %s', node_id, type(node_id)) node = Node(node_id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) # do our crawling await asyncio.gather(*ds) # now republish keys older than one hour for dkey, value in self.storage.iteritemsOlderThan(3600): await self.set_digest(dkey, value) def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [tuple(n)[-2:] for n in neighbors] async def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ log.debug("Attempting to bootstrap node with %i initial contacts", len(addrs)) cos = list(map(self.bootstrap_node, addrs)) gathered = await asyncio.gather(*cos) nodes = [node for node in gathered if node is not None] spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return await spider.find() async def bootstrap_node(self, addr): result = await self.protocol.ping(addr, self.node.id) return Node(result[1], addr[0], addr[1]) if result[0] else None async def get_digest(self, key): """ Get a key if the network has it. Returns: :class:`None` if not found, the value otherwise. """ dkey = key # if this node has it, return it if self.storage.get(dkey) is not None: return self.storage.get(dkey) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) # log.debug('found nearest node nearest=%s', nearest) if len(nearest) == 0: log.warning("There are no known neighbors to get key %s", key) return None spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return await spider.find() async def get(self, key): """ Hash query and return if the network has it. Returns: :class:`None` if not found, the value otherwise. """ dkey = digest(key) # if this node has it, return it if self.storage.get(dkey) is not None: return self.storage.get(dkey) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: log.warning("There are no known neighbors to get key %s", key) return None spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return await spider.find() async def set(self, key, value): """ Set the given string key to the given value in the network. """ if not check_dht_value_type(value): raise TypeError( "Value must be of type int, float, bool, str, or bytes" ) # log.debug("Setting '%s' = '%s' over kademlia network", key, value) dkey = digest(key) return await self.set_digest(dkey, value) async def set_digest(self, dkey: bytes, value: bytes, store_local=False): """ Set the given Keccak digest key (bytes) to the given value in the network. """ log.debug("Setting digest '%s' = '%s' (store_local=%s) over kademlia network", dkey, value, store_local) node = Node(dkey) # log.debug("Number of keys in storage %s", self.storage.size) if store_local: log.debug("Storing dkey=%s to local storage", dkey) self.storage[dkey] = value # this saves always to local storage nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: log.warning("There are no known neighbors to set key %s", dkey.hex()) return False spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) nodes = await spider.find() for n in nodes: log.debug('Spider found node %s', n) # log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes))) # if this node is close too, then store here as well biggest = max([n.distanceTo(node) for n in nodes]) if self.node.distanceTo(node) < biggest: self.storage[dkey] = value for n in nodes: log.debug("Asking node %s to store key=%s", n, dkey) ds = [self.protocol.callStore(n, dkey, value) for n in nodes] # return true only if at least one store call succeeded return any(await asyncio.gather(*ds)) def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ log.info("Saving state to %s", fname) data = { 'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'neighbors': self.bootstrappableNeighbors() } if len(data['neighbors']) == 0: log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'wb') as f: pickle.dump(data, f) def fillStorage(self, model_id): """ Fills missing transactions for model_id by requesting to neighbors This is necessary to calculate proof-of-traininig and submit to contract """ # TODO pass @classmethod def loadState(self, fname): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ log.info("Loading state from %s", fname) with open(fname, 'rb') as f: data = pickle.load(f) s = Server(data['ksize'], data['alpha'], data['id']) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequency: Frequency in seconds that the state should be saved. By default, 10 minutes. """ self.saveState(fname) loop = asyncio.get_event_loop() self.save_state_loop = loop.call_later(frequency, self.saveStateRegularly, fname, frequency)
class Network(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ protocol_class = KademliaProtocol def __init__(self, ksize=20, alpha=3, node_id=None, storage=None, discovery_mode='neighborhood', loop=None, max_peers=64, dht=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.loop = loop if loop else asyncio.get_event_loop() asyncio.set_event_loop(self.loop) self.ksize = ksize self.alpha = alpha self.port = os.getenv('NETWORK_PORT', 5678) self.storage = storage or ForgetfulStorage() self.node = Node(digest(node_id) or digest(random.getrandbits(255))) self.dht = dht self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None self.max_peers = max_peers self.setup_stethoscope() def setup_stethoscope(self): socket.setdefaulttimeout(0.1) self.heartbeat_port = os.getenv('HEARTBEAT_PORT', 31233) self.stethoscope_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.stethoscope_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.stethoscope_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.stethoscope_sock.setblocking(0) self.stethoscope_sock.bind(('0.0.0.0', self.heartbeat_port)) self.stethoscope_sock.listen(self.max_peers) asyncio.ensure_future(self.stethoscope()) async def stethoscope(self): self.connections = {} # self.poll = select.poll() # self.poll.register(self.stethoscope_sock.fileno(), select.POLLIN) self.poll = select.epoll() self.poll.register(self.stethoscope_sock.fileno(), select.EPOLLIN) # | select.EPOLLONESHOT) try: while True: events = self.poll.poll(1) for fileno, event in events: if fileno == self.stethoscope_sock.fileno(): conn, addr = self.stethoscope_sock.accept() log.info("Client (%s, %s) connected to server" % addr) self.dht.status_update( status='client_connect', msg="Client (%s, %s) connected to server" % addr) elif event & (select.EPOLLIN): # | select.EPOLLONESHOT): # elif event & select.POLLIN: conn, addr, node = self.connections[fileno] try: conn.connect(addr) except Exception as e: if e.args[0] == 104: log.info("Client (%s, %s) disconnected" % addr) self.dht.status_update( status='client_disconnect', msg="Client (%s, %s) disconnected" % addr) self.poll.unregister(fileno) conn.close() del self.connections[fileno] self.protocol.router.removeContact(node) await asyncio.sleep(0.1) finally: self.poll.unregister(self.stethoscope_sock.fileno()) self.poll.close() self.stethoscope_sock.close() def connect_to_neighbor(self, node): if self.node.id == node.id: return addr = (node.ip, self.heartbeat_port) conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.connections[conn.fileno()] = (conn, addr, node) try: conn.connect(addr) self.poll.register(conn.fileno(), select.EPOLLIN | select.EPOLLONESHOT) # self.poll.register(conn.fileno(), select.POLLIN) log.info("Client (%s, %s) connected" % addr) except: del self.connections[conn.fileno()] pass async def lookup_ip(self, node_key): node_id = digest(node_key) node = Node(node_id) nearest = self.protocol.router.findNeighbors(node) spider = NodeSpiderCrawl(self.protocol, self.node, nearest, self.ksize, self.alpha) log.debug("Starting lookup for node_key {}".format(node_key)) res_node = await spider.find_ip(node_id=node_id) if type(res_node) == list: res_node = None log.debug('{} resolves to {}'.format(node_key, res_node)) return res_node def stop(self): if self.transport is not None: self.transport.close() if self.refresh_loop: self.refresh_loop.cancel() if self.save_state_loop: self.save_state_loop.cancel() def _create_protocol(self): return self.protocol_class(self.node, self.storage, self.ksize, self) def listen(self, port, interface='0.0.0.0'): """ Start listening on the given port. Provide interface="::" to accept ipv6 address """ listen = self.loop.create_datagram_endpoint(self._create_protocol, local_addr=(interface, port)) log.info("Node %i listening on %s:%i", self.node.long_id, interface, port) self.transport, self.protocol = self.loop.run_until_complete(listen) # finally, schedule refreshing table self.refresh_table() def refresh_table(self): log.debug("Refreshing routing table") asyncio.ensure_future(self._refresh_table()) self.refresh_loop = self.loop.call_later(3600, self.refresh_table) async def _refresh_table(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for node_id in self.protocol.getRefreshIDs(): node = Node(node_id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) # do our crawling await asyncio.gather(*ds) # now republish keys older than one hour for dkey, value in self.storage.iteritemsOlderThan(3600): await self.set_digest(dkey, value) def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [tuple(n)[-2:] for n in neighbors] async def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ log.debug("Attempting to bootstrap node with %i initial contacts", len(addrs)) cos = list(map(self.bootstrap_node, addrs)) gathered = await asyncio.gather(*cos) nodes = [node for node in gathered if node is not None] spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return await spider.find() async def bootstrap_node(self, addr): result = await self.protocol.ping(addr, self.node.id) if result[0]: return Node(result[1], addr[0], addr[1]) async def get(self, key): """ Get a key if the network has it. Returns: :class:`None` if not found, the value otherwise. """ log.info("Looking up key %s", key) dkey = digest(key) # if this node has it, return it if self.storage.get(dkey) is not None: return self.storage.get(dkey) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: log.warning("There are no known neighbors to get key %s", key) return None spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return await spider.find() async def set(self, key, value): """ Set the given string key to the given value in the network. """ if not check_dht_value_type(value): raise TypeError( "Value must be of type int, float, bool, str, or bytes") log.info("setting '%s' = '%s' on network", key, value) dkey = digest(key) return await self.set_digest(dkey, value) async def set_digest(self, dkey, value): """ Set the given sha1 digest key (bytes) to the given value in the network. """ node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: log.warning("There are no known neighbors to set key %s", dkey.hex()) return False spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) nodes = await spider.find() log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes))) # if this node is close too, then store here as well biggest = max([n.distanceTo(node) for n in nodes]) if self.node.distanceTo(node) < biggest: self.storage[dkey] = value ds = [self.protocol.callStore(n, dkey, value) for n in nodes] # return true only if at least one store call succeeded return any(await asyncio.gather(*ds)) def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ log.info("Saving state to %s", fname) data = { 'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'neighbors': self.bootstrappableNeighbors() } if len(data['neighbors']) == 0: log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'wb') as f: pickle.dump(data, f) @classmethod def loadState(self, fname): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ log.info("Loading state from %s", fname) with open(fname, 'rb') as f: data = pickle.load(f) s = Network(data['ksize'], data['alpha'], data['id']) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequency: Frequency in seconds that the state should be saved. By default, 10 minutes. """ self.saveState(fname) self.save_state_loop = self.loop.call_later(frequency, self.saveStateRegularly, fname, frequency)
class TalosDHTServer(object): """ Modified implementation of bmullers DHT for talos High level view of a node instance. This is the object that should be created to start listening as an active node on the network. We assume public ip addresses! No NAT etc """ def __init__(self, ksize=20, alpha=3, id=None, storage=None, talos_vc=None, rebub_delay=3600, tls_port=-1): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or TalosLevelDBDHTStorage("./leveldb") self.node = Node(id or digest(random.getrandbits(255))) def start_looping_call(num_seconds): self.refreshLoop = LoopingCall(self.refreshTable).start(num_seconds) self.delay = rebub_delay task.deferLater(reactor, rebub_delay, start_looping_call, rebub_delay) self.talos_vc = talos_vc or AsyncPolicyApiClient() self.protocol = TalosKademliaProtocol(self.node, self.storage, ksize, talos_vc=self.talos_vc) self.httpprotocol_client = None self.tls_port = tls_port def listen(self, port, interface="127.0.0.1"): """ Init tcp/udp protocol on the given port Start listening on the given port. """ if self.tls_port != -1: root1 = Resource() root2 = Resource() root1.putChild("get_chunk", QueryChunk(self.storage, talos_vc=self.talos_vc)) root2.putChild("storelargechunk", StoreLargeChunk(self.storage, self.protocol, talos_vc=self.talos_vc)) factory1 = Site(root1) factory2 = Site(root2) certData = getModule(__name__).filePath.sibling('server.pem').getContent() certificate = ssl.PrivateCertificate.loadPEM(certData) self.httpprotocol_client = TalosHTTPClient(self.protocol, port) self.protocol.http_client = self.httpprotocol_client reactor.listenTCP(port, factory1, interface=interface) reactor.listenSSL(self.tls_port, factory2, certificate.options(), interface=interface) return reactor.listenUDP(port, self.protocol, interface, maxPacketSize=65535) else: root = Resource() root.putChild("get_chunk", QueryChunk(self.storage, talos_vc=self.talos_vc)) root.putChild("storelargechunk", StoreLargeChunk(self.storage, self.protocol, talos_vc=self.talos_vc)) factory = Site(root) self.httpprotocol_client = TalosHTTPClient(self.protocol, port) self.protocol.http_client = self.httpprotocol_client reactor.listenTCP(port, factory, interface=interface) return reactor.listenUDP(port, self.protocol, interface, maxPacketSize=65535) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ self.log.info("Refreshing table") ds = [] for id in self.protocol.getRefreshIDs(): node = Node(id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) def republishKeys(_): ds = [] # Republish keys older than one hour for dkey, value in self.storage.iteritemsOlderThan(self.delay): ds.append(self.digest_set(digest(dkey), value)) return defer.gatherResults(ds) return defer.gatherResults(ds).addCallback(republishKeys) def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [tuple(n)[-2:] for n in neighbors] def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: nodes.append(Node(result[1], addr[0], addr[1])) spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find() ds = {} for addr in addrs: ds[addr] = self.protocol.ping(addr, self.node.id) return deferredDict(ds).addCallback(initTable) def inetVisibleIP(self): """ Get the internet visible IP's of this node as other nodes see it. Returns: A `list` of IP's. If no one can be contacted, then the `list` will be empty. """ def handle(results): ips = [result[1][0] for result in results if result[0]] self.log.debug("other nodes think our ip is %s" % str(ips)) return ips ds = [] for neighbor in self.bootstrappableNeighbors(): ds.append(self.protocol.stun(neighbor)) return defer.gatherResults(ds).addCallback(handle) def store_chunk(self, chunk, policy=None, time_keeper=TimeKeeper()): dkey = digest(chunk.key) self.log.debug("Storing chunk with key %s" % (binascii.hexlify(dkey),)) result = self.digest_set(dkey, chunk.encode(), policy_in=policy, time_keeper=time_keeper) return result def get_addr_chunk(self, chunk_key, policy_in=None, time_keeper=TimeKeeper()): # if this node has it, return it if self.storage.has_value(chunk_key): addr = self.protocol.get_address() return defer.succeed("%s:%d" % (addr[0], addr[1])) dkey = digest(chunk_key) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) self.log.debug("Crawling for key %s" % (binascii.hexlify(dkey),)) if len(nearest) == 0: self.log.warning("There are no known neighbors to get key %s" % binascii.hexlify(dkey)) return defer.succeed(None) spider = TalosChunkSpiderCrawl(self.protocol, self.httpprotocol_client, node, chunk_key, nearest, self.ksize, self.alpha, time_keeper=time_keeper) return spider.find() def digest_set(self, dkey, value, policy_in=None, time_keeper=TimeKeeper()): """ Set the given SHA1 digest key to the given value in the network. """ node = Node(dkey) # this is useful for debugging messages hkey = binascii.hexlify(dkey) def _anyRespondSuccess(responses, time_keeper, id, name): """ Given the result of a DeferredList of calls to peers, ensure that at least one of them was contacted and responded with a Truthy result. """ time_keeper.stop_clock_unique(name, id) for deferSuccess, result in responses: peerReached, peerResponse = result if deferSuccess and peerReached and peerResponse: return True return False def store(nodes): self.log.info("setting '%s' on %s" % (hkey, map(str, nodes))) # if this node is close too, then store here as well if self.node.distanceTo(node) < max([n.distanceTo(node) for n in nodes]): chunk = CloudChunk.decode(value) if not digest(chunk.key) == dkey: return {'error': 'key missmatch'} def handle_policy(policy): time_keeper.stop_clock(ENTRY_FETCH_POLICY) # Hack no chunk id given -> no key checks, key is in the encoded chunk id = time_keeper.start_clock_unique() self.storage.store_check_chunk(chunk, None, policy, time_keeper=time_keeper) time_keeper.stop_clock_unique(ENTRY_STORE_CHECK, id) id = time_keeper.start_clock_unique() ds = [self.protocol.callStore(n, dkey, value) for n in nodes] return defer.DeferredList(ds).addCallback(_anyRespondSuccess, time_keeper, id, ENTRY_STORE_TO_ALL_NODES) if not policy_in is None: return handle_policy(policy_in) time_keeper.start_clock() return self.talos_vc.get_policy_with_txid(chunk.get_tag_hex()).addCallback(handle_policy) id = time_keeper.start_clock_unique() ds = [self.protocol.callStore(n, dkey, value) for n in nodes] return defer.DeferredList(ds).addCallback(_anyRespondSuccess, time_keeper, id, ENTRY_STORE_TO_ALL_NODES) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to set key %s" % hkey) return defer.succeed(False) spider = TimedNodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha, time_keeper=time_keeper) return spider.find().addCallback(store) def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ self.log.info("Save state to file %s" % fname) data = {'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'neighbors': self.bootstrappableNeighbors()} if len(data['neighbors']) == 0: self.log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'w') as f: pickle.dump(data, f) @classmethod def loadState(self, fname, storage=None, talos_vc=None): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ with open(fname, 'r') as f: data = pickle.load(f) s = TalosDHTServer(data['ksize'], data['alpha'], data['id'], storage=None, talos_vc=None) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequencey: Frequency in seconds that the state should be saved. By default, 10 minutes. """ def run_looping_call(freq): loop = LoopingCall(self.saveState, fname).start(freq) return loop return task.deferLater(reactor, frequency, run_looping_call, frequency)
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, ksize=20, alpha=3, id=None, storage=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = Node(id or digest(random.getrandbits(255))) print(random.getrandbits(255)) self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) def listen(self, port, interface=""): """ Start listening on the given port. This is the same as calling:: reactor.listenUDP(port, server.protocol) Provide interface="::" to accept ipv6 address """ return reactor.listenUDP(port, self.protocol, interface) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for id in self.protocol.getRefreshIDs(): node = Node(id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) def republishKeys(_): ds = [] # Republish keys older than one hour for dkey, value in self.storage.iteritemsOlderThan(3600): ds.append(self.digest_set(dkey, value)) return defer.gatherResults(ds) return defer.gatherResults(ds).addCallback(republishKeys) def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [tuple(n)[-2:] for n in neighbors] def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: nodes.append(Node(result[1], addr[0], addr[1])) spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find() ds = {} for addr in addrs: ds[addr] = self.protocol.ping(addr, self.node.id) return deferredDict(ds).addCallback(initTable) def inetVisibleIP(self): """ Get the internet visible IP's of this node as other nodes see it. Returns: A `list` of IP's. If no one can be contacted, then the `list` will be empty. """ def handle(results): ips = [result[1][0] for result in results if result[0]] self.log.debug("other nodes think our ip is %s" % str(ips)) return ips ds = [] for neighbor in self.bootstrappableNeighbors(): ds.append(self.protocol.stun(neighbor)) return defer.gatherResults(ds).addCallback(handle) def get(self, key): """ Get a key if the network has it. Returns: :class:`None` if not found, the value otherwise. """ dkey = digest(key) # if this node has it, return it if self.storage.get(dkey) is not None: return defer.succeed(self.storage.get(dkey)) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to get key %s" % key) return defer.succeed(None) spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find() def set(self, key, value): """ Set the given key to the given value in the network. """ self.log.debug("setting '%s' = '%s' on network" % (key, value)) dkey = digest(key) return self.digest_set(dkey, value) def digest_set(self, dkey, value): """ Set the given SHA1 digest key to the given value in the network. """ node = Node(dkey) # this is useful for debugging messages hkey = binascii.hexlify(dkey) def store(nodes): self.log.info("setting '%s' on %s" % (hkey, map(str, nodes))) # if this node is close too, then store here as well if self.node.distanceTo(node) < max( [n.distanceTo(node) for n in nodes]): self.storage[dkey] = value ds = [self.protocol.callStore(n, dkey, value) for n in nodes] return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to set key %s" % hkey) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find().addCallback(store) def _anyRespondSuccess(self, responses): """ Given the result of a DeferredList of calls to peers, ensure that at least one of them was contacted and responded with a Truthy result. """ for deferSuccess, result in responses: peerReached, peerResponse = result if deferSuccess and peerReached and peerResponse: return True return False def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ data = { 'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'neighbors': self.bootstrappableNeighbors() } if len(data['neighbors']) == 0: self.log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'w') as f: pickle.dump(data, f) @classmethod def loadState(self, fname): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ with open(fname, 'r') as f: data = pickle.load(f) s = Server(data['ksize'], data['alpha'], data['id']) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequencey: Frequency in seconds that the state should be saved. By default, 10 minutes. """ loop = LoopingCall(self.saveState, fname) loop.start(frequency) return loop
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, ksize=20, alpha=3, id=None, storage=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = Node(id or digest(random.getrandbits(255))) self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) def listen(self, port): """ Start listening on the given port. This is the same as calling:: reactor.listenUDP(port, server.protocol) """ return reactor.listenUDP(port, self.protocol) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for id in self.protocol.getRefreshIDs(): node = Node(id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest) ds.append(spider.find()) def republishKeys(_): ds = [] # Republish keys older than one hour for key, value in self.storage.iteritemsOlderThan(3600): ds.append(self.set(key, value)) return defer.gatherResults(ds) return defer.gatherResults(ds).addCallback(republishKeys) def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [ tuple(n)[-2:] for n in neighbors ] def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: nodes.append(Node(result[1], addr[0], addr[1])) spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find() ds = {} for addr in addrs: ds[addr] = self.protocol.ping(addr, self.node.id) return deferredDict(ds).addCallback(initTable) def inetVisibleIP(self): """ Get the internet visible IP's of this node as other nodes see it. Returns: A `list` of IP's. If no one can be contacted, then the `list` will be empty. """ def handle(results): ips = [ result[1][0] for result in results if result[0] ] self.log.debug("other nodes think our ip is %s" % str(ips)) return ips ds = [] for neighbor in self.bootstrappableNeighbors(): ds.append(self.protocol.stun(neighbor)) return defer.gatherResults(ds).addCallback(handle) def get(self, key): """ Get a key if the network has it. Returns: :class:`None` if not found, the value otherwise. """ dkey = digest(key) # if this node has it, return it if self.storage.get(dkey) is not None: return defer.succeed(self.storage.get(dkey)) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to get key %s" % key) return defer.succeed(None) spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find() def set(self, key, value): """ Set the given key to the given value in the network. """ self.log.debug("setting '%s' = '%s' on network" % (key, value)) dkey = digest(key) node = Node(dkey) def store(nodes): self.log.info("setting '%s' on %s" % (key, map(str, nodes))) # if this node is close too, then store here as well if self.node.distanceTo(node) < max([n.distanceTo(node) for n in nodes]): self.storage[dkey] = value ds = [self.protocol.callStore(n, dkey, value) for n in nodes] return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to set key %s" % key) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find().addCallback(store) def getKnowNeighbours(self): """ Get a list of the known peers in the overlay. """ return self.protocol.router.getRoutingNeighbors(self.node) def _anyRespondSuccess(self, responses): """ Given the result of a DeferredList of calls to peers, ensure that at least one of them was contacted and responded with a Truthy result. """ for deferSuccess, result in responses: peerReached, peerResponse = result if deferSuccess and peerReached and peerResponse: return True return False def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ data = { 'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'neighbors': self.bootstrappableNeighbors() } if len(data['neighbors']) == 0: self.log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'w') as f: pickle.dump(data, f) @classmethod def loadState(self, fname): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ with open(fname, 'r') as f: data = pickle.load(f) s = Server(data['ksize'], data['alpha'], data['id']) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequencey: Frequency in seconds that the state should be saved. By default, 10 minutes. """ loop = LoopingCall(self.saveState, fname) loop.start(frequency) return loop
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ protocol_class = KademliaProtocol def __init__(self, ksize=20, alpha=3, node_id=None, storage=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.storage = storage or ForgetfulStorage() self.node = Node(node_id or digest(random.getrandbits(255))) self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None def stop(self): if self.transport is not None: self.transport.close() if self.refresh_loop: self.refresh_loop.cancel() if self.save_state_loop: self.save_state_loop.cancel() def _create_protocol(self): return self.protocol_class(self.node, self.storage, self.ksize) def listen(self, port, interface='0.0.0.0'): """ Start listening on the given port. Provide interface="::" to accept ipv6 address """ loop = asyncio.get_event_loop() listen = loop.create_datagram_endpoint(self._create_protocol, local_addr=(interface, port)) log.info("Node %i listening on %s:%i", self.node.long_id, interface, port) self.transport, self.protocol = loop.run_until_complete(listen) # finally, schedule refreshing table self.refresh_table() def refresh_table(self): log.debug("Refreshing routing table") asyncio.ensure_future(self._refresh_table()) loop = asyncio.get_event_loop() self.refresh_loop = loop.call_later(3600, self.refresh_table) async def _refresh_table(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for node_id in self.protocol.getRefreshIDs(): node = Node(node_id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) # do our crawling await asyncio.gather(*ds) # now republish keys older than one hour for dkey, value in self.storage.iteritemsOlderThan(3600): await self.set_digest(dkey, value) def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [tuple(n)[-2:] for n in neighbors] async def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ log.debug("Attempting to bootstrap node with %i initial contacts", len(addrs)) cos = list(map(self.bootstrap_node, addrs)) gathered = await asyncio.gather(*cos) nodes = [node for node in gathered if node is not None] spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return await spider.find() async def bootstrap_node(self, addr): result = await self.protocol.ping(addr, self.node.id) return Node(result[1], addr[0], addr[1]) if result[0] else None async def get(self, key): """ Get a key if the network has it. Returns: :class:`None` if not found, the value otherwise. """ log.info("Looking up key %s", key) dkey = digest(key) # if this node has it, return it if self.storage.get(dkey) is not None: return self.storage.get(dkey) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: log.warning("There are no known neighbors to get key %s", key) return None spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return await spider.find() async def set(self, key, value): """ Set the given string key to the given value in the network. """ if not check_dht_value_type(value): raise TypeError( "Value must be of type int, float, bool, str, or bytes" ) log.info("setting '%s' = '%s' on network", key, value) dkey = digest(key) return await self.set_digest(dkey, value) async def set_digest(self, dkey, value): """ Set the given SHA1 digest key (bytes) to the given value in the network. """ node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: log.warning("There are no known neighbors to set key %s", dkey.hex()) return False spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) nodes = await spider.find() log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes))) # if this node is close too, then store here as well biggest = max([n.distanceTo(node) for n in nodes]) if self.node.distanceTo(node) < biggest: self.storage[dkey] = value ds = [self.protocol.callStore(n, dkey, value) for n in nodes] # return true only if at least one store call succeeded return any(await asyncio.gather(*ds)) def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ log.info("Saving state to %s", fname) data = { 'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'neighbors': self.bootstrappableNeighbors() } if len(data['neighbors']) == 0: log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'wb') as f: pickle.dump(data, f) @classmethod def loadState(self, fname): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ log.info("Loading state from %s", fname) with open(fname, 'rb') as f: data = pickle.load(f) s = Server(data['ksize'], data['alpha'], data['id']) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequency: Frequency in seconds that the state should be saved. By default, 10 minutes. """ self.saveState(fname) loop = asyncio.get_event_loop() self.save_state_loop = loop.call_later(frequency, self.saveStateRegularly, fname, frequency)