def __init__(self, key, ksize=20, alpha=3, storage=None): """ Create a server instance. This will start listening on the given port. Args: key: bitcoin wif/hwif to be used as id and for signing/encryption ksize (int): The k parameter from the kademlia paper alpha (int): The alpha parameter from the kademlia paper storage: implements :interface:`~kademlia.storage.IStorage` """ # TODO validate key is valid wif/hwif for mainnet or testnet testnet = False # FIXME get from wif/hwif self._btctxstore = btctxstore.BtcTxStore(testnet=testnet) # allow hwifs is_hwif = self._btctxstore.validate_wallet(key) self._key = self._btctxstore.get_key(key) if is_hwif else key # XXX Server.__init__ cannot call super because of StorjProtocol # passing the protocol class should be added upstream self.ksize = ksize self.alpha = alpha self.log = logging.getLogger(__name__) self.storage = storage or ForgetfulStorage() self.node = Node(self.get_id()) self.protocol = StorjProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None, timeout=5, record=False): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` timeout: Maximum ping time, initial param for RPCProtocol.wait_timeout """ self.ksize = ksize self.alpha = alpha self.storage = storage or ForgetfulStorage() self.node = Node(node_id or digest(random.getrandbits(255))) self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None self.timeout = timeout self.record = record if self.record == True: print('In record mode:')
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None, discovery_mode='neighborhood', loop=None, max_peers=64, dht=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.loop = loop if loop else asyncio.get_event_loop() asyncio.set_event_loop(self.loop) self.ksize = ksize self.alpha = alpha self.port = os.getenv('NETWORK_PORT', 5678) self.storage = storage or ForgetfulStorage() self.node = Node(digest(node_id) or digest(random.getrandbits(255))) self.dht = dht self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None self.max_peers = max_peers self.setup_stethoscope()
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None, custom_event_loop=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.storage = storage or ForgetfulStorage() self.node = Node(node_id or digest(random.getrandbits(255))) self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None self.custom_event_loop = custom_event_loop
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha # self.storage = storage or ForgetfulStorage() self.storage = ForgetfulStorage() # PermanentStorage() self.node = Node(node_id or digest(random.getrandbits(255))) self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None # print('from network:Server node_id =', self.node, type(self.node)) log.debug('node_id=%s type=%s', self.node, type(self.node)) log.debug('storage type: %s', type(self.storage))
def __init__(self, ksize=20, alpha=3, id=None, storage=None): """ Create a server instance. This will start listening on the given port. @param port: UDP port to listen on @param k: The k parameter from the paper @param alpha: The alpha parameter from the paper """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = Node(id or digest(random.getrandbits(255))) self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self, ksize=20, alpha=3, id=None): """ Create a server instance. This will start listening on the given port. @param port: UDP port to listen on @param k: The k parameter from the paper @param alpha: The alpha parameter from the paper """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = ForgetfulStorage() self.node = Node(id or digest(random.getrandbits(255))) self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self, ksize=20, alpha=3, id=None, storage=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = Node(id or digest(random.getrandbits(255))) self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self, key, ksize=20, alpha=3, storage=None, max_messages=1024, default_hop_limit=64, refresh_neighbours_interval=0.0): """ Create a server instance. This will start listening on the given port. Args: key (str): Bitcoin wif/hwif for auth, encryption and node id. ksize (int): The k parameter from the kademlia paper. alpha (int): The alpha parameter from the kademlia paper storage: implements :interface:`~kademlia.storage.IStorage` refresh_neighbours_interval (float): Auto refresh neighbours. """ self._default_hop_limit = default_hop_limit self._refresh_neighbours_interval = refresh_neighbours_interval # TODO validate key is valid wif/hwif for mainnet or testnet testnet = False # FIXME get from wif/hwif self._btctxstore = btctxstore.BtcTxStore(testnet=testnet) # allow hwifs is_hwif = self._btctxstore.validate_wallet(key) self.key = self._btctxstore.get_key(key) if is_hwif else key # XXX Server.__init__ cannot call super because of StorjProtocol # passing the protocol class should be added upstream self.ksize = ksize self.alpha = alpha self.log = logging.getLogger(__name__) self.storage = storage or ForgetfulStorage() self.node = Node(self.get_id()) self.protocol = StorjProtocol(self.node, self.storage, ksize, max_messages=max_messages, max_hop_limit=self._default_hop_limit) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) self._start_threads()
def test_forgetting(self): storage = ForgetfulStorage(0) storage['one'] = 'two' self.assertEqual(storage.get('one'), None)
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ protocol_class = KademliaProtocol def __init__(self, ksize=20, alpha=3, node_id=None, storage=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha # self.storage = storage or ForgetfulStorage() self.storage = ForgetfulStorage() # PermanentStorage() self.node = Node(node_id or digest(random.getrandbits(255))) self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None # print('from network:Server node_id =', self.node, type(self.node)) log.debug('node_id=%s type=%s', self.node, type(self.node)) log.debug('storage type: %s', type(self.storage)) def stop(self): if self.transport is not None: self.transport.close() if self.refresh_loop: self.refresh_loop.cancel() if self.save_state_loop: self.save_state_loop.cancel() def _create_protocol(self): return self.protocol_class(self.node, self.storage, self.ksize) def listen(self, port, interface='0.0.0.0'): """ Start listening on the given port. Provide interface="::" to accept ipv6 address """ loop = asyncio.get_event_loop() listen = loop.create_datagram_endpoint(self._create_protocol, local_addr=(interface, port)) log.info("Node %i listening on %s:%i", self.node.long_id, interface, port) self.transport, self.protocol = loop.run_until_complete(listen) # finally, schedule refreshing table self.refresh_table() def refresh_table(self): log.debug("Refreshing routing table") asyncio.ensure_future(self._refresh_table()) loop = asyncio.get_event_loop() self.refresh_loop = loop.call_later(3600, self.refresh_table) async def _refresh_table(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for node_id in self.protocol.getRefreshIDs(): log.debug('node_id=%s type %s', node_id, type(node_id)) node = Node(node_id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) # do our crawling await asyncio.gather(*ds) # now republish keys older than one hour for dkey, value in self.storage.iteritemsOlderThan(3600): await self.set_digest(dkey, value) def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [tuple(n)[-2:] for n in neighbors] async def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ log.debug("Attempting to bootstrap node with %i initial contacts", len(addrs)) cos = list(map(self.bootstrap_node, addrs)) gathered = await asyncio.gather(*cos) nodes = [node for node in gathered if node is not None] spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return await spider.find() async def bootstrap_node(self, addr): result = await self.protocol.ping(addr, self.node.id) return Node(result[1], addr[0], addr[1]) if result[0] else None async def get_digest(self, key): """ Get a key if the network has it. Returns: :class:`None` if not found, the value otherwise. """ dkey = key # if this node has it, return it if self.storage.get(dkey) is not None: return self.storage.get(dkey) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) # log.debug('found nearest node nearest=%s', nearest) if len(nearest) == 0: log.warning("There are no known neighbors to get key %s", key) return None spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return await spider.find() async def get(self, key): """ Hash query and return if the network has it. Returns: :class:`None` if not found, the value otherwise. """ dkey = digest(key) # if this node has it, return it if self.storage.get(dkey) is not None: return self.storage.get(dkey) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: log.warning("There are no known neighbors to get key %s", key) return None spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return await spider.find() async def set(self, key, value): """ Set the given string key to the given value in the network. """ if not check_dht_value_type(value): raise TypeError( "Value must be of type int, float, bool, str, or bytes" ) # log.debug("Setting '%s' = '%s' over kademlia network", key, value) dkey = digest(key) return await self.set_digest(dkey, value) async def set_digest(self, dkey: bytes, value: bytes, store_local=False): """ Set the given Keccak digest key (bytes) to the given value in the network. """ log.debug("Setting digest '%s' = '%s' (store_local=%s) over kademlia network", dkey, value, store_local) node = Node(dkey) # log.debug("Number of keys in storage %s", self.storage.size) if store_local: log.debug("Storing dkey=%s to local storage", dkey) self.storage[dkey] = value # this saves always to local storage nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: log.warning("There are no known neighbors to set key %s", dkey.hex()) return False spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) nodes = await spider.find() for n in nodes: log.debug('Spider found node %s', n) # log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes))) # if this node is close too, then store here as well biggest = max([n.distanceTo(node) for n in nodes]) if self.node.distanceTo(node) < biggest: self.storage[dkey] = value for n in nodes: log.debug("Asking node %s to store key=%s", n, dkey) ds = [self.protocol.callStore(n, dkey, value) for n in nodes] # return true only if at least one store call succeeded return any(await asyncio.gather(*ds)) def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ log.info("Saving state to %s", fname) data = { 'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'neighbors': self.bootstrappableNeighbors() } if len(data['neighbors']) == 0: log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'wb') as f: pickle.dump(data, f) def fillStorage(self, model_id): """ Fills missing transactions for model_id by requesting to neighbors This is necessary to calculate proof-of-traininig and submit to contract """ # TODO pass @classmethod def loadState(self, fname): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ log.info("Loading state from %s", fname) with open(fname, 'rb') as f: data = pickle.load(f) s = Server(data['ksize'], data['alpha'], data['id']) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequency: Frequency in seconds that the state should be saved. By default, 10 minutes. """ self.saveState(fname) loop = asyncio.get_event_loop() self.save_state_loop = loop.call_later(frequency, self.saveStateRegularly, fname, frequency)
def test_storing(self): storage = ForgetfulStorage(10) storage['one'] = 'two' self.assertEqual(storage['one'], 'two')
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, ksize=20, alpha=3, id=None): """ Create a server instance. This will start listening on the given port. @param port: UDP port to listen on @param k: The k parameter from the paper @param alpha: The alpha parameter from the paper """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = ForgetfulStorage() self.node = Node(id or digest(random.getrandbits(255))) self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) def listen(self, port): """ Start listening on the given port. This is the same as calling: C{reactor.listenUDP(port, server.protocol)} """ return reactor.listenUDP(port, self.protocol) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for id in self.protocol.getRefreshIDs(): node = Node(id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest) ds.append(spider.find()) def republishKeys(_): ds = [] # Republish keys older than one hour for key, value in self.storage.iteritemsOlderThan(3600): ds.append(self.set(key, value)) return defer.gatherResults(ds) return defer.gatherResults(ds).addCallback(republishKeys) def bootstrappableNeighbors(self): """ Get a C{list} of (ip, port) C{tuple}s suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [ tuple(n)[-2:] for n in neighbors ] def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. @param addrs: A C{list} of (ip, port) C{tuple}s. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: nodes.append(Node(result[1], addr[0], addr[1])) spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find() ds = {} for addr in addrs: ds[addr] = self.protocol.ping(addr, self.node.id) return deferredDict(ds).addCallback(initTable) def inetVisibleIP(self): """ Get the internet visible IP's of this node as other nodes see it. @return: An C{list} of IP's. If no one can be contacted, then the C{list} will be empty. """ def handle(results): ips = [ result[1][0] for result in results if result[0] ] self.log.debug("other nodes think our ip is %s" % str(ips)) return ips ds = [] for neighbor in self.bootstrappableNeighbors(): ds.append(self.protocol.stun(neighbor)) return defer.gatherResults(ds).addCallback(handle) def get(self, key): """ Get a key if the network has it. @return: C{None} if not found, the value otherwise. """ node = Node(digest(key)) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to get key %s" % key) return defer.succeed(None) spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find() def set(self, key, value): """ Set the given key to the given value in the network. """ self.log.debug("setting '%s' = '%s' on network" % (key, value)) dkey = digest(key) def store(nodes): self.log.info("setting '%s' on %s" % (key, map(str, nodes))) ds = [self.protocol.callStore(node, dkey, value) for node in nodes] return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to set key %s" % key) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find().addCallback(store) def _anyRespondSuccess(self, responses): """ Given the result of a DeferredList of calls to peers, ensure that at least one of them was contacted and responded with a Truthy result. """ for deferSuccess, result in responses: peerReached, peerResponse = result if deferSuccess and peerReached and peerResponse: return True return False def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ data = { 'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'neighbors': self.bootstrappableNeighbors() } with open(fname, 'w') as f: pickle.dump(data, f) @classmethod def loadState(self, fname): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ with open(fname, 'r') as f: data = pickle.load(f) s = Server(data['ksize'], data['alpha'], data['id']) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. @param fname: File to save retularly to @param frequencey: Frequency in seconds that the state should be saved. By default, 10 minutes. """ loop = LoopingCall(self.saveState, fname) loop.start(frequency) return loop
def test_iter_old(self): # pylint: disable=no-self-use storage = ForgetfulStorage(10) storage['one'] = 'two' for key, value in storage.iter_older_than(0): assert key == 'one' assert value == 'two'
def test_iter_old(self): storage = ForgetfulStorage(10) storage['one'] = 'two' for key, value in storage.iter_older_than(0): self.assertEqual(key, 'one') self.assertEqual(value, 'two')
def setUp(self): self.ts = ForgetfulStorage()
class ForgetfulStorageTests(unittest.TestCase): def setUp(self): self.ts = ForgetfulStorage() @patch('kademlia.storage.time') def test___setitem_(self, mocked_time): """ __setitem__ should set value to storage and add timestamp __setitem__ should call cull method """ mocked_time.monotonic = Mock(return_value=1) self.ts.cull = Mock() self.ts['key1'] = 'value1' self.assertEqual(self.ts.data['key1'], (1, 'value1')) self.assertTrue(self.ts.cull.called) def test_cull(self): """ cull should remove items older then ttl from storage """ self.ts.iteritemsOlderThan = Mock(return_value=[('key1', 'value1')]) self.ts.data['key1'] = (1, 'value1') self.ts.data['key2'] = (2, 'value2') self.ts.cull() self.assertTupleEqual(self.ts.data['key2'], (2, 'value2')) with self.assertRaises(KeyError): self.ts.data['key1'] def test_get(self): """ Get should get value from storage or return dafault value Get should call cull method """ self.ts.cull = Mock() self.ts['key1'] = 'value1' self.assertEqual(self.ts.get('key1'), 'value1') self.assertTrue(self.ts.cull.called) self.assertIsNone(self.ts.get('key2')) self.assertEqual(self.ts.get('key2', 'value2'), 'value2') self.assertEqual(self.ts.get('key1', 'value2'), 'value1') def test___getitem__(self): """ Get should get value from storage or raise exception Get should call cull method """ self.ts.cull = Mock() self.ts['key1'] = 'value1' self.assertEqual(self.ts['key1'], 'value1') self.assertTrue(self.ts.cull.called) with self.assertRaises(KeyError): self.ts['key2'] @patch('kademlia.storage.time') def test_iteritemsOlderThan(self, mocked_time): """ iteritemsOlderThan should return tuples with keys and values from storage for items not older than specified value """ self.ts.cull = Mock() mocked_time.monotonic = Mock(side_effect=[1, 2, 3, 4]) self.ts['key1'] = 'value1' self.ts['key2'] = 'value2' self.ts['key3'] = 'value3' self.assertEqual(len(self.ts.iteritemsOlderThan(2)), 2)
def test_forgetting(self): # pylint: disable=no-self-use storage = ForgetfulStorage(0) storage['one'] = 'two' assert storage.get('one') is None
def test_storing(self): # pylint: disable=no-self-use storage = ForgetfulStorage(10) storage['one'] = 'two' assert storage['one'] == 'two'