class Server(object):
"""
High level view of a node instance. This is the object that should be
created to start listening as an active node on the network.
"""
protocol_class = KademliaProtocol
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
# self.storage = storage or ForgetfulStorage()
self.storage = ForgetfulStorage() # PermanentStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
# print('from network:Server node_id =', self.node, type(self.node))
log.debug('node_id=%s type=%s', self.node, type(self.node))
log.debug('storage type: %s', type(self.storage))
def stop(self):
if self.transport is not None:
self.transport.close()
if self.refresh_loop:
self.refresh_loop.cancel()
if self.save_state_loop:
self.save_state_loop.cancel()
def _create_protocol(self):
return self.protocol_class(self.node, self.storage, self.ksize)
def listen(self, port, interface='0.0.0.0'):
"""
Start listening on the given port.
Provide interface="::" to accept ipv6 address
"""
loop = asyncio.get_event_loop()
listen = loop.create_datagram_endpoint(self._create_protocol,
local_addr=(interface, port))
log.info("Node %i listening on %s:%i",
self.node.long_id, interface, port)
self.transport, self.protocol = loop.run_until_complete(listen)
# finally, schedule refreshing table
self.refresh_table()
def refresh_table(self):
log.debug("Refreshing routing table")
asyncio.ensure_future(self._refresh_table())
loop = asyncio.get_event_loop()
self.refresh_loop = loop.call_later(3600, self.refresh_table)
async def _refresh_table(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for node_id in self.protocol.getRefreshIDs():
log.debug('node_id=%s type %s', node_id, type(node_id))
node = Node(node_id)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
ds.append(spider.find())
# do our crawling
await asyncio.gather(*ds)
# now republish keys older than one hour
for dkey, value in self.storage.iteritemsOlderThan(3600):
await self.set_digest(dkey, value)
def bootstrappableNeighbors(self):
"""
Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for
use as an argument to the bootstrap method.
The server should have been bootstrapped
already - this is just a utility for getting some neighbors and then
storing them if this server is going down for a while. When it comes
back up, the list of nodes can be used to bootstrap.
"""
neighbors = self.protocol.router.findNeighbors(self.node)
return [tuple(n)[-2:] for n in neighbors]
async def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP
addresses are acceptable - hostnames will cause an error.
"""
log.debug("Attempting to bootstrap node with %i initial contacts",
len(addrs))
cos = list(map(self.bootstrap_node, addrs))
gathered = await asyncio.gather(*cos)
nodes = [node for node in gathered if node is not None]
spider = NodeSpiderCrawl(self.protocol, self.node, nodes,
self.ksize, self.alpha)
return await spider.find()
async def bootstrap_node(self, addr):
result = await self.protocol.ping(addr, self.node.id)
return Node(result[1], addr[0], addr[1]) if result[0] else None
async def get_digest(self, key):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = key
# if this node has it, return it
if self.storage.get(dkey) is not None:
return self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
# log.debug('found nearest node nearest=%s', nearest)
if len(nearest) == 0:
log.warning("There are no known neighbors to get key %s", key)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
return await spider.find()
async def get(self, key):
"""
Hash query and return if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = digest(key)
# if this node has it, return it
if self.storage.get(dkey) is not None:
return self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
log.warning("There are no known neighbors to get key %s", key)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
return await spider.find()
async def set(self, key, value):
"""
Set the given string key to the given value in the network.
"""
if not check_dht_value_type(value):
raise TypeError(
"Value must be of type int, float, bool, str, or bytes"
)
# log.debug("Setting '%s' = '%s' over kademlia network", key, value)
dkey = digest(key)
return await self.set_digest(dkey, value)
async def set_digest(self, dkey: bytes, value: bytes, store_local=False):
"""
Set the given Keccak digest key (bytes) to the given value in the
network.
"""
log.debug("Setting digest '%s' = '%s' (store_local=%s) over kademlia network", dkey, value, store_local)
node = Node(dkey)
# log.debug("Number of keys in storage %s", self.storage.size)
if store_local:
log.debug("Storing dkey=%s to local storage", dkey)
self.storage[dkey] = value # this saves always to local storage
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
log.warning("There are no known neighbors to set key %s", dkey.hex())
return False
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
nodes = await spider.find()
for n in nodes:
log.debug('Spider found node %s', n)
# log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes)))
# if this node is close too, then store here as well
biggest = max([n.distanceTo(node) for n in nodes])
if self.node.distanceTo(node) < biggest:
self.storage[dkey] = value
for n in nodes:
log.debug("Asking node %s to store key=%s", n, dkey)
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
# return true only if at least one store call succeeded
return any(await asyncio.gather(*ds))
def saveState(self, fname):
"""
Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname.
"""
log.info("Saving state to %s", fname)
data = {
'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.id,
'neighbors': self.bootstrappableNeighbors()
}
if len(data['neighbors']) == 0:
log.warning("No known neighbors, so not writing to cache.")
return
with open(fname, 'wb') as f:
pickle.dump(data, f)
def fillStorage(self, model_id):
"""
Fills missing transactions for model_id by requesting to neighbors
This is necessary to calculate proof-of-traininig and submit to contract
"""
# TODO
pass
@classmethod
def loadState(self, fname):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
log.info("Loading state from %s", fname)
with open(fname, 'rb') as f:
data = pickle.load(f)
s = Server(data['ksize'], data['alpha'], data['id'])
if len(data['neighbors']) > 0:
s.bootstrap(data['neighbors'])
return s
def saveStateRegularly(self, fname, frequency=600):
"""
Save the state of node with a given regularity to the given
filename.
Args:
fname: File name to save retularly to
frequency: Frequency in seconds that the state should be saved.
By default, 10 minutes.
"""
self.saveState(fname)
loop = asyncio.get_event_loop()
self.save_state_loop = loop.call_later(frequency,
self.saveStateRegularly,
fname,
frequency)
def test_forgetting(self): # pylint: disable=no-self-use
storage = ForgetfulStorage(0)
storage['one'] = 'two'
assert storage.get('one') is None
class ForgetfulStorageTests(unittest.TestCase):
def setUp(self):
self.ts = ForgetfulStorage()
@patch('kademlia.storage.time')
def test___setitem_(self, mocked_time):
"""
__setitem__ should set value to storage and add timestamp
__setitem__ should call cull method
"""
mocked_time.monotonic = Mock(return_value=1)
self.ts.cull = Mock()
self.ts['key1'] = 'value1'
self.assertEqual(self.ts.data['key1'], (1, 'value1'))
self.assertTrue(self.ts.cull.called)
def test_cull(self):
"""
cull should remove items older then ttl from storage
"""
self.ts.iteritemsOlderThan = Mock(return_value=[('key1', 'value1')])
self.ts.data['key1'] = (1, 'value1')
self.ts.data['key2'] = (2, 'value2')
self.ts.cull()
self.assertTupleEqual(self.ts.data['key2'], (2, 'value2'))
with self.assertRaises(KeyError):
self.ts.data['key1']
def test_get(self):
"""
Get should get value from storage or return dafault value
Get should call cull method
"""
self.ts.cull = Mock()
self.ts['key1'] = 'value1'
self.assertEqual(self.ts.get('key1'), 'value1')
self.assertTrue(self.ts.cull.called)
self.assertIsNone(self.ts.get('key2'))
self.assertEqual(self.ts.get('key2', 'value2'), 'value2')
self.assertEqual(self.ts.get('key1', 'value2'), 'value1')
def test___getitem__(self):
"""
Get should get value from storage or raise exception
Get should call cull method
"""
self.ts.cull = Mock()
self.ts['key1'] = 'value1'
self.assertEqual(self.ts['key1'], 'value1')
self.assertTrue(self.ts.cull.called)
with self.assertRaises(KeyError):
self.ts['key2']
@patch('kademlia.storage.time')
def test_iteritemsOlderThan(self, mocked_time):
"""
iteritemsOlderThan should return tuples with keys and values
from storage for items not older than specified value
"""
self.ts.cull = Mock()
mocked_time.monotonic = Mock(side_effect=[1, 2, 3, 4])
self.ts['key1'] = 'value1'
self.ts['key2'] = 'value2'
self.ts['key3'] = 'value3'
self.assertEqual(len(self.ts.iteritemsOlderThan(2)), 2)
def test_forgetting(self):
storage = ForgetfulStorage(0)
storage['one'] = 'two'
self.assertEqual(storage.get('one'), None)