def __init__(self, key, ksize=20, alpha=3, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
key: bitcoin wif/hwif to be used as id and for signing/encryption
ksize (int): The k parameter from the kademlia paper
alpha (int): The alpha parameter from the kademlia paper
storage: implements :interface:`~kademlia.storage.IStorage`
"""
# TODO validate key is valid wif/hwif for mainnet or testnet
testnet = False # FIXME get from wif/hwif
self._btctxstore = btctxstore.BtcTxStore(testnet=testnet)
# allow hwifs
is_hwif = self._btctxstore.validate_wallet(key)
self._key = self._btctxstore.get_key(key) if is_hwif else key
# XXX Server.__init__ cannot call super because of StorjProtocol
# passing the protocol class should be added upstream
self.ksize = ksize
self.alpha = alpha
self.log = logging.getLogger(__name__)
self.storage = storage or ForgetfulStorage()
self.node = Node(self.get_id())
self.protocol = StorjProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self,
ksize=20,
alpha=3,
node_id=None,
storage=None,
timeout=5,
record=False):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
timeout: Maximum ping time, initial param for RPCProtocol.wait_timeout
"""
self.ksize = ksize
self.alpha = alpha
self.storage = storage or ForgetfulStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
self.timeout = timeout
self.record = record
if self.record == True:
print('In record mode:')
def __init__(self,
ksize=20,
alpha=3,
node_id=None,
storage=None,
discovery_mode='neighborhood',
loop=None,
max_peers=64,
dht=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.loop = loop if loop else asyncio.get_event_loop()
asyncio.set_event_loop(self.loop)
self.ksize = ksize
self.alpha = alpha
self.port = os.getenv('NETWORK_PORT', 5678)
self.storage = storage or ForgetfulStorage()
self.node = Node(digest(node_id) or digest(random.getrandbits(255)))
self.dht = dht
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
self.max_peers = max_peers
self.setup_stethoscope()
def __init__(self,
ksize=20,
alpha=3,
node_id=None,
storage=None,
custom_event_loop=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.storage = storage or ForgetfulStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
self.custom_event_loop = custom_event_loop
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
# self.storage = storage or ForgetfulStorage()
self.storage = ForgetfulStorage() # PermanentStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
# print('from network:Server node_id =', self.node, type(self.node))
log.debug('node_id=%s type=%s', self.node, type(self.node))
log.debug('storage type: %s', type(self.storage))
def __init__(self, ksize=20, alpha=3, id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
@param port: UDP port to listen on
@param k: The k parameter from the paper
@param alpha: The alpha parameter from the paper
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = Node(id or digest(random.getrandbits(255)))
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self, ksize=20, alpha=3, id=None):
"""
Create a server instance. This will start listening on the given port.
@param port: UDP port to listen on
@param k: The k parameter from the paper
@param alpha: The alpha parameter from the paper
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = ForgetfulStorage()
self.node = Node(id or digest(random.getrandbits(255)))
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self, ksize=20, alpha=3, id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
id: The id for this node on the network.
storage: An instance that implements :interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = Node(id or digest(random.getrandbits(255)))
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self,
key,
ksize=20,
alpha=3,
storage=None,
max_messages=1024,
default_hop_limit=64,
refresh_neighbours_interval=0.0):
"""
Create a server instance. This will start listening on the given port.
Args:
key (str): Bitcoin wif/hwif for auth, encryption and node id.
ksize (int): The k parameter from the kademlia paper.
alpha (int): The alpha parameter from the kademlia paper
storage: implements :interface:`~kademlia.storage.IStorage`
refresh_neighbours_interval (float): Auto refresh neighbours.
"""
self._default_hop_limit = default_hop_limit
self._refresh_neighbours_interval = refresh_neighbours_interval
# TODO validate key is valid wif/hwif for mainnet or testnet
testnet = False # FIXME get from wif/hwif
self._btctxstore = btctxstore.BtcTxStore(testnet=testnet)
# allow hwifs
is_hwif = self._btctxstore.validate_wallet(key)
self.key = self._btctxstore.get_key(key) if is_hwif else key
# XXX Server.__init__ cannot call super because of StorjProtocol
# passing the protocol class should be added upstream
self.ksize = ksize
self.alpha = alpha
self.log = logging.getLogger(__name__)
self.storage = storage or ForgetfulStorage()
self.node = Node(self.get_id())
self.protocol = StorjProtocol(self.node,
self.storage,
ksize,
max_messages=max_messages,
max_hop_limit=self._default_hop_limit)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
self._start_threads()
def test_forgetting(self):
storage = ForgetfulStorage(0)
storage['one'] = 'two'
self.assertEqual(storage.get('one'), None)
class Server(object):
"""
High level view of a node instance. This is the object that should be
created to start listening as an active node on the network.
"""
protocol_class = KademliaProtocol
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
# self.storage = storage or ForgetfulStorage()
self.storage = ForgetfulStorage() # PermanentStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
# print('from network:Server node_id =', self.node, type(self.node))
log.debug('node_id=%s type=%s', self.node, type(self.node))
log.debug('storage type: %s', type(self.storage))
def stop(self):
if self.transport is not None:
self.transport.close()
if self.refresh_loop:
self.refresh_loop.cancel()
if self.save_state_loop:
self.save_state_loop.cancel()
def _create_protocol(self):
return self.protocol_class(self.node, self.storage, self.ksize)
def listen(self, port, interface='0.0.0.0'):
"""
Start listening on the given port.
Provide interface="::" to accept ipv6 address
"""
loop = asyncio.get_event_loop()
listen = loop.create_datagram_endpoint(self._create_protocol,
local_addr=(interface, port))
log.info("Node %i listening on %s:%i",
self.node.long_id, interface, port)
self.transport, self.protocol = loop.run_until_complete(listen)
# finally, schedule refreshing table
self.refresh_table()
def refresh_table(self):
log.debug("Refreshing routing table")
asyncio.ensure_future(self._refresh_table())
loop = asyncio.get_event_loop()
self.refresh_loop = loop.call_later(3600, self.refresh_table)
async def _refresh_table(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for node_id in self.protocol.getRefreshIDs():
log.debug('node_id=%s type %s', node_id, type(node_id))
node = Node(node_id)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
ds.append(spider.find())
# do our crawling
await asyncio.gather(*ds)
# now republish keys older than one hour
for dkey, value in self.storage.iteritemsOlderThan(3600):
await self.set_digest(dkey, value)
def bootstrappableNeighbors(self):
"""
Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for
use as an argument to the bootstrap method.
The server should have been bootstrapped
already - this is just a utility for getting some neighbors and then
storing them if this server is going down for a while. When it comes
back up, the list of nodes can be used to bootstrap.
"""
neighbors = self.protocol.router.findNeighbors(self.node)
return [tuple(n)[-2:] for n in neighbors]
async def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP
addresses are acceptable - hostnames will cause an error.
"""
log.debug("Attempting to bootstrap node with %i initial contacts",
len(addrs))
cos = list(map(self.bootstrap_node, addrs))
gathered = await asyncio.gather(*cos)
nodes = [node for node in gathered if node is not None]
spider = NodeSpiderCrawl(self.protocol, self.node, nodes,
self.ksize, self.alpha)
return await spider.find()
async def bootstrap_node(self, addr):
result = await self.protocol.ping(addr, self.node.id)
return Node(result[1], addr[0], addr[1]) if result[0] else None
async def get_digest(self, key):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = key
# if this node has it, return it
if self.storage.get(dkey) is not None:
return self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
# log.debug('found nearest node nearest=%s', nearest)
if len(nearest) == 0:
log.warning("There are no known neighbors to get key %s", key)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
return await spider.find()
async def get(self, key):
"""
Hash query and return if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = digest(key)
# if this node has it, return it
if self.storage.get(dkey) is not None:
return self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
log.warning("There are no known neighbors to get key %s", key)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
return await spider.find()
async def set(self, key, value):
"""
Set the given string key to the given value in the network.
"""
if not check_dht_value_type(value):
raise TypeError(
"Value must be of type int, float, bool, str, or bytes"
)
# log.debug("Setting '%s' = '%s' over kademlia network", key, value)
dkey = digest(key)
return await self.set_digest(dkey, value)
async def set_digest(self, dkey: bytes, value: bytes, store_local=False):
"""
Set the given Keccak digest key (bytes) to the given value in the
network.
"""
log.debug("Setting digest '%s' = '%s' (store_local=%s) over kademlia network", dkey, value, store_local)
node = Node(dkey)
# log.debug("Number of keys in storage %s", self.storage.size)
if store_local:
log.debug("Storing dkey=%s to local storage", dkey)
self.storage[dkey] = value # this saves always to local storage
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
log.warning("There are no known neighbors to set key %s", dkey.hex())
return False
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
nodes = await spider.find()
for n in nodes:
log.debug('Spider found node %s', n)
# log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes)))
# if this node is close too, then store here as well
biggest = max([n.distanceTo(node) for n in nodes])
if self.node.distanceTo(node) < biggest:
self.storage[dkey] = value
for n in nodes:
log.debug("Asking node %s to store key=%s", n, dkey)
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
# return true only if at least one store call succeeded
return any(await asyncio.gather(*ds))
def saveState(self, fname):
"""
Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname.
"""
log.info("Saving state to %s", fname)
data = {
'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.id,
'neighbors': self.bootstrappableNeighbors()
}
if len(data['neighbors']) == 0:
log.warning("No known neighbors, so not writing to cache.")
return
with open(fname, 'wb') as f:
pickle.dump(data, f)
def fillStorage(self, model_id):
"""
Fills missing transactions for model_id by requesting to neighbors
This is necessary to calculate proof-of-traininig and submit to contract
"""
# TODO
pass
@classmethod
def loadState(self, fname):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
log.info("Loading state from %s", fname)
with open(fname, 'rb') as f:
data = pickle.load(f)
s = Server(data['ksize'], data['alpha'], data['id'])
if len(data['neighbors']) > 0:
s.bootstrap(data['neighbors'])
return s
def saveStateRegularly(self, fname, frequency=600):
"""
Save the state of node with a given regularity to the given
filename.
Args:
fname: File name to save retularly to
frequency: Frequency in seconds that the state should be saved.
By default, 10 minutes.
"""
self.saveState(fname)
loop = asyncio.get_event_loop()
self.save_state_loop = loop.call_later(frequency,
self.saveStateRegularly,
fname,
frequency)
def test_storing(self):
storage = ForgetfulStorage(10)
storage['one'] = 'two'
self.assertEqual(storage['one'], 'two')
class Server(object):
"""
High level view of a node instance. This is the object that should be created
to start listening as an active node on the network.
"""
def __init__(self, ksize=20, alpha=3, id=None):
"""
Create a server instance. This will start listening on the given port.
@param port: UDP port to listen on
@param k: The k parameter from the paper
@param alpha: The alpha parameter from the paper
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = ForgetfulStorage()
self.node = Node(id or digest(random.getrandbits(255)))
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def listen(self, port):
"""
Start listening on the given port.
This is the same as calling:
C{reactor.listenUDP(port, server.protocol)}
"""
return reactor.listenUDP(port, self.protocol)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for id in self.protocol.getRefreshIDs():
node = Node(id)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest)
ds.append(spider.find())
def republishKeys(_):
ds = []
# Republish keys older than one hour
for key, value in self.storage.iteritemsOlderThan(3600):
ds.append(self.set(key, value))
return defer.gatherResults(ds)
return defer.gatherResults(ds).addCallback(republishKeys)
def bootstrappableNeighbors(self):
"""
Get a C{list} of (ip, port) C{tuple}s suitable for use as an argument
to the bootstrap method.
The server should have been bootstrapped
already - this is just a utility for getting some neighbors and then
storing them if this server is going down for a while. When it comes
back up, the list of nodes can be used to bootstrap.
"""
neighbors = self.protocol.router.findNeighbors(self.node)
return [ tuple(n)[-2:] for n in neighbors ]
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
@param addrs: A C{list} of (ip, port) C{tuple}s. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
nodes.append(Node(result[1], addr[0], addr[1]))
spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
return spider.find()
ds = {}
for addr in addrs:
ds[addr] = self.protocol.ping(addr, self.node.id)
return deferredDict(ds).addCallback(initTable)
def inetVisibleIP(self):
"""
Get the internet visible IP's of this node as other nodes see it.
@return: An C{list} of IP's. If no one can be contacted, then the
C{list} will be empty.
"""
def handle(results):
ips = [ result[1][0] for result in results if result[0] ]
self.log.debug("other nodes think our ip is %s" % str(ips))
return ips
ds = []
for neighbor in self.bootstrappableNeighbors():
ds.append(self.protocol.stun(neighbor))
return defer.gatherResults(ds).addCallback(handle)
def get(self, key):
"""
Get a key if the network has it.
@return: C{None} if not found, the value otherwise.
"""
node = Node(digest(key))
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" % key)
return defer.succeed(None)
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find()
def set(self, key, value):
"""
Set the given key to the given value in the network.
"""
self.log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
def store(nodes):
self.log.info("setting '%s' on %s" % (key, map(str, nodes)))
ds = [self.protocol.callStore(node, dkey, value) for node in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def _anyRespondSuccess(self, responses):
"""
Given the result of a DeferredList of calls to peers, ensure that at least
one of them was contacted and responded with a Truthy result.
"""
for deferSuccess, result in responses:
peerReached, peerResponse = result
if deferSuccess and peerReached and peerResponse:
return True
return False
def saveState(self, fname):
"""
Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname.
"""
data = { 'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.id,
'neighbors': self.bootstrappableNeighbors() }
with open(fname, 'w') as f:
pickle.dump(data, f)
@classmethod
def loadState(self, fname):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
with open(fname, 'r') as f:
data = pickle.load(f)
s = Server(data['ksize'], data['alpha'], data['id'])
if len(data['neighbors']) > 0:
s.bootstrap(data['neighbors'])
return s
def saveStateRegularly(self, fname, frequency=600):
"""
Save the state of node with a given regularity to the given
filename.
@param fname: File to save retularly to
@param frequencey: Frequency in seconds that the state
should be saved. By default, 10 minutes.
"""
loop = LoopingCall(self.saveState, fname)
loop.start(frequency)
return loop
def test_iter_old(self): # pylint: disable=no-self-use
storage = ForgetfulStorage(10)
storage['one'] = 'two'
for key, value in storage.iter_older_than(0):
assert key == 'one'
assert value == 'two'
def test_iter_old(self):
storage = ForgetfulStorage(10)
storage['one'] = 'two'
for key, value in storage.iter_older_than(0):
self.assertEqual(key, 'one')
self.assertEqual(value, 'two')
def setUp(self):
self.ts = ForgetfulStorage()
class ForgetfulStorageTests(unittest.TestCase):
def setUp(self):
self.ts = ForgetfulStorage()
@patch('kademlia.storage.time')
def test___setitem_(self, mocked_time):
"""
__setitem__ should set value to storage and add timestamp
__setitem__ should call cull method
"""
mocked_time.monotonic = Mock(return_value=1)
self.ts.cull = Mock()
self.ts['key1'] = 'value1'
self.assertEqual(self.ts.data['key1'], (1, 'value1'))
self.assertTrue(self.ts.cull.called)
def test_cull(self):
"""
cull should remove items older then ttl from storage
"""
self.ts.iteritemsOlderThan = Mock(return_value=[('key1', 'value1')])
self.ts.data['key1'] = (1, 'value1')
self.ts.data['key2'] = (2, 'value2')
self.ts.cull()
self.assertTupleEqual(self.ts.data['key2'], (2, 'value2'))
with self.assertRaises(KeyError):
self.ts.data['key1']
def test_get(self):
"""
Get should get value from storage or return dafault value
Get should call cull method
"""
self.ts.cull = Mock()
self.ts['key1'] = 'value1'
self.assertEqual(self.ts.get('key1'), 'value1')
self.assertTrue(self.ts.cull.called)
self.assertIsNone(self.ts.get('key2'))
self.assertEqual(self.ts.get('key2', 'value2'), 'value2')
self.assertEqual(self.ts.get('key1', 'value2'), 'value1')
def test___getitem__(self):
"""
Get should get value from storage or raise exception
Get should call cull method
"""
self.ts.cull = Mock()
self.ts['key1'] = 'value1'
self.assertEqual(self.ts['key1'], 'value1')
self.assertTrue(self.ts.cull.called)
with self.assertRaises(KeyError):
self.ts['key2']
@patch('kademlia.storage.time')
def test_iteritemsOlderThan(self, mocked_time):
"""
iteritemsOlderThan should return tuples with keys and values
from storage for items not older than specified value
"""
self.ts.cull = Mock()
mocked_time.monotonic = Mock(side_effect=[1, 2, 3, 4])
self.ts['key1'] = 'value1'
self.ts['key2'] = 'value2'
self.ts['key3'] = 'value3'
self.assertEqual(len(self.ts.iteritemsOlderThan(2)), 2)
def test_forgetting(self): # pylint: disable=no-self-use
storage = ForgetfulStorage(0)
storage['one'] = 'two'
assert storage.get('one') is None
def test_storing(self): # pylint: disable=no-self-use
storage = ForgetfulStorage(10)
storage['one'] = 'two'
assert storage['one'] == 'two'
