def rpc_find_node(self, sender, nodeid, key):
log.info("finding neighbors of %i in local table",
int(nodeid.hex(), 16))
source = Node(nodeid, sender[0], sender[1])
self.welcomeIfNewNode(source)
node = Node(key)
neighbors = self.router.findNeighbors(node, exclude=source)
return list(map(tuple, neighbors))
def test_distanceCalculation(self):
ridone = hashlib.sha1(str(random.getrandbits(255)).encode())
ridtwo = hashlib.sha1(str(random.getrandbits(255)).encode())
shouldbe = int(ridone.hexdigest(), 16) ^ int(ridtwo.hexdigest(), 16)
none = Node(ridone.digest())
ntwo = Node(ridtwo.digest())
self.assertEqual(none.distanceTo(ntwo), shouldbe)
def test_distance_calculation(self):
ridone = hashlib.sha1(str(random.getrandbits(255)).encode())
ridtwo = hashlib.sha1(str(random.getrandbits(255)).encode())
shouldbe = int(ridone.hexdigest(), 16) ^ int(ridtwo.hexdigest(), 16)
none = Node(ridone.digest())
ntwo = Node(ridtwo.digest())
self.assertEqual(none.distance_to(ntwo), shouldbe)
def test_distanceCalculation(self):
ridone = hashlib.sha1(os.urandom(32))
ridtwo = hashlib.sha1(os.urandom(32))
shouldbe = long(ridone.hexdigest(), 16) ^ long(ridtwo.hexdigest(), 16)
none = Node(ridone.digest())
ntwo = Node(ridtwo.digest())
self.assertEqual(none.distanceTo(ntwo), shouldbe)
def test_distance_calculation(self): # pylint: disable=no-self-use
ridone = hashlib.sha1(str(random.getrandbits(255)).encode())
ridtwo = hashlib.sha1(str(random.getrandbits(255)).encode())
shouldbe = int(ridone.hexdigest(), 16) ^ int(ridtwo.hexdigest(), 16)
none = Node(ridone.digest())
ntwo = Node(ridtwo.digest())
assert none.distance_to(ntwo) == shouldbe
def rpc_find_node(self, sender, nodeid, key):
self.log.info("finding neighbors of %i in local table" %
long(nodeid.encode('hex'), 16))
source = Node(nodeid, sender[0], sender[1])
_log.debug("rpc_find_node sender=%s, source=%s, key=%s" %
(sender, source, base64.b64encode(key)))
self.maybeTransferKeyValues(source)
self.router.addContact(source)
node = Node(key)
return map(tuple, self.router.findNeighbors(node, exclude=source))
def rpc_find_node(self, sender, nodeid, key, puzzle):
if self.verify_integrity(sender, nodeid, puzzle):
log.info("finding neighbors of %i in local table",
int(nodeid.hex(), 16))
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
node = Node(key)
neighbors = self.router.find_neighbors(node, exclude=source)
return list(map(tuple, neighbors))
return False
def poison_routing_tables(self):
fakeid = hashlib.sha1(str(random.getrandbits(255))).digest()
self.neighbours = map(list,
self.router.findNeighbors(Node(fakeid), k=20))
my_randoms = random.sample(xrange(len(self.neighbours)), 1)
for nodeToAttack in my_randoms:
for nodeToImpersonate in range(0, len(self.neighbours)):
if nodeToImpersonate != nodeToAttack:
node = Node(self.neighbours[nodeToAttack][0],
self.neighbours[nodeToAttack][1],
self.neighbours[nodeToAttack][2])
self.callPing(node, self.neighbours[nodeToImpersonate][0])
class MessageRelayer(object):
def __init__(self, server, dest, hop_limit, message):
self.server = server
self.node = self.server.node
self.dest = KademliaNode(dest)
self.hop_limit = hop_limit
self.message = message
self.nearest = None
@run_in_reactor
def start(self):
self.nearest = self.server.protocol.router.findNeighbors(
self.dest, exclude=self.server.node
)
txt = "{1}: Relaying to nearest peers: {0}"
_log.debug(txt.format(repr(self.nearest), self.server.get_address()))
self.nearest.reverse() # reverse so you can pop the next
self.attempt_relay([True, None])
def __call__(self, result):
self.attempt_relay(result)
def attempt_relay(self, result):
success = bool(result[0] and result[1])
dest_address = storjnode.util.node_id_to_address(self.dest.id)
if success:
txt = "{1}: Successfully relayed message for {0}"
_log.debug(txt.format(dest_address, self.server.get_address()))
return # relay only to nearest peer, avoid amplification attacks!
elif not self.nearest:
txt = "{1}: Failed to relay message for {0}"
_log.debug(txt.format(dest_address, self.server.get_address()))
return
relay_node = self.nearest.pop()
address = storjnode.util.node_id_to_address(relay_node.id)
# do not relay away from node
if self.dest.distanceTo(self.node) <= self.dest.distanceTo(relay_node):
txt = "{1}: Aborting relay attempt, {0} farther then self."
_log.debug(txt.format(address, self.get_address()))
return
# attempt to relay message
txt = "{1}: Attempting to relay message for {0}"
_log.debug(txt.format(address, self.server.get_address()))
self.server.protocol.callRelayMessage(
relay_node, self.dest.id, self.hop_limit, self.message
).addCallback(self)
def test_bucket_basic(self):
ecd_key = generate_secret_key()
sourceNode = Node(digest(random.getrandbits(255)), ip="127.0.0.1", port=12345)
dummy_protocol = DummyProt(ecd_key, sourceNode, None, 4, talos_vc=None)
nodes = []
for i in range(1000):
nodes.append(Node(digest(random.getrandbits(255)), ip="127.0.0.1", port=i+10000))
for i in range(1000):
dummy_protocol.router.addContact(nodes[i])
for i in range(1000):
self.assertFalse(dummy_protocol.router.isNewNode(nodes[i]))
async def bootstrap_node(self, addr):
puz = solve_puzzle(self.node.id)
log.debug('sending puzzle during bootstrap: %s', puz.hex())
result = await self.protocol.ping(addr, self.node.id, puz)
if result[0] and result[1]:
return Node(result[1], addr[0], addr[1])
return None
def transferKeyValues(self, node):
"""
Given a new node, send it all the keys/values it should be storing.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
ds = []
for key, value in self.storage.iteritems():
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
if len(neighbors) > 0:
newNodeClose = node.distanceTo(
keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(
keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
ds.append(self.callStore(node, key, value))
return defer.gatherResults(ds)
def set(self, key, value):
"""
Set the given key to the given value in the network.
"""
_log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
node = Node(dkey)
def store(nodes):
_log.debug("setting '%s' to %s on %s" %
(key, value, map(str, nodes)))
# if this node is close too, then store here as well
if (not nodes or self.node.distanceTo(node) < max(
[n.distanceTo(node) for n in nodes]) or dkey in self.storage):
_log.debug("setting '%s' to %s locally" % (key, value))
self.storage[dkey] = value
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
_log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.alpha)
return spider.find().addCallback(store)
def rpc_append(self, sender, mid, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
self.router.addContact(source)
self.log.debug("got a append request from %s, storing value" %
str(sender))
try:
pvalue = json.loads(value)
if key not in self.storage:
self.storage[key] = value
self.storage[mid] = True
else:
if mid in self.storage:
return False
old_value = json.loads(self.storage[key])
# TODO: What happens if we dont have list ?
self.storage[key] = json.dumps(old_value + pvalue)
self.storage[mid] = True
return True
except:
import traceback
traceback.print_exc()
return False
def append(self, key, value):
"""
Append the given key to the given value in the network.
"""
self.log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
def append(nodes, mid):
self.log.info("setting '%s' on %s" % (key, map(str, nodes)))
# TODO: Must add transaction ID so we dont append multiple times.
print "org mid", mid
mid = uuid.uuid1().hex
print "new mid", mid
ds = [self.protocol.callAppend(node, mid, dkey, value) for node in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(append, "hej")
def get_addr_chunk(self,
chunk_key,
policy_in=None,
time_keeper=TimeKeeper()):
# if this node has it, return it
if self.storage.has_value(chunk_key):
addr = self.protocol.get_address()
return defer.succeed("%s:%d" % (addr[0], addr[1]))
dkey = digest(chunk_key)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
self.log.debug("Crawling for key %s" % (binascii.hexlify(dkey), ))
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" %
binascii.hexlify(dkey))
return defer.succeed(None)
spider = TalosChunkSpiderCrawl(self.protocol,
self.httpprotocol_client,
node,
chunk_key,
nearest,
self.ksize,
self.alpha,
time_keeper=time_keeper)
return spider.find()
def rpc_find_value(self, sender, nodeid, key):
source = Node(nodeid, sender[0], sender[1])
self.router.addContact(source)
value = self.storage.get(key, None)
if value is None:
return self.rpc_find_node(sender, nodeid, key)
return { 'value': value }
def __init__(self, server, dest, hop_limit, message):
self.server = server
self.node = self.server.node
self.dest = KademliaNode(dest)
self.hop_limit = hop_limit
self.message = message
self.nearest = None
def rpc_store(self, sender, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
self.router.addContact(source)
self.log.debug("got a store request from %s, storing value" %
str(sender))
self.storage[key] = value
return True
async def set_digest(self, dkey, value):
"""
Set the given SHA1 digest key (bytes) to the given value in the
network.
"""
node = Node(dkey)
nearest = self.protocol.router.find_neighbors(node)
if not nearest:
log.warning("There are no known neighbors to set key %s",
dkey.hex())
return False
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
nodes = await spider.find()
log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes)))
# if this node is close too, then store here as well
biggest = max([n.distance_to(node) for n in nodes])
if self.node.distance_to(node) < biggest:
self.storage[dkey] = value
results = [self.protocol.call_store(n, dkey, value) for n in nodes]
# return true only if at least one store call succeeded
return any(await asyncio.gather(*results))
def __init__(self, ksize=20, alpha=3, priv_key=None, storage=None,
talos_vc=None, rebub_delay=3600, c1bits=1, tls_port=-1):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
id: The id for this node on the network.
storage: An instance that implements :interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or TalosLevelDBDHTStorage("./leveldb")
self.c1bits = c1bits
if priv_key is None:
self.priv_key, node_id = generate_keys_with_crypto_puzzle(c1bits)
else:
self.priv_key = priv_key
node_id = pub_to_node_id(self.priv_key.public_key())
self.node = Node(node_id)
def start_looping_call(num_seconds):
self.refreshLoop = LoopingCall(self.refreshTable).start(num_seconds)
self.delay = rebub_delay
task.deferLater(reactor, rebub_delay, start_looping_call, rebub_delay)
self.talos_vc = talos_vc or AsyncPolicyApiClient()
self.protocol = TalosSKademliaProtocol(self.priv_key, self.node,
self.storage, ksize, talos_vc=self.talos_vc, cbits=c1bits)
self.httpprotocol_client = None
self.tls_port = tls_port
def get_node_list(self):
"""
Get the node list in the response. If there's no value, this should
be set.
"""
nodelist = self.response[1] or []
return [Node(*nodeple) for nodeple in nodelist]
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
nodes.append(Node(result[1], addr[0], addr[1]))
spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
return spider.find()
def get_concat(self, key):
"""
Get a key if the network has it. Assuming it is a list that should be combined.
@return: C{None} if not found, the value otherwise.
"""
dkey = digest(key)
# Always try to do a find even if we have it, due to the concatenation of all results
exists, value = self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
_log.debug(
"Server:get_concat key=%s, value=%s, exists=%s, nbr nearest=%d" %
(base64.b64encode(dkey), value, exists, len(nearest)))
if len(nearest) == 0:
# No neighbors but we had it, return that value
if exists:
return defer.succeed(value)
self.log.warning("There are no known neighbors to get key %s" %
key)
return defer.succeed(None)
spider = ValueListSpiderCrawl(self.protocol,
node,
nearest,
self.ksize,
self.alpha,
local_value=value if exists else None)
return spider.find()
def rpc_append(self, sender, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
self.router.addContact(source)
try:
pvalue = json.loads(value)
self.set_keys.add(key)
if key not in self.storage:
_log.debug(
"%s append key: %s not in storage set value: %s" %
(base64.b64encode(nodeid), base64.b64encode(key), pvalue))
self.storage[key] = value
else:
old_value_ = self.storage[key]
old_value = json.loads(old_value_)
new_value = list(set(old_value + pvalue))
_log.debug("%s append key: %s old: %s add: %s new: %s" %
(base64.b64encode(nodeid), base64.b64encode(key),
old_value, pvalue, new_value))
self.storage[key] = json.dumps(new_value)
return True
except:
_log.debug("Trying to append somthing not a JSON coded list %s" %
value,
exc_info=True)
return False
def transferKeyValues(self, node):
"""
Given a new node, send it all the keys/values it should be storing.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
_log.debug("**** transfer key values %s ****" % node)
ds = []
for key, value in self.storage.iteritems():
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
_log.debug("transfer? nbr neighbors=%d, key=%s, value=%s" %
(len(neighbors), base64.b64encode(key), str(value)))
if len(neighbors) > 0:
newNodeClose = node.distanceTo(
keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(
keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
if key in self.set_keys:
_log.debug("transfer append key value key=%s, value=%s" %
(base64.b64encode(key), str(value)))
ds.append(self.callAppend(node, key, value))
else:
_log.debug("transfer store key value key=%s, value=%s" %
(base64.b64encode(key), str(value)))
ds.append(self.callStore(node, key, value))
return defer.gatherResults(ds)
def rpc_remove(self, sender, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
_log.debug("rpc_remove sender=%s, source=%s, key=%s, value=%s" %
(sender, source, base64.b64encode(key), str(value)))
self.maybeTransferKeyValues(source)
self.router.addContact(source)
try:
pvalue = json.loads(value)
self.set_keys.add(key)
if key in self.storage:
old_value = json.loads(self.storage[key])
new_value = list(set(old_value) - set(pvalue))
self.storage[key] = json.dumps(new_value)
_log.debug("%s remove key: %s old: %s remove: %s new: %s" %
(base64.b64encode(nodeid), base64.b64encode(key),
old_value, pvalue, new_value))
return True
except:
_log.debug("Trying to remove somthing not a JSON coded list %s" %
value,
exc_info=True)
return False
def welcomeIfNewNode(self, node):
"""
Given a new node, send it all the keys/values it should be storing,
then add it to the routing table.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
if not self.router.isNewNode(node):
return
self.log.info(
"never seen %s before, adding to router and setting nearby " %
node)
# TODO: 331 and 340 next two lines
ursulas = [(id, bytes(node))
for id, node in self.sourceNode._node_storage.items()]
for key, value in tuple(
self.sourceNode._treasure_maps.items()) + tuple(ursulas):
keynode = Node(digest(key))
neighbors = self.router.findNeighbors(keynode)
if len(neighbors) > 0:
newNodeClose = node.distanceTo(
keynode) < neighbors[-1].distanceTo(keynode)
thisNodeClosest = self.sourceNode.distanceTo(
keynode) < neighbors[0].distanceTo(keynode)
if len(neighbors) == 0 or (newNodeClose and thisNodeClosest):
asyncio.ensure_future(self.callStore(node, key, value))
self.router.addContact(node)
def rpc_store(self, sender, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
self.welcomeIfNewNode(source)
self.log.debug(
"got a store request from %s, but THIS VALUE WILL NOT BE STORED as this is a seed-only node." % str(
sender))
return True
def digest_set(self, dkey, value):
"""
Set the given SHA1 digest key to the given value in the network.
"""
node = Node(dkey)
# this is useful for debugging messages
hkey = binascii.hexlify(dkey)
def store(nodes):
self.log.info("setting '%s' on %s" % (hkey, map(str, nodes)))
# if this node is close too, then store here as well
if self.node.distanceTo(node) < max(
[n.distanceTo(node) for n in nodes]):
self.storage[dkey] = value
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" %
hkey)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.alpha)
return spider.find().addCallback(store)
def rpc_find_value(self, sender, nodeid, key):
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
value = self.storage.get(key, None)
if value is None:
return self.rpc_find_node(sender, nodeid, key)
return {'value': value}
def welcome_if_new(self, node):
"""
Given a new node, send it all the keys/values it should be storing,
then add it to the routing table.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
if not self.router.is_new_node(node):
return
log.info("never seen %s before, adding to router", node)
for key, value in self.storage:
keynode = Node(digest(key))
neighbors = self.router.find_neighbors(keynode)
if neighbors:
last = neighbors[-1].distance_to(keynode)
new_node_close = node.distance_to(keynode) < last
first = neighbors[0].distance_to(keynode)
this_closest = self.source_node.distance_to(keynode) < first
if not neighbors or (new_node_close and this_closest):
asyncio.ensure_future(self.call_store(node, key, value))
self.router.add_contact(node)
def rpc_store(self, sender, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
log.debug("got a store request from %s, storing '%s'='%s'", sender,
key.hex(), value)
self.storage[key] = value
return True
def rpc_append(self, sender, nodeid, key, value):
source = Node(nodeid, sender[0], sender[1])
_log.debug("rpc_append sender=%s, source=%s, key=%s, value=%s" % (sender, source, base64.b64encode(key), str(value)))
self.maybeTransferKeyValues(source)
self.router.addContact(source)
try:
pvalue = json.loads(value)
self.set_keys.add(key)
if key not in self.storage:
_log.debug("%s append key: %s not in storage set value: %s" % (base64.b64encode(nodeid), base64.b64encode(key), pvalue))
self.storage[key] = value
else:
old_value_ = self.storage[key]
try:
old_value = json.loads(old_value_)
new_value = list(set(old_value + pvalue))
except:
# When the key have been used for single values it does not contain a list
# When have been deleted contains None
# Just replace old value
old_value = old_value_
new_value = pvalue
_log.debug("%s append key: %s old: %s add: %s new: %s" % (base64.b64encode(nodeid), base64.b64encode(key), old_value, pvalue, new_value))
self.storage[key] = json.dumps(new_value)
return True
except:
_log.debug("Trying to append something not a JSON coded list %s" % value, exc_info=True)
return False
def _relay_message(self, entry):
"""Returns entry if failed to relay to a closer node or None"""
dest = KademliaNode(entry["dest"])
nearest = self.protocol.router.findNeighbors(dest, exclude=self.node)
_log.debug("Relaying to nearest: %s" % repr(nearest))
for relay_node in nearest:
# do not relay away from node
if dest.distanceTo(self.node) <= dest.distanceTo(relay_node):
msg = "Skipping %s, farther then self."
_log.debug(msg % repr(relay_node))
continue
# relay message
address = storjnode.util.node_id_to_address(relay_node.id)
_log.debug("Attempting to relay message for %s" % address)
defered = self.protocol.callRelayMessage(
relay_node, entry["dest"], entry["hop_limit"], entry["message"]
)
defered = storjnode.util.default_defered(defered, None)
# wait for relay result
try:
result = storjnode.util.wait_for_defered(defered,
timeout=QUERY_TIMEOUT)
except TimeoutError: # pragma: no cover
msg = "Timeout while relayed message to %s" # pragma: no cover
_log.debug(msg % address) # pragma: no cover
result = None # pragma: no cover
# successfull relay
if result is not None:
_log.debug("Successfully relayed message to %s" % address)
return # relay to nearest peer, avoid amplification attacks
# failed to relay message
dest_address = storjnode.util.node_id_to_address(entry["dest"])
_log.debug("Failed to relay message for %s" % dest_address)
def __init__(self, ksize=20, alpha=3, id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
id: The id for this node on the network.
storage: An instance that implements :interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = Node(id or digest(random.getrandbits(255)))
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.storage = storage or ForgetfulStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
#!/usr/bin/env python from kademlia.node import Node print "Starting dashpot..." node = Node() node.joinNetwork() print "Shutting down..."
class Server(object):
"""
High level view of a node instance. This is the object that should be created
to start listening as an active node on the network.
"""
def __init__(self, ksize=20, alpha=3, id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
id: The id for this node on the network.
storage: An instance that implements :interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.log = Logger(system=self)
self.storage = storage or ForgetfulStorage()
self.node = Node(id or digest(random.getrandbits(255)))
self.protocol = KademliaProtocol(self.node, self.storage, ksize)
self.refreshLoop = LoopingCall(self.refreshTable).start(3600)
def listen(self, port):
"""
Start listening on the given port.
This is the same as calling::
reactor.listenUDP(port, server.protocol)
"""
return reactor.listenUDP(port, self.protocol)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for id in self.protocol.getRefreshIDs():
node = Node(id)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest)
ds.append(spider.find())
def republishKeys(_):
ds = []
# Republish keys older than one hour
for key, value in self.storage.iteritemsOlderThan(3600):
ds.append(self.set(key, value))
return defer.gatherResults(ds)
return defer.gatherResults(ds).addCallback(republishKeys)
def bootstrappableNeighbors(self):
"""
Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument
to the bootstrap method.
The server should have been bootstrapped
already - this is just a utility for getting some neighbors and then
storing them if this server is going down for a while. When it comes
back up, the list of nodes can be used to bootstrap.
"""
neighbors = self.protocol.router.findNeighbors(self.node)
return [ tuple(n)[-2:] for n in neighbors ]
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
nodes.append(Node(result[1], addr[0], addr[1]))
spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
return spider.find()
ds = {}
for addr in addrs:
ds[addr] = self.protocol.ping(addr, self.node.id)
return deferredDict(ds).addCallback(initTable)
def inetVisibleIP(self):
"""
Get the internet visible IP's of this node as other nodes see it.
Returns:
A `list` of IP's. If no one can be contacted, then the `list` will be empty.
"""
def handle(results):
ips = [ result[1][0] for result in results if result[0] ]
self.log.debug("other nodes think our ip is %s" % str(ips))
return ips
ds = []
for neighbor in self.bootstrappableNeighbors():
ds.append(self.protocol.stun(neighbor))
return defer.gatherResults(ds).addCallback(handle)
def get(self, key):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
dkey = digest(key)
# if this node has it, return it
if self.storage.get(dkey) is not None:
return defer.succeed(self.storage.get(dkey))
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to get key %s" % key)
return defer.succeed(None)
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find()
def set(self, key, value):
"""
Set the given key to the given value in the network.
"""
self.log.debug("setting '%s' = '%s' on network" % (key, value))
dkey = digest(key)
node = Node(dkey)
def store(nodes):
self.log.info("setting '%s' on %s" % (key, map(str, nodes)))
# if this node is close too, then store here as well
if self.node.distanceTo(node) < max([n.distanceTo(node) for n in nodes]):
self.storage[dkey] = value
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def getKnowNeighbours(self):
"""
Get a list of the known peers in the overlay.
"""
return self.protocol.router.getRoutingNeighbors(self.node)
def _anyRespondSuccess(self, responses):
"""
Given the result of a DeferredList of calls to peers, ensure that at least
one of them was contacted and responded with a Truthy result.
"""
for deferSuccess, result in responses:
peerReached, peerResponse = result
if deferSuccess and peerReached and peerResponse:
return True
return False
def saveState(self, fname):
"""
Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname.
"""
data = { 'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.id,
'neighbors': self.bootstrappableNeighbors() }
if len(data['neighbors']) == 0:
self.log.warning("No known neighbors, so not writing to cache.")
return
with open(fname, 'w') as f:
pickle.dump(data, f)
@classmethod
def loadState(self, fname):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
with open(fname, 'r') as f:
data = pickle.load(f)
s = Server(data['ksize'], data['alpha'], data['id'])
if len(data['neighbors']) > 0:
s.bootstrap(data['neighbors'])
return s
def saveStateRegularly(self, fname, frequency=600):
"""
Save the state of node with a given regularity to the given
filename.
Args:
fname: File name to save retularly to
frequencey: Frequency in seconds that the state should be saved.
By default, 10 minutes.
"""
loop = LoopingCall(self.saveState, fname)
loop.start(frequency)
return loop
class Server(object):
"""
High level view of a node instance. This is the object that should be
created to start listening as an active node on the network.
"""
protocol_class = KademliaProtocol
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
Args:
ksize (int): The k parameter from the paper
alpha (int): The alpha parameter from the paper
node_id: The id for this node on the network.
storage: An instance that implements
:interface:`~kademlia.storage.IStorage`
"""
self.ksize = ksize
self.alpha = alpha
self.storage = storage or ForgetfulStorage()
self.node = Node(node_id or digest(random.getrandbits(255)))
self.transport = None
self.protocol = None
self.refresh_loop = None
self.save_state_loop = None
def stop(self):
if self.transport is not None:
self.transport.close()
if self.refresh_loop:
self.refresh_loop.cancel()
if self.save_state_loop:
self.save_state_loop.cancel()
def _create_protocol(self):
return self.protocol_class(self.node, self.storage, self.ksize)
def listen(self, port, interface='0.0.0.0'):
"""
Start listening on the given port.
Provide interface="::" to accept ipv6 address
"""
loop = asyncio.get_event_loop()
listen = loop.create_datagram_endpoint(self._create_protocol,
local_addr=(interface, port))
log.info("Node %i listening on %s:%i",
self.node.long_id, interface, port)
self.transport, self.protocol = loop.run_until_complete(listen)
# finally, schedule refreshing table
self.refresh_table()
def refresh_table(self):
log.debug("Refreshing routing table")
asyncio.ensure_future(self._refresh_table())
loop = asyncio.get_event_loop()
self.refresh_loop = loop.call_later(3600, self.refresh_table)
async def _refresh_table(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for node_id in self.protocol.getRefreshIDs():
node = Node(node_id)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
ds.append(spider.find())
# do our crawling
await asyncio.gather(*ds)
# now republish keys older than one hour
for dkey, value in self.storage.iteritemsOlderThan(3600):
await self.set_digest(dkey, value)
def bootstrappableNeighbors(self):
"""
Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for
use as an argument to the bootstrap method.
The server should have been bootstrapped
already - this is just a utility for getting some neighbors and then
storing them if this server is going down for a while. When it comes
back up, the list of nodes can be used to bootstrap.
"""
neighbors = self.protocol.router.findNeighbors(self.node)
return [tuple(n)[-2:] for n in neighbors]
async def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP
addresses are acceptable - hostnames will cause an error.
"""
log.debug("Attempting to bootstrap node with %i initial contacts",
len(addrs))
cos = list(map(self.bootstrap_node, addrs))
gathered = await asyncio.gather(*cos)
nodes = [node for node in gathered if node is not None]
spider = NodeSpiderCrawl(self.protocol, self.node, nodes,
self.ksize, self.alpha)
return await spider.find()
async def bootstrap_node(self, addr):
result = await self.protocol.ping(addr, self.node.id)
return Node(result[1], addr[0], addr[1]) if result[0] else None
async def get(self, key):
"""
Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
"""
log.info("Looking up key %s", key)
dkey = digest(key)
# if this node has it, return it
if self.storage.get(dkey) is not None:
return self.storage.get(dkey)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
log.warning("There are no known neighbors to get key %s", key)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
return await spider.find()
async def set(self, key, value):
"""
Set the given string key to the given value in the network.
"""
if not check_dht_value_type(value):
raise TypeError(
"Value must be of type int, float, bool, str, or bytes"
)
log.info("setting '%s' = '%s' on network", key, value)
dkey = digest(key)
return await self.set_digest(dkey, value)
async def set_digest(self, dkey, value):
"""
Set the given SHA1 digest key (bytes) to the given value in the
network.
"""
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
log.warning("There are no known neighbors to set key %s",
dkey.hex())
return False
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
nodes = await spider.find()
log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes)))
# if this node is close too, then store here as well
biggest = max([n.distanceTo(node) for n in nodes])
if self.node.distanceTo(node) < biggest:
self.storage[dkey] = value
ds = [self.protocol.callStore(n, dkey, value) for n in nodes]
# return true only if at least one store call succeeded
return any(await asyncio.gather(*ds))
def saveState(self, fname):
"""
Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname.
"""
log.info("Saving state to %s", fname)
data = {
'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.id,
'neighbors': self.bootstrappableNeighbors()
}
if len(data['neighbors']) == 0:
log.warning("No known neighbors, so not writing to cache.")
return
with open(fname, 'wb') as f:
pickle.dump(data, f)
@classmethod
def loadState(self, fname):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
log.info("Loading state from %s", fname)
with open(fname, 'rb') as f:
data = pickle.load(f)
s = Server(data['ksize'], data['alpha'], data['id'])
if len(data['neighbors']) > 0:
s.bootstrap(data['neighbors'])
return s
def saveStateRegularly(self, fname, frequency=600):
"""
Save the state of node with a given regularity to the given
filename.
Args:
fname: File name to save retularly to
frequency: Frequency in seconds that the state should be saved.
By default, 10 minutes.
"""
self.saveState(fname)
loop = asyncio.get_event_loop()
self.save_state_loop = loop.call_later(frequency,
self.saveStateRegularly,
fname,
frequency)