def poison_rpc_find_node(self, sender, nodeid, key, challenge, signature): source = Node(nodeid, sender[0], sender[1]) # self.maybeTransferKeyValues(source) self.router.addContact(source) node = Node(key) decider = random.random() fakeid = hashlib.sha1(str(random.getrandbits(255))).digest() self.neighbours = map(list, self.router.findNeighbors(Node(fakeid), k=20)) if decider < 0.1: self.poison_routing_tables() elif decider > 0.95: fakeid1 = hashlib.sha1(str(random.getrandbits(255))).digest() fakeid2 = hashlib.sha1(str(random.getrandbits(255))).digest() self.find_value((self.neighbours[0][1], self.neighbours[0][2]), fakeid1, fakeid2, challenge, signature) elif decider > 0.9: self.find_node((self.neighbours[0][1], self.neighbours[0][2]), nodeid, self.neighbours[0][0], challenge, signature) neighbourList = list(self.neighbours) for i in range(0, len(neighbourList)): neighbourList[i] = [ neighbourList[i][0], neighbourList[i][1], self.router.node.port ] mergedlist = [] mergedlist.extend(neighbourList) mergedlist.extend(self.false_neighbour_list) try: signature = self.runtime_credentials.sign_data(challenge) except: _log.debug("signing poison find node failed") return {'bucket': mergedlist, 'signature': signature}
def rpc_find_node(self, sender, nodeid, key): self.log.info("finding neighbors of %i in local table" % long(nodeid.encode('hex'), 16)) source = Node(nodeid, sender[0], sender[1]) self.router.addContact(source) node = Node(key) return map(tuple, self.router.findNeighbors(node, exclude=source))
def initTable(results, challenge, id): nodes = [] for addr, result in results.items(): ip = addr[0] port = addr[1] if result[0]: resultId = result[1]['id'] resultIdHex = resultId.encode('hex').upper() resultSign = result[1]['signature'] data = self.protocol.certificateExists(resultIdHex) if not data: identifier = "{}cert".format(resultIdHex) self.protocol.callCertFindValue( Node(resultId, ip, port), Node(identifier)) else: cert_stored = self.protocol.searchForCertificate( resultIdHex) try: self.runtime_credentials.verify_signed_data_from_certstring( cert_stored, resultSign, challenge, certificate.TRUSTSTORE_TRANSPORT) except: traceback.print_exc() nodes.append(Node(resultId, ip, port)) spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find()
def rpc_find_node(self, sender, nodeid, key): self.log.info("finding neighbors of %i in local table" % long(nodeid.encode('hex'), 16)) source = Node(nodeid, sender[0], sender[1]) _log.debug("rpc_find_node sender=%s, source=%s, key=%s" % (sender, source, base64.b64encode(key))) self.maybeTransferKeyValues(source) self.router.addContact(source) node = Node(key) return map(tuple, self.router.findNeighbors(node, exclude=source))
def test_distanceCalculation(self): ridone = hashlib.sha1(str(random.getrandbits(255))) ridtwo = hashlib.sha1(str(random.getrandbits(255))) shouldbe = long(ridone.hexdigest(), 16) ^ long(ridtwo.hexdigest(), 16) none = Node(ridone.digest()) ntwo = Node(ridtwo.digest()) self.assertEqual(none.distanceTo(ntwo), shouldbe)
def rpc_find_node(self, sender, nodeid, key): log.info("finding neighbors of %i in local table", int(nodeid.hex(), 16)) source = Node(nodeid, sender[0], sender[1]) self.welcome_if_new(source) node = Node(key) neighbors = self.router.find_neighbors(node, exclude=source) return list(map(tuple, neighbors))
def test_distance_calculation(self): # pylint: disable=no-self-use ridone = hashlib.sha1(str(random.getrandbits(255)).encode()) ridtwo = hashlib.sha1(str(random.getrandbits(255)).encode()) shouldbe = int(ridone.hexdigest(), 16) ^ int(ridtwo.hexdigest(), 16) none = Node(ridone.digest()) ntwo = Node(ridtwo.digest()) assert none.distance_to(ntwo) == shouldbe
def poison_routing_tables(self): fakeid = hashlib.sha1(str(random.getrandbits(255))).digest() self.neighbours = map(list, self.router.findNeighbors(Node(fakeid), k=20)) my_randoms = random.sample(xrange(len(self.neighbours)), 1) for nodeToAttack in my_randoms: for nodeToImpersonate in range(0, len(self.neighbours)): if nodeToImpersonate != nodeToAttack: node = Node(self.neighbours[nodeToAttack][0], self.neighbours[nodeToAttack][1], self.neighbours[nodeToAttack][2]) self.callPing(node, self.neighbours[nodeToImpersonate][0])
def test_bucket_basic(self): ecd_key = generate_secret_key() sourceNode = Node(digest(random.getrandbits(255)), ip="127.0.0.1", port=12345) dummy_protocol = DummyProt(ecd_key, sourceNode, None, 4, talos_vc=None) nodes = [] for i in range(1000): nodes.append(Node(digest(random.getrandbits(255)), ip="127.0.0.1", port=i+10000)) for i in range(1000): dummy_protocol.router.addContact(nodes[i]) for i in range(1000): self.assertFalse(dummy_protocol.router.isNewNode(nodes[i]))
def transferKeyValues(self, node): """ Given a new node, send it all the keys/values it should be storing. @param node: A new node that just joined (or that we just found out about). Process: For each key in storage, get k closest nodes. If newnode is closer than the furtherst in that list, and the node for this server is closer than the closest in that list, then store the key/value on the new node (per section 2.5 of the paper) """ ds = [] for key, value in self.storage.iteritems(): keynode = Node(digest(key)) neighbors = self.router.findNeighbors(keynode) if len(neighbors) > 0: newNodeClose = node.distanceTo( keynode) < neighbors[-1].distanceTo(keynode) thisNodeClosest = self.sourceNode.distanceTo( keynode) < neighbors[0].distanceTo(keynode) if len(neighbors) == 0 or (newNodeClose and thisNodeClosest): ds.append(self.callStore(node, key, value)) return defer.gatherResults(ds)
def rpc_store(self, sender, nodeid, key, value): source = Node(nodeid, sender[0], sender[1]) self.welcomeIfNewNode(source) self.log.debug( "got a store request from %s, but THIS VALUE WILL NOT BE STORED as this is a seed-only node." % str( sender)) return True
def set(self, key, value): """ Set the given key to the given value in the network. """ _log.debug("setting '%s' = '%s' on network" % (key, value)) dkey = digest(key) node = Node(dkey) def store(nodes): _log.debug("setting '%s' to %s on %s" % (key, value, map(str, nodes))) # if this node is close too, then store here as well if (not nodes or self.node.distanceTo(node) < max( [n.distanceTo(node) for n in nodes]) or dkey in self.storage): _log.debug("setting '%s' to %s locally" % (key, value)) self.storage[dkey] = value ds = [self.protocol.callStore(n, dkey, value) for n in nodes] return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: _log.warning("There are no known neighbors to set key %s" % key) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find().addCallback(store)
def rpc_find_value(self, sender, nodeid, key): source = Node(nodeid, sender[0], sender[1]) self.router.addContact(source) value = self.storage.get(key, None) if value is None: return self.rpc_find_node(sender, nodeid, key) return { 'value': value }
def __init__(self, ksize=20, alpha=3, priv_key=None, storage=None, talos_vc=None, rebub_delay=3600, c1bits=1, tls_port=-1): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or TalosLevelDBDHTStorage("./leveldb") self.c1bits = c1bits if priv_key is None: self.priv_key, node_id = generate_keys_with_crypto_puzzle(c1bits) else: self.priv_key = priv_key node_id = pub_to_node_id(self.priv_key.public_key()) self.node = Node(node_id) def start_looping_call(num_seconds): self.refreshLoop = LoopingCall(self.refreshTable).start(num_seconds) self.delay = rebub_delay task.deferLater(reactor, rebub_delay, start_looping_call, rebub_delay) self.talos_vc = talos_vc or AsyncPolicyApiClient() self.protocol = TalosSKademliaProtocol(self.priv_key, self.node, self.storage, ksize, talos_vc=self.talos_vc, cbits=c1bits) self.httpprotocol_client = None self.tls_port = tls_port
def __init__(self, ksize=20, alpha=3, id=None, storage=None, talos_vc=None, rebub_delay=3600, tls_port=-1): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or TalosLevelDBDHTStorage("./leveldb") self.node = Node(id or digest(random.getrandbits(255))) def start_looping_call(num_seconds): self.refreshLoop = LoopingCall(self.refreshTable).start(num_seconds) self.delay = rebub_delay task.deferLater(reactor, rebub_delay, start_looping_call, rebub_delay) self.talos_vc = talos_vc or AsyncPolicyApiClient() self.protocol = TalosKademliaProtocol(self.node, self.storage, ksize, talos_vc=self.talos_vc) self.httpprotocol_client = None self.tls_port = tls_port
def get_node_list(self): """ Get the node list in the response. If there's no value, this should be set. """ nodelist = self.response[1] or [] return [Node(*nodeple) for nodeple in nodelist]
def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: nodes.append(Node(result[1], addr[0], addr[1])) spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find()
def welcome_if_new(self, node): """ Given a new node, send it all the keys/values it should be storing, then add it to the routing table. @param node: A new node that just joined (or that we just found out about). Process: For each key in storage, get k closest nodes. If newnode is closer than the furtherst in that list, and the node for this server is closer than the closest in that list, then store the key/value on the new node (per section 2.5 of the paper) """ if not self.router.is_new_node(node): return log.info("never seen %s before, adding to router", node) for key, value in self.storage: keynode = Node(digest(key)) neighbors = self.router.find_neighbors(keynode) if neighbors: last = neighbors[-1].distance_to(keynode) new_node_close = node.distance_to(keynode) < last first = neighbors[0].distance_to(keynode) this_closest = self.source_node.distance_to(keynode) < first if not neighbors or (new_node_close and this_closest): asyncio.ensure_future(self.call_store(node, key, value)) self.router.add_contact(node)
def rpc_append(self, sender, nodeid, key, value): source = Node(nodeid, sender[0], sender[1]) _log.debug("rpc_append sender=%s, source=%s, key=%s, value=%s" % (sender, source, base64.b64encode(key), str(value))) self.maybeTransferKeyValues(source) self.router.addContact(source) try: pvalue = json.loads(value) self.set_keys.add(key) if key not in self.storage: _log.debug("%s append key: %s not in storage set value: %s" % (base64.b64encode(nodeid), base64.b64encode(key), pvalue)) self.storage[key] = value else: old_value_ = self.storage[key] try: old_value = json.loads(old_value_) new_value = list(set(old_value + pvalue)) except: # When the key have been used for single values it does not contain a list # When have been deleted contains None # Just replace old value old_value = old_value_ new_value = pvalue _log.debug("%s append key: %s old: %s add: %s new: %s" % (base64.b64encode(nodeid), base64.b64encode(key), old_value, pvalue, new_value)) self.storage[key] = json.dumps(new_value) return True except: _log.debug("Trying to append something not a JSON coded list %s" % value, exc_info=True) return False
def rpc_append(self, sender, nodeid, key, value): source = Node(nodeid, sender[0], sender[1]) self.router.addContact(source) try: pvalue = json.loads(value) self.set_keys.add(key) if key not in self.storage: _log.debug( "%s append key: %s not in storage set value: %s" % (base64.b64encode(nodeid), base64.b64encode(key), pvalue)) self.storage[key] = value else: old_value_ = self.storage[key] old_value = json.loads(old_value_) new_value = list(set(old_value + pvalue)) _log.debug("%s append key: %s old: %s add: %s new: %s" % (base64.b64encode(nodeid), base64.b64encode(key), old_value, pvalue, new_value)) self.storage[key] = json.dumps(new_value) return True except: _log.debug("Trying to append somthing not a JSON coded list %s" % value, exc_info=True) return False
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None, discovery_mode='neighborhood', loop=None, max_peers=64, dht=None): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` """ self.loop = loop if loop else asyncio.get_event_loop() asyncio.set_event_loop(self.loop) self.ksize = ksize self.alpha = alpha self.port = os.getenv('NETWORK_PORT', 5678) self.storage = storage or ForgetfulStorage() self.node = Node(digest(node_id) or digest(random.getrandbits(255))) self.dht = dht self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None self.max_peers = max_peers self.setup_stethoscope()
def welcomeIfNewNode(self, node): """ Given a new node, send it all the keys/values it should be storing, then add it to the routing table. @param node: A new node that just joined (or that we just found out about). Process: For each key in storage, get k closest nodes. If newnode is closer than the furtherst in that list, and the node for this server is closer than the closest in that list, then store the key/value on the new node (per section 2.5 of the paper) """ if not self.router.isNewNode(node): return self.log.info( "never seen %s before, adding to router and setting nearby " % node) # TODO: 331 and 340 next two lines ursulas = [(id, bytes(node)) for id, node in self.sourceNode._node_storage.items()] for key, value in tuple( self.sourceNode._treasure_maps.items()) + tuple(ursulas): keynode = Node(digest(key)) neighbors = self.router.findNeighbors(keynode) if len(neighbors) > 0: newNodeClose = node.distanceTo( keynode) < neighbors[-1].distanceTo(keynode) thisNodeClosest = self.sourceNode.distanceTo( keynode) < neighbors[0].distanceTo(keynode) if len(neighbors) == 0 or (newNodeClose and thisNodeClosest): asyncio.ensure_future(self.callStore(node, key, value)) self.router.addContact(node)
def digest_set(self, dkey, value): """ Set the given SHA1 digest key to the given value in the network. """ node = Node(dkey) # this is useful for debugging messages hkey = binascii.hexlify(dkey) def store(nodes): self.log.info("setting '%s' on %s" % (hkey, map(str, nodes))) # if this node is close too, then store here as well if self.node.distanceTo(node) < max( [n.distanceTo(node) for n in nodes]): self.storage[dkey] = value ds = [self.protocol.callStore(n, dkey, value) for n in nodes] return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to set key %s" % hkey) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find().addCallback(store)
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None, timeout=5, record=False): """ Create a server instance. This will start listening on the given port. Args: ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper node_id: The id for this node on the network. storage: An instance that implements :interface:`~kademlia.storage.IStorage` timeout: Maximum ping time, initial param for RPCProtocol.wait_timeout """ self.ksize = ksize self.alpha = alpha self.storage = storage or ForgetfulStorage() self.node = Node(node_id or digest(random.getrandbits(255))) self.transport = None self.protocol = None self.refresh_loop = None self.save_state_loop = None self.timeout = timeout self.record = record if self.record == True: print('In record mode:')
def rpc_store(self, sender, nodeid, key, value): source = Node(nodeid, sender[0], sender[1]) self.welcome_if_new(source) log.debug("got a store request from %s, storing '%s'='%s'", sender, key.hex(), value) self.storage[key] = value return True
def transferKeyValues(self, node): """ Given a new node, send it all the keys/values it should be storing. @param node: A new node that just joined (or that we just found out about). Process: For each key in storage, get k closest nodes. If newnode is closer than the furtherst in that list, and the node for this server is closer than the closest in that list, then store the key/value on the new node (per section 2.5 of the paper) """ _log.debug("**** transfer key values %s ****" % node) ds = [] for key, value in self.storage.iteritems(): keynode = Node(digest(key)) neighbors = self.router.findNeighbors(keynode) _log.debug("transfer? nbr neighbors=%d, key=%s, value=%s" % (len(neighbors), base64.b64encode(key), str(value))) if len(neighbors) > 0: newNodeClose = node.distanceTo( keynode) < neighbors[-1].distanceTo(keynode) thisNodeClosest = self.sourceNode.distanceTo( keynode) < neighbors[0].distanceTo(keynode) if len(neighbors) == 0 or (newNodeClose and thisNodeClosest): if key in self.set_keys: _log.debug("transfer append key value key=%s, value=%s" % (base64.b64encode(key), str(value))) ds.append(self.callAppend(node, key, value)) else: _log.debug("transfer store key value key=%s, value=%s" % (base64.b64encode(key), str(value))) ds.append(self.callStore(node, key, value)) return defer.gatherResults(ds)
def rpc_find_value(self, sender, nodeid, key): source = Node(nodeid, sender[0], sender[1]) self.welcome_if_new(source) value = self.storage.get(key, None) if value is None: return self.rpc_find_node(sender, nodeid, key) return {'value': value}
def rpc_remove(self, sender, nodeid, key, value): source = Node(nodeid, sender[0], sender[1]) _log.debug("rpc_remove sender=%s, source=%s, key=%s, value=%s" % (sender, source, base64.b64encode(key), str(value))) self.maybeTransferKeyValues(source) self.router.addContact(source) try: pvalue = json.loads(value) self.set_keys.add(key) if key in self.storage: old_value = json.loads(self.storage[key]) new_value = list(set(old_value) - set(pvalue)) self.storage[key] = json.dumps(new_value) _log.debug("%s remove key: %s old: %s remove: %s new: %s" % (base64.b64encode(nodeid), base64.b64encode(key), old_value, pvalue, new_value)) return True except: _log.debug("Trying to remove somthing not a JSON coded list %s" % value, exc_info=True) return False
async def set_digest(self, dkey, value): """ Set the given SHA1 digest key (bytes) to the given value in the network. """ node = Node(dkey) nearest = self.protocol.router.find_neighbors(node) if not nearest: log.warning("There are no known neighbors to set key %s", dkey.hex()) return False spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) nodes = await spider.find() log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes))) # if this node is close too, then store here as well biggest = max([n.distance_to(node) for n in nodes]) if self.node.distance_to(node) < biggest: self.storage[dkey] = value results = [self.protocol.call_store(n, dkey, value) for n in nodes] # return true only if at least one store call succeeded return any(await asyncio.gather(*results))
def get_concat(self, key): """ Get a key if the network has it. Assuming it is a list that should be combined. @return: C{None} if not found, the value otherwise. """ dkey = digest(key) # Always try to do a find even if we have it, due to the concatenation of all results exists, value = self.storage.get(dkey) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) _log.debug( "Server:get_concat key=%s, value=%s, exists=%s, nbr nearest=%d" % (base64.b64encode(dkey), value, exists, len(nearest))) if len(nearest) == 0: # No neighbors but we had it, return that value if exists: return defer.succeed(value) self.log.warning("There are no known neighbors to get key %s" % key) return defer.succeed(None) spider = ValueListSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha, local_value=value if exists else None) return spider.find()