def republish_keys(self): """ This means retransmitting url and content hashes we're serving for and signing the message. Signing permits peers to relay the message and permits us to republish for our peers. """ data = self.storage.get_entries_for(self.source_node) messages = [] threads = [] # Organise our keys for republishing if data: data = base64.b64encode(json.dumps(data)) hash = SHA256.new(data).digest() signature = app.key.sign(hash, '')[0] messages.append( {'node': [self.source_node.threeple, self.source_node.pubkey], 'keys': {signature: data}} ) # Grab keys we've seen our peers republish for key in self.received_keys: messages.append(self.received_keys[key][-1]) # Tell everyone if messages: dht.log("Republishing keys.") for node in self.router: threads.append(gevent.spawn(self.rpc_republish, node, messages)) gevent.joinall(threads)
def read_envelope(self, data): """ Take an incoming message and either update the last_seen time for the sender or add the sender as a new contact. peers.py should also call this method once it's determined the network a message is for. That way we can inspect the 'peers' attribute and use read_envelope to also learn of new peers. """ # Validate the senders' node ID seed = "%s:%i:%s" % (data['node'][1],data['node'][2],data['pubkey']) if data['node'][0] != long(utils.generate_node_id(seed).encode('hex'), 16): dht.log("\"%s\" is using an incorrect node ID." % data['node'][1], "warning") return # Learn of peers # TODO: Spawn green threads to ping these nodes. # NOTE: Don't permit a spammer to abuse the routing topology. # This can include decrementing the senders trust rating for # referring us to dead nodes. if 'peers' in data: for peer in data['peers']: node = dht.Node(*peer['node'], pubkey=peer['pubkey'], router=self.router) if node != self.source_node and self.router.is_new_node(node): self.router.add_contact(node) # Update last_seen times for contacts or add if new node = dht.Node(*data['node'], pubkey=data['pubkey'], router=self.router) existing_node = self.router.get_existing_node(node) if existing_node: existing_node.last_seen = time.time() return existing_node elif node != self.source_node: self.router.add_contact(node) return node
def rpc_chat(self, nodeple, data): """ Implements CHAT where we encrypt a message destined for the user with UID on the receiving node. Message data should be of the form { 'to': 'uid', 'from': ['uid', 'username'], 'type': Can be any of "message", "init", "close" 'body': {'m':'content'} } """ # Worth retaining this ping call for the routing information we get. node = self.rpc_ping(nodeple) if node == None: return data = base64.b64encode(json.dumps(data)) key = RSA.importKey(node.pubkey) data = key.encrypt(data, 32) data = base64.b64encode(data[0]) response = dht.transmit(self.router, node, {'rpc_chat': data}) dht.log(response, "debug") return response
def decrement_trust(self, addr, severity=1): """ Implements the feedback mechanism for our trust metric. "addr" is an (ip, port) tuple to match to a known peer. "severity" is a floating point severity level indicating how bad the content in question was perceived to be. Notes on how this all works in sum as a distributed system are here: http://nlp.stanford.edu/pubs/eigentrust.pdf http://www.cc.gatech.edu/~lingliu/papers/2012/XinxinFan-EigenTrust++.pdf http://dimacs.rutgers.edu/Workshops/InformationSecurity/slides/gamesandreputation.pdf The second PDF is recommended. """ for node in self.router: if node.ip == addr[0] and node.port == addr[1]: amount = severity / 100.0 dht.log("Decrementing trust rating for %s by %f." % (node, amount), "warning") node.trust -= 2 * self.epsilon # peer = Peer.query.filter( # and_(Peer.network == self.network, # Peer.ip == addr[0], # Peer.port == addr[1]) # ).first() # if peer: # peer.trust -= amount # db.session.add(peer) # db.session.commit() return True return False
def fetch_revision(self, url, content_hash, nodeples): """ Accesses the most trustworthy peers' routing table and see if they have any references to themselves for the desired url/content_hash pair. Updates local download references and then returns the revision. """ urls = [] nodes = [] routers = [] hashed_url = hashlib.sha1(url).digest() revision = Revision.query.filter(Revision.hash == content_hash).first() if revision == None: dht.log("No match for %s could be found in the current database.", "error") for n in nodeples: if n[1] == self.source_node.ip and n[2] == self.source_node.port: continue # Get local references to peer nodes node = self.router.get_existing_node(n) if node: nodes.append(node) continue node = self.rpc_ping(n) if node == None: continue # Get remote peer by its own routing table # TestProtocol.peers is test suite shorthand for all peers for node in dht.sort_nodes_by_trust(nodes): for router in self.peers.values(): if node.threeple == router.node.threeple: routers.append(router) if not any(routers): return None for router in routers: node = self.router.get_existing_node(router.node.threeple) node.trust += self.epsilon references = router.protocol.storage\ .get(binascii.hexlify(hashed_url), None) if not references: continue # downloads is a list of (timestamp, node_triple) pairs. downloads = references.get(content_hash) if not any(downloads): continue for download in downloads: if download[1] == router.node.threeple: self.downloads[url] = {revision.hash: (node.ip, node.port)} return revision return None
def handle_add_friend(self, data): """ Match to UID and return a new Friend instance representing our side. """ assert "rpc_add_friend" in data dht.log(data, "debug") request = data['rpc_add_friend'] if not "from" in request or not "to" in request: return False node = self.read_envelope(data) user = User.query.filter(User.uid == request['to']).first() if not user: return None from_addr = "/".join([self.router.network, str(node.long_id), request['from']]) friend = Friend.query.filter( and_(Friend.address == from_addr, Friend.user == user) ).first() if friend: # This permits the remote side to see if they're added or blocked. return dht.envelope(self.router, {"response": friend.jsonify()}) node = dht.Node(*data['node']) network = Network.query.filter(Network.name == self.router.network).first() if network != None: network = Network(name = self.router.network) peer = Peer.query.filter( and_(Peer.network == network, Peer.ip == node.ip, Peer.port == node.port) ).first() if peer == None: peer = Peer() peer.load_node(node) peer.network = network friend = Friend(address=from_addr) friend.state = 1 friend.received = True # TODO: Make this correspond to the existing rpc_friend method. user.friends.append(friend) peer.friends.append(friend) db.session.add(user) db.session.add(peer) db.session.add(friend) db.session.add(network) db.session.commit() return dht.envelope(self.router, {"response": friend.jsonify()})
def handle_find_node(self, data): """ Used for finding existing nodes near to a target ID. """ source = self.read_envelope(data) if not 'key' in data: return "No target key specified.", 400 node = dht.Node(data['key']) dht.log("Finding neighbours of %s." % node.long_id) nodes = {'nodes': [p.jsonify() for p in \ self.router.find_neighbours(node, exclude=source)]} return dht.envelope(self.router, nodes)
def create_peers(peer_amount, ksize, storage_method): peers = {} for i in range(peer_amount): peers[i] = dht.RoutingTable( "127.0.0.1", random.randint(0, 999999), app.key.publickey().exportKey(), None, ) peers[i].buckets = [dht.KBucket(0, 2**160, 20)] rpcmethod = getattr(peers[i].protocol, storage_method, None) if not rpcmethod: raise Exception("Unknown storage method: %s" % storage_method) peers[i].storage_method = rpcmethod # Our TestProtocol is a subclass that calls methods on peer routing # tables directly instead of making network calls and database commits. peers[i].protocol = TestProtocol( peers[i], dht.Storage(), peers[i].ksize, peers ) # Attempted pings in add_contact would cause some previously # added peers to be promptly removed. We manually swap the method # for a mockup and reintroduce the original once we have our set of peers. peers[i].protocol.original_ping = peers[i].protocol.rpc_ping peers[i].protocol.rpc_ping = mock_ping log = dht.log # We check for unique port numbers because addr is /usually/ an (ip, port) # tuple when calling dht.transmit. ports = [p.node.port for p in peers.values()] unique_ports = len(set(ports)) == len(peers.keys()) dht.log("Unique port numbers: %s" % str("Yes." if unique_ports else "No. Recreating.")) if not unique_ports: return create_peers(peer_amount, storage_method) dht.log("Introducing peers to one another.") dht.log = lambda x, y=None: x for peer in peers.values(): [peer.add_contact(router.node) for router in peers.values()] dht.log = log print pprint.pformat(peers) # Please god, forgive this fourth loop? for p in peers.values(): peer.protocol.rpc_ping = peer.protocol.original_ping return peers
def handle_republish(self, data): """ Retain signed messages here so they can be relayed. """ node = self.read_envelope(data) if max(node.trust, 0) == 0: dht.log("%s with negative trust rating tried to republish." % node, "warning") return False dht.log("Received rpc_republish from %s." % node) republished_keys = data['rpc_republish'] for message in republished_keys: if max(node.trust, 0) == 0: return False signature = (long(message['keys'].keys()[0]),) data = message['keys'].values()[0] hash = SHA256.new(data).digest() key = RSA.importKey(message['node'][1]) if not key.verify(hash, signature): dht.log("Invalid signatures for keys provided by %s." % node, "warning") node.trust -= self.epsilon continue try: keys = json.loads(base64.b64decode(data)) except Exception, e: dht.log("Error unserialising republished keys: %s" % e.message, "error") continue referee = dht.Node(*message['node'][0], pubkey=message['node'][1]) if self.router.is_new_node(referee): self.rpc_ping(referee) # Get the trust rating of this referee referee = self.router.get_existing_node(referee) if not referee or referee.trust < 0: dht.log("%s is currently republishing for %s." % (node, referee), "warning") continue self.storage.merge(keys) self.received_keys[referee.id] = message
def delete(self, network): """ Remove a network. """ user = auth(session, required=True) routes = app.routes.get(network, None) if routes == None: return {}, 404 if not user.can("manage_networks"): return {}, 403 log("%s is removing network \"%s\"." % (user.username, network)) app.routes.leave(network) # Remove the database representation, too network_in_db = Network.query.filter(Network.name == network).first() if network_in_db != None: network_in_db.delete()
def setUp(self): print "\nCreating %i peers configured to use %s." % \ (self.peer_amount, self.storage_method) self.peers = create_peers(self.peer_amount, self.ksize, self.storage_method) count = int(len(self.peers) * self.dfp) # dht.log("%i peers will be malicious for this test." % count) # for i in range(count): # self.peers[i] ... # All remaining peers considered honest are automatically # added to one anothers' bucket of pre-trusted peers. # # Your test suite(s) will want to adjust this manually after the fact. self.honest_peers = {} honest_count = len(self.peers) - count dht.log("Creating %i pre-trusted peers." % honest_count) for j in range(honest_count): self.honest_peers[self.peers[count+j].node.long_id] = self.peers[count+j] for router in self.honest_peers.values(): for r in self.honest_peers.values(): if router.node.threeple == r.node.threeple: continue node = router.get_existing_node(r.node.threeple) if not node: continue node.trust += router.protocol.epsilon router.tbucket[node.long_id] = node # dht.log("Introduced %s to %s as a pre-trusted peer." % (node, router)) # We add these RoutingTable objects as an attributes of mocked methods # so they can find other nodes and work on their protocol instances. # # Even though we're emulating dht.SynchronyProtocol to avoid network # and database calls, we still stick our mock functions on those instances # for the time being. mock_transmit.peers = self.peers dht.transmit = mock_transmit mock_get.peers = self.peers dht.get = mock_get
def post(self, network): """ Hosts here is a comma-seperated list of ip:port pairs. """ user = auth(session, required=True) parser = restful.reqparse.RequestParser() parser.add_argument("hosts", type=str) args = parser.parse_args() routes = app.routes.get(network, None) if routes == None: return {}, 404 if not user.can("manage_networks"): return {}, 403 # Get hosts as a list of "ip:port" strings hosts = args.hosts.replace(" ", "").split(',') def tuplify(host): if not ':' in host: return host = host.split(':') return tuple([host[0], int(host[1])]) hosts = [tuplify(h) for h in hosts] # Emulate RoutingTable.bootstrap nodes = [] for host in hosts: if host == None: continue log("Pinging %s:%i" % host) nodes.append(routes.protocol.rpc_ping(host)) response = [] for node in nodes: if node: response.append(node.jsonify(string_id=True)) return response
def put(self): user = auth(session, required=True) parser = restful.reqparse.RequestParser() parser.add_argument("name", type=str, required=True) args = parser.parse_args() if not user.can("manage_networks"): return {}, 403 if app.routes.get(args.name, None): return {}, 409 log('%s is creating a network named "%s".' % (user.username, args.name)) # Derive settings from the default routing table settings = app.routes._default pubkey = app.key.publickey().exportKey() ip = settings.node.ip port = settings.node.port router = RoutingTable(ip, port, pubkey, settings.httpd, network=args.name) app.routes.append(router) return router.jsonify()
def rpc_add_friend(self, local_uid, addr): """ addr is of the form "network_name/node_id/remote_user_id" Implements ADD_FRIEND where we find the node in addr and tell them a local user wants to add the remote UID as a friend. """ if addr.count("/") != 2: return False, None network, node_id, remote_uid = addr.split("/") if network != self.router.network: return False, None node = dht.Node(long(node_id)) nearest = self.router.find_neighbours(node) if len(nearest) == 0: dht.log("There are no neighbours to help us add users on %s as friends." % node_id) return False, None spider = NodeSpider(self, node, nearest, self.ksize, self.router.alpha) nodes = spider.find() if len(nodes) != 1: return False, None node = nodes[0] # Sometimes spidering doesn't get us all the way there. # Check who we already know: if node.long_id != long(node_id): nodes = [n for n in self.router if n.long_id == long(node_id)] if len(nodes) != 1: return False, None node = nodes[0] dht.log(node_id, "debug") dht.log(node.long_id, "debug") dht.log("Found remote instance %s." % node) message = {"rpc_add_friend": {"from": local_uid, "to": remote_uid}} response = dht.transmit(self.router, node, message) if not isinstance(response, dict) or not "response" in response: return False, None return response['response'], node
def handle_chat(self, data): """ Move a message from a remote node up to the UI if the recipient UID has an active connection to the chat stream. """ node = self.read_envelope(data) # With the ciphertext being a binary string we also b64encode it message_content = base64.b64decode(data['rpc_chat']) message_content = app.key.decrypt((message_content,)) data = json.loads(base64.b64decode(message_content)) dht.log(message_content, "debug") dht.log(data, "debug") user = User.query.filter(User.uid == data['to']).first() if user == None: return {"error": "No such user."} friend = Friend.query.filter(and_(Friend.user == user, Friend.network == self.router.network, Friend.node_id == str(node.long_id), Friend.uid == data['from'][0]) ).first() if friend: available = utils.check_availability(self.router.httpd, "chat", user) if not available: return {"error": "The intended recipient isn't connected to chat."} if data['type'] == "init": # Enable the recipient to reply by forcing them into the channel dht.log("Changing chat channel of %s to %s." % \ (user.username, friend.address), "debug") utils.change_channel(self.router.httpd, "chat", user, friend.address) utils.broadcast(self.router.httpd, "chat", "rpc_chat_init", data['from'], user=user) if data['type'] == "message": utils.broadcast(self.router.httpd, "chat", "rpc_chat", data, user=user) return {"state": "delivered"}
def handle_find_value(self, data): source = self.read_envelope(data) if not source: return if not 'rpc_find_value' in data: return # usually comes in as unicode if not isinstance(data['rpc_find_value'], (unicode, str)): return key = data['rpc_find_value'] dht.log("Finding value for %s" % key) value = self.storage.get(key, None) if key is None: dht.log("No value found for %s" % key) return self.rpc_find_node(sender, nodeid, key) dht.log("Found %s" % value) return dht.envelope(self.router,{'value':value})
def mock_transmit(routes, addr, data): """ Put dht.RoutingTable instances through to one another without calling out to the network. """ # Test case setup method should set a peers attr on this function beforehand if not hasattr(mock_transmit, "peers"): dht.log("Can't find test peers.") dht.log("synchrony.test.utils.mock_transmit is missing a peers dictionary.") return if isinstance(addr, dht.Node): addr = (addr.ip, addr.port) # Filter for everyone who isn't the intended recipient peer_routes = filter( lambda r: r if r.node.port == addr[1] else None, [r for r in mock_transmit.peers.values()] ) if not peer_routes: dht.log("Unknown peer %s:%i" % addr) return peer_routes = peer_routes[0] data = dht.envelope(routes, data) for field in data.keys(): if field.startswith('rpc_'): rpc_name = 'handle_%s' % field.replace('rpc_', '') rpc_method = getattr(peer_routes.protocol, rpc_name, None) if not rpc_method: dht.log("%s tried to call unknown procedure %s." % \ (routes.node, rpc_name), "warning") return return rpc_method(data)
def handle_append(self, data): """ Handle messages of the form {'rpc_append': {'url_hash': 'content_hash'}} We do this by inserting the data into a structure that looks like { 'url_hash': {'content_hash': [(ts,nodeple)]}} """ node = self.read_envelope(data) if max(node.trust, 0) == 0: dht.log("%s with negative trust rating tried to append." % node, "warning") return False url_hash, content_hash = data['rpc_append'].items()[0] dht.log("Received rpc_append request from %s." % node) dht.log("Adjusting known peers for %s." % url_hash) self.storage[url_hash] = (content_hash, node) # if self.router.options and self.router.options.autoreplicate: # if not Revision.query.filter(Revision.hash == content_hash).first(): # revision = self.fetch_revision(content_hash, [source]) # if revision: # db.session.add(revision) # db.session.commit() return True
def handle_ping(self, data): node = self.read_envelope(data) dht.log("Received rpc_ping from %s." % node) return dht.envelope(self.router, {'ping':"pong"})